counterweight/data-dwh-dbt-project
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merged PR 1681: Xero Raw

Browse source 
First stuff coming in from Xero. Just contacts and invoices for now since it's the most critical for business overview at this point.
This commit is contained in:
Pablo Martín 2024-04-17 15:10:53 +00:00
commit 8be9e700bb
4 changed files with 467 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

17
models/staging/xero/_xero_sources.yml Normal file
View file

@ -0,0 +1,17 @@
version: 2
sources:
- name: xero
description: Data from our Xero accounting system (tenant is Superhog Limited).
schema: sync_xero_superhog_limited
tables:
- name: contacts
identifier: contacts
description: |
Xero contacts identify other parties we transact with: customers, suppliers, employees, etc.
The most interesting contacts are our customers.
- name: invoices
identifier: invoices
description: |
Invoices that we have raised within our accounting books.

353
models/staging/xero/schema.yml Normal file
View file

@ -0,0 +1,353 @@
version: 2
models:
- name: stg_xero__contacts
description: |
Xero contacts identify other parties we transact with: customers, suppliers, employees, etc.
The most interesting contacts are our customers.
columns:
- name: id_contact
data_type: character varying
description: Xero's unique identifier for the contact.
- name: account_number
data_type: character varying
description: A Superhog set identifier. For customers, this is Hubspot's Deal Id.
- name: contact_name
data_type: character varying
description: The name for the customer.
- name: is_customer
data_type: boolean
description: Flag that shows if the contact is a Superhog customer.
- name: is_supplier
data_type: boolean
description: Flag that shows if the contact is a Superhog supplier.
- name: tax_number
data_type: character varying
description: Tax number for this contact.
- name: contact_first_name
data_type: character varying
description: ""
- name: contact_last_name
data_type: character varying
description: ""
- name: phones
data_type: jsonb
description: ""
- name: website
data_type: character varying
description: ""
- name: balances
data_type: jsonb
description: ""
- name: discount
data_type: numeric
description: ""
- name: addresses
data_type: jsonb
description: ""
- name: attachments
data_type: jsonb
description: ""
- name: email_addresses
data_type: character varying
description: ""
- name: payment_terms
data_type: jsonb
description: ""
- name: batch_payments
data_type: jsonb
description: ""
- name: branding_theme
data_type: jsonb
description: ""
- name: contact_groups
data_type: jsonb
description: ""
- name: contact_number
data_type: character varying
description: ""
- name: contact_status
data_type: character varying
description: ""
- name: skyper_user_name
data_type: character varying
description: ""
- name: contact_persons
data_type: jsonb
description: ""
- name: has_attachments
data_type: boolean
description: ""
- name: updated_at_utc
data_type: timestamp with time zone
description: ""
- name: updated_date_utc
data_type: date
description: ""
- name: xero_network_key
data_type: character varying
description: ""
- name: default_currency_iso_4217
data_type: character varying
description: |
The default currency used to interact with this contact.
For customers, this is the currency we will usually use to invoice.
- name: validation_errors
data_type: jsonb
description: ""
- name: bank_account_details
data_type: character varying
description: ""
- name: has_validation_errors
data_type: boolean
description: ""
- name: tracking_category_name
data_type: character varying
description: ""
- name: account_payable_tax_type
data_type: character varying
description: ""
- name: tracking_category_option
data_type: character varying
description: ""
- name: sales_default_account_code
data_type: character varying
description: ""
- name: sales_tracking_categories
data_type: jsonb
description: ""
- name: account_receivable_tax_type
data_type: character varying
description: ""
- name: purchases_default_account_code
data_type: character varying
description: ""
- name: purchases_tracking_categories
data_type: jsonb
description: ""
- name: dwh_extracted_at_utc
data_type: timestamp with time zone
description: ""
- name: stg_xero__invoices
description: |
Invoices that we have raised within our accounting books.
This includes both our invoices towards customers and invoices provided by our suppliers.
You can read more here: https://developer.xero.com/documentation/api/accounting/invoices
columns:
- name: id_invoice
data_type: character varying
description: Xero's unique identifier for the invoice.
- name: invoice_url
data_type: character varying
description: ""
- name: invoice_issued_date_utc
data_type: timestamp with time zone
description: Date on which the invoice was issued.
- name: invoice_type
data_type: character varying
description: |
This field indicates whether the invoice is from Superhog towards a customer
(value is "ACCREC") or from a supplier towards Superhog (value is "ACCPAY").
- name: total_amount_local_curr
data_type: numeric
description: The total amount to be paid, in the currency the invoice is denominated in.
- name: total_amount_wo_tax_local_curr
data_type: numeric
description: The total amount to be paid minus taxes, in the currency the invoice is denominated in.
- name: total_tax_local_curr
data_type: numeric
description: The total tax, in the currency the invoice is denominated in.
- name: total_due_local_curr
data_type: numeric
description: The total amount outstanding right now, in the currency the invoice is denominated in.
- name: total_paid_local_curr
data_type: numeric
description: The total amount that has already been paid, in the currency the invoice is denominated in.
- name: invoice_status
data_type: character varying
description: |
The status of the invoice.
Can be one of: PAID, VOIDED, DRAFT, DELETED, AUTHORISED, SUBMITTED.
You can read more here: https://developer.xero.com/documentation/api/accounting/types#invoices
- name: contact
data_type: jsonb
description: |
The contact related to this invoice.
The customer if it's an Accounts Receivables one, the supplier if it's an Accounts Payables one.
- name: invoice_due_date_utc
data_type: timestamp with time zone
description: The date were the invoice should be paid the latest.
- name: payments
data_type: jsonb
description: |
An array with the details of the payments that are satisfying this invoice.
- name: has_errors
data_type: boolean
description: ""
- name: line_items
data_type: jsonb
description: An array with all the line items and their details.
- name: reference
data_type: character varying
description: |
The Superhog-set reference for the invoices. Only relevant for
records of invoice type (ACCREC).
- name: date_string
data_type: timestamp with time zone
description: ""
- name: credit_notes
data_type: jsonb
description: An array with any related credit notes, along with all their details.
- name: prepayments
data_type: jsonb
description: ""
- name: invoice_currency_iso_4217
data_type: character varying
description: The ISO 4217 code for the currency which this invoice uses as unit of account.
- name: exchange_rate_to_gbp
data_type: numeric
description: |
The implied exchange rate between the invoice currency and GBP.
This is the XXX to GBP rate. So the number represented in this field is the units of currency GBP
that 1 unit of XXX buys you.
For example, if the invoice is 135 ZAR, and this rate is 0.0167, the invoice value in GBP is 2.25
(135ZAR * 0.0167GBP/ZAR = 2.25GBP).
- name: is_discounted
data_type: boolean
description: ""
- name: overpayments
data_type: jsonb
description: ""
- name: due_date_string
data_type: timestamp with time zone
description: ""
- name: invoice_number
data_type: character varying
description: ""
- name: has_been_sent_to_contact
data_type: boolean
description: ""
- name: total_discount
data_type: numeric
description: ""
- name: total_credited_local_curr
data_type: numeric
description:
Total amount credited by related credit notes, over-payments and
pre-payments.
- name: has_attachments
data_type: boolean
description: Flag indicating if the invoice has an attahcment.
- name: updated_at_utc
data_type: timestamp with time zone
description: ""
- name: updated_date_utc
data_type: date
description: ""
- name: id_branding_theme
data_type: character varying
description: ""
- name: was_fully_paid_on_date_utc
data_type: timestamp with time zone
description: |
The date on which the invoice was fully paid. If the invoice is still
not fully paid, it returns null.
- name: line_amount_tax_inclusiveness
data_type: character varying
description: |
Indicates whether the amounts included in line items are tax
inclusive (Inclusive), tax exclusive (Exclusive) or simply,
there are no taxes on the invoice (NoTax).
- name: payment_planned_date_utc
data_type: timestamp with time zone
description: ""
- name: payment_expected_date_utc
data_type: timestamp with time zone
description: ""
- name: dwh_extracted_at_utc
data_type: timestamp with time zone
description: ""

52
models/staging/xero/stg_xero__contacts.sql Normal file
View file

@ -0,0 +1,52 @@
with
raw_contacts as (select * from {{ source("xero", "contacts") }}),
stg_xero__contacts as (
select
{{ adapter.quote("ContactID") }} as id_contact,
{{ adapter.quote("AccountNumber") }} as account_number,
{{ adapter.quote("Name") }} as contact_name,
{{ adapter.quote("IsCustomer") }} as is_customer,
{{ adapter.quote("IsSupplier") }} as is_supplier,
{{ adapter.quote("TaxNumber") }} as tax_number,
{{ adapter.quote("FirstName") }} as contact_first_name,
{{ adapter.quote("LastName") }} as contact_last_name,
{{ adapter.quote("Phones") }} as phones,
{{ adapter.quote("Website") }} as website,
{{ adapter.quote("Balances") }} as balances,
{{ adapter.quote("Discount") }} as discount,
{{ adapter.quote("Addresses") }} as addresses,
{{ adapter.quote("Attachments") }} as attachments,
{{ adapter.quote("EmailAddress") }} as email_addresses,
{{ adapter.quote("PaymentTerms") }} as payment_terms,
{{ adapter.quote("BatchPayments") }} as batch_payments,
{{ adapter.quote("BrandingTheme") }} as branding_theme,
{{ adapter.quote("ContactGroups") }} as contact_groups,
{{ adapter.quote("ContactNumber") }} as contact_number,
{{ adapter.quote("ContactStatus") }} as contact_status,
{{ adapter.quote("SkypeUserName") }} as skyper_user_name,
{{ adapter.quote("ContactPersons") }} as contact_persons,
{{ adapter.quote("HasAttachments") }} as has_attachments,
{{ adapter.quote("UpdatedDateUTC") }} as updated_at_utc,
cast({{ adapter.quote("UpdatedDateUTC") }} as date) as updated_date_utc,
{{ adapter.quote("XeroNetworkKey") }} as xero_network_key,
{{ adapter.quote("DefaultCurrency") }} as default_currency_iso_4217,
{{ adapter.quote("ValidationErrors") }} as validation_errors,
{{ adapter.quote("BankAccountDetails") }} as bank_account_details,
{{ adapter.quote("HasValidationErrors") }} as has_validation_errors,
{{ adapter.quote("TrackingCategoryName") }} as tracking_category_name,
{{ adapter.quote("AccountsPayableTaxType") }} as account_payable_tax_type,
{{ adapter.quote("TrackingCategoryOption") }} as tracking_category_option,
{{ adapter.quote("SalesDefaultAccountCode") }}
as sales_default_account_code,
{{ adapter.quote("SalesTrackingCategories") }} as sales_tracking_categories,
{{ adapter.quote("AccountsReceivableTaxType") }}
as account_receivable_tax_type,
{{ adapter.quote("PurchasesDefaultAccountCode") }}
as purchases_default_account_code,
{{ adapter.quote("PurchasesTrackingCategories") }}
as purchases_tracking_categories,
{{ adapter.quote("_airbyte_extracted_at") }} as dwh_extracted_at_utc
from raw_contacts
)
select *
from stg_xero__contacts

45
models/staging/xero/stg_xero__invoices.sql Normal file
View file

@ -0,0 +1,45 @@
with
raw_invoices as (select * from {{ source("xero", "invoices") }}),
stg_xero__invoices as (
select
{{ adapter.quote("InvoiceID") }} as id_invoice,
{{ adapter.quote("Reference") }} as reference,
{{ adapter.quote("InvoiceNumber") }} as invoice_number,
{{ adapter.quote("Date") }} as invoice_issued_date_utc,
{{ adapter.quote("DueDate") }} as invoice_due_date_utc,
{{ adapter.quote("FullyPaidOnDate") }} as was_fully_paid_on_date_utc,
{{ adapter.quote("Type") }} as invoice_type,
{{ adapter.quote("CurrencyCode") }} as invoice_currency_iso_4217,
1 / {{ adapter.quote("CurrencyRate") }} as exchange_rate_to_gbp,
{{ adapter.quote("Total") }} as total_amount_local_curr,
{{ adapter.quote("SubTotal") }} as total_amount_wo_tax_local_curr,
{{ adapter.quote("TotalTax") }} as total_tax_local_curr,
{{ adapter.quote("AmountDue") }} as total_due_local_curr,
{{ adapter.quote("AmountPaid") }} as total_paid_local_curr,
{{ adapter.quote("Status") }} as invoice_status,
{{ adapter.quote("Contact") }} as contact,
{{ adapter.quote("Payments") }} as payments,
{{ adapter.quote("HasErrors") }} as has_errors,
{{ adapter.quote("LineItems") }} as line_items,
{{ adapter.quote("CreditNotes") }} as credit_notes,
{{ adapter.quote("Prepayments") }} as prepayments,
{{ adapter.quote("Overpayments") }} as overpayments,
{{ adapter.quote("IsDiscounted") }} as is_discounted,
{{ adapter.quote("DateString") }} as date_string,
{{ adapter.quote("DueDateString") }} as due_date_string,
{{ adapter.quote("SentToContact") }} as has_been_sent_to_contact,
{{ adapter.quote("TotalDiscount") }} as total_discount,
{{ adapter.quote("AmountCredited") }} as total_credited_local_curr,
{{ adapter.quote("HasAttachments") }} as has_attachments,
{{ adapter.quote("UpdatedDateUTC") }} as updated_at_utc,
cast({{ adapter.quote("UpdatedDateUTC") }} as date) as updated_date_utc,
{{ adapter.quote("BrandingThemeID") }} as id_branding_theme,
{{ adapter.quote("Url") }} as invoice_url,
{{ adapter.quote("LineAmountTypes") }} as line_amount_tax_inclusiveness,
{{ adapter.quote("PlannedPaymentDate") }} as payment_planned_date_utc,
{{ adapter.quote("ExpectedPaymentDate") }} as payment_expected_date_utc,
{{ adapter.quote("_airbyte_extracted_at") }} as dwh_extracted_at_utc
from raw_invoices
)
select *
from stg_xero__invoices