From cf0f1f9a0c57777f503698af33facbd832892821 Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Tue, 16 Apr 2024 15:41:23 +0200 Subject: [PATCH 1/4] start source with contacts --- models/staging/xero/_xero_sources.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 models/staging/xero/_xero_sources.yml diff --git a/models/staging/xero/_xero_sources.yml b/models/staging/xero/_xero_sources.yml new file mode 100644 index 0000000..12bca25 --- /dev/null +++ b/models/staging/xero/_xero_sources.yml @@ -0,0 +1,13 @@ +version: 2 + +sources: + - name: xero + description: Data from our Xero accounting system (tenant is Superhog Limited). + schema: sync_xero_superhog_limited + tables: + - name: contacts + identifier: contacts + description: | + Xero contacts identify other parties we transact with: customers, suppliers, employees, etc. + + The most interesting contacts are our customers. From 7d79dc8e4b7ad5aa71a1469973f1cf72324131e1 Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Tue, 16 Apr 2024 16:07:25 +0200 Subject: [PATCH 2/4] contacts model and schema --- models/staging/xero/schema.yml | 171 +++++++++++++++++++++ models/staging/xero/stg_xero__contacts.sql | 52 +++++++ 2 files changed, 223 insertions(+) create mode 100644 models/staging/xero/schema.yml create mode 100644 models/staging/xero/stg_xero__contacts.sql diff --git a/models/staging/xero/schema.yml b/models/staging/xero/schema.yml new file mode 100644 index 0000000..ec6c99e --- /dev/null +++ b/models/staging/xero/schema.yml @@ -0,0 +1,171 @@ +version: 2 + +models: + - name: stg_xero__contacts + description: | + Xero contacts identify other parties we transact with: customers, suppliers, employees, etc. + + The most interesting contacts are our customers. + columns: + - name: id_contact + data_type: character varying + description: Xero's unique identifier for the contact. + + - name: account_number + data_type: character varying + description: A Superhog set identifier. For customers, this is Hubspot's Deal Id. + + - name: contact_name + data_type: character varying + description: The name for the customer. + + - name: is_customer + data_type: boolean + description: Flag that shows if the contact is a Superhog customer. + + - name: is_supplier + data_type: boolean + description: Flag that shows if the contact is a Superhog supplier. + + - name: tax_number + data_type: character varying + description: Tax number for this contact. + + - name: contact_first_name + data_type: character varying + description: "" + + - name: contact_last_name + data_type: character varying + description: "" + + - name: phones + data_type: jsonb + description: "" + + - name: website + data_type: character varying + description: "" + + - name: balances + data_type: jsonb + description: "" + + - name: discount + data_type: numeric + description: "" + + - name: addresses + data_type: jsonb + description: "" + + - name: attachments + data_type: jsonb + description: "" + + - name: email_addresses + data_type: character varying + description: "" + + - name: payment_terms + data_type: jsonb + description: "" + + - name: batch_payments + data_type: jsonb + description: "" + + - name: branding_theme + data_type: jsonb + description: "" + + - name: contact_groups + data_type: jsonb + description: "" + + - name: contact_number + data_type: character varying + description: "" + + - name: contact_status + data_type: character varying + description: "" + + - name: skyper_user_name + data_type: character varying + description: "" + + - name: contact_persons + data_type: jsonb + description: "" + + - name: has_attachments + data_type: boolean + description: "" + + - name: updated_at_utc + data_type: timestamp with time zone + description: "" + + - name: updated_date_utc + data_type: date + description: "" + + - name: xero_network_key + data_type: character varying + description: "" + + - name: default_currency_iso_4217 + data_type: character varying + description: | + The default currency used to interact with this contact. + + For customers, this is the currency we will usually use to invoice. + + - name: validation_errors + data_type: jsonb + description: "" + + - name: bank_account_details + data_type: character varying + description: "" + + - name: has_validation_errors + data_type: boolean + description: "" + + - name: tracking_category_name + data_type: character varying + description: "" + + - name: account_payable_tax_type + data_type: character varying + description: "" + + - name: tracking_category_option + data_type: character varying + description: "" + + - name: sales_default_account_code + data_type: character varying + description: "" + + - name: sales_tracking_categories + data_type: jsonb + description: "" + + - name: account_receivable_tax_type + data_type: character varying + description: "" + + - name: purchases_default_account_code + data_type: character varying + description: "" + + - name: purchases_tracking_categories + data_type: jsonb + description: "" + + - name: dwh_extracted_at_utc + data_type: timestamp with time zone + description: "" \ No newline at end of file diff --git a/models/staging/xero/stg_xero__contacts.sql b/models/staging/xero/stg_xero__contacts.sql new file mode 100644 index 0000000..f006e6e --- /dev/null +++ b/models/staging/xero/stg_xero__contacts.sql @@ -0,0 +1,52 @@ +with + raw_contacts as (select * from {{ source("xero", "contacts") }}), + stg_xero__contacts as ( + select + {{ adapter.quote("ContactID") }} as id_contact, + {{ adapter.quote("AccountNumber") }} as account_number, + {{ adapter.quote("Name") }} as contact_name, + {{ adapter.quote("IsCustomer") }} as is_customer, + {{ adapter.quote("IsSupplier") }} as is_supplier, + {{ adapter.quote("TaxNumber") }} as tax_number, + {{ adapter.quote("FirstName") }} as contact_first_name, + {{ adapter.quote("LastName") }} as contact_last_name, + {{ adapter.quote("Phones") }} as phones, + {{ adapter.quote("Website") }} as website, + {{ adapter.quote("Balances") }} as balances, + {{ adapter.quote("Discount") }} as discount, + {{ adapter.quote("Addresses") }} as addresses, + {{ adapter.quote("Attachments") }} as attachments, + {{ adapter.quote("EmailAddress") }} as email_addresses, + {{ adapter.quote("PaymentTerms") }} as payment_terms, + {{ adapter.quote("BatchPayments") }} as batch_payments, + {{ adapter.quote("BrandingTheme") }} as branding_theme, + {{ adapter.quote("ContactGroups") }} as contact_groups, + {{ adapter.quote("ContactNumber") }} as contact_number, + {{ adapter.quote("ContactStatus") }} as contact_status, + {{ adapter.quote("SkypeUserName") }} as skyper_user_name, + {{ adapter.quote("ContactPersons") }} as contact_persons, + {{ adapter.quote("HasAttachments") }} as has_attachments, + {{ adapter.quote("UpdatedDateUTC") }} as updated_at_utc, + cast({{ adapter.quote("UpdatedDateUTC") }} as date) as updated_date_utc, + {{ adapter.quote("XeroNetworkKey") }} as xero_network_key, + {{ adapter.quote("DefaultCurrency") }} as default_currency_iso_4217, + {{ adapter.quote("ValidationErrors") }} as validation_errors, + {{ adapter.quote("BankAccountDetails") }} as bank_account_details, + {{ adapter.quote("HasValidationErrors") }} as has_validation_errors, + {{ adapter.quote("TrackingCategoryName") }} as tracking_category_name, + {{ adapter.quote("AccountsPayableTaxType") }} as account_payable_tax_type, + {{ adapter.quote("TrackingCategoryOption") }} as tracking_category_option, + {{ adapter.quote("SalesDefaultAccountCode") }} + as sales_default_account_code, + {{ adapter.quote("SalesTrackingCategories") }} as sales_tracking_categories, + {{ adapter.quote("AccountsReceivableTaxType") }} + as account_receivable_tax_type, + {{ adapter.quote("PurchasesDefaultAccountCode") }} + as purchases_default_account_code, + {{ adapter.quote("PurchasesTrackingCategories") }} + as purchases_tracking_categories, + {{ adapter.quote("_airbyte_extracted_at") }} as dwh_extracted_at_utc + from raw_contacts + ) +select * +from stg_xero__contacts From 2a2fa613ee0f1a6a1d12b0b5cb9bd9a3ecc66d20 Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Tue, 16 Apr 2024 16:27:44 +0200 Subject: [PATCH 3/4] add invoices to sources --- models/staging/xero/_xero_sources.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/models/staging/xero/_xero_sources.yml b/models/staging/xero/_xero_sources.yml index 12bca25..eec9672 100644 --- a/models/staging/xero/_xero_sources.yml +++ b/models/staging/xero/_xero_sources.yml @@ -11,3 +11,7 @@ sources: Xero contacts identify other parties we transact with: customers, suppliers, employees, etc. The most interesting contacts are our customers. + - name: invoices + identifier: invoices + description: | + Invoices that we have raised within our accounting books. From 8ab6d27241a7795ed02be720d418f2a381bb7203 Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Wed, 17 Apr 2024 17:05:08 +0200 Subject: [PATCH 4/4] invoices and schema --- models/staging/xero/schema.yml | 182 +++++++++++++++++++++ models/staging/xero/stg_xero__invoices.sql | 45 +++++ 2 files changed, 227 insertions(+) create mode 100644 models/staging/xero/stg_xero__invoices.sql diff --git a/models/staging/xero/schema.yml b/models/staging/xero/schema.yml index ec6c99e..398af13 100644 --- a/models/staging/xero/schema.yml +++ b/models/staging/xero/schema.yml @@ -166,6 +166,188 @@ models: data_type: jsonb description: "" + - name: dwh_extracted_at_utc + data_type: timestamp with time zone + description: "" + - name: stg_xero__invoices + description: | + Invoices that we have raised within our accounting books. + + This includes both our invoices towards customers and invoices provided by our suppliers. + + You can read more here: https://developer.xero.com/documentation/api/accounting/invoices + columns: + - name: id_invoice + data_type: character varying + description: Xero's unique identifier for the invoice. + + - name: invoice_url + data_type: character varying + description: "" + + - name: invoice_issued_date_utc + data_type: timestamp with time zone + description: Date on which the invoice was issued. + + - name: invoice_type + data_type: character varying + description: | + This field indicates whether the invoice is from Superhog towards a customer + (value is "ACCREC") or from a supplier towards Superhog (value is "ACCPAY"). + + - name: total_amount_local_curr + data_type: numeric + description: The total amount to be paid, in the currency the invoice is denominated in. + + - name: total_amount_wo_tax_local_curr + data_type: numeric + description: The total amount to be paid minus taxes, in the currency the invoice is denominated in. + + - name: total_tax_local_curr + data_type: numeric + description: The total tax, in the currency the invoice is denominated in. + + - name: total_due_local_curr + data_type: numeric + description: The total amount outstanding right now, in the currency the invoice is denominated in. + + - name: total_paid_local_curr + data_type: numeric + description: The total amount that has already been paid, in the currency the invoice is denominated in. + + - name: invoice_status + data_type: character varying + description: | + The status of the invoice. + + Can be one of: PAID, VOIDED, DRAFT, DELETED, AUTHORISED, SUBMITTED. + + You can read more here: https://developer.xero.com/documentation/api/accounting/types#invoices + + - name: contact + data_type: jsonb + description: | + The contact related to this invoice. + + The customer if it's an Accounts Receivables one, the supplier if it's an Accounts Payables one. + + - name: invoice_due_date_utc + data_type: timestamp with time zone + description: The date were the invoice should be paid the latest. + + - name: payments + data_type: jsonb + description: | + An array with the details of the payments that are satisfying this invoice. + + - name: has_errors + data_type: boolean + description: "" + + - name: line_items + data_type: jsonb + description: An array with all the line items and their details. + + - name: reference + data_type: character varying + description: | + The Superhog-set reference for the invoices. Only relevant for + records of invoice type (ACCREC). + + - name: date_string + data_type: timestamp with time zone + description: "" + + - name: credit_notes + data_type: jsonb + description: An array with any related credit notes, along with all their details. + + - name: prepayments + data_type: jsonb + description: "" + + - name: invoice_currency_iso_4217 + data_type: character varying + description: The ISO 4217 code for the currency which this invoice uses as unit of account. + + - name: exchange_rate_to_gbp + data_type: numeric + description: | + The implied exchange rate between the invoice currency and GBP. + + This is the XXX to GBP rate. So the number represented in this field is the units of currency GBP + that 1 unit of XXX buys you. + + For example, if the invoice is 135 ZAR, and this rate is 0.0167, the invoice value in GBP is 2.25 + (135ZAR * 0.0167GBP/ZAR = 2.25GBP). + + - name: is_discounted + data_type: boolean + description: "" + + - name: overpayments + data_type: jsonb + description: "" + + - name: due_date_string + data_type: timestamp with time zone + description: "" + + - name: invoice_number + data_type: character varying + description: "" + + - name: has_been_sent_to_contact + data_type: boolean + description: "" + + - name: total_discount + data_type: numeric + description: "" + + - name: total_credited_local_curr + data_type: numeric + description: + Total amount credited by related credit notes, over-payments and + pre-payments. + + - name: has_attachments + data_type: boolean + description: Flag indicating if the invoice has an attahcment. + + - name: updated_at_utc + data_type: timestamp with time zone + description: "" + + - name: updated_date_utc + data_type: date + description: "" + + - name: id_branding_theme + data_type: character varying + description: "" + + - name: was_fully_paid_on_date_utc + data_type: timestamp with time zone + description: | + The date on which the invoice was fully paid. If the invoice is still + not fully paid, it returns null. + + - name: line_amount_tax_inclusiveness + data_type: character varying + description: | + Indicates whether the amounts included in line items are tax + inclusive (Inclusive), tax exclusive (Exclusive) or simply, + there are no taxes on the invoice (NoTax). + + - name: payment_planned_date_utc + data_type: timestamp with time zone + description: "" + + - name: payment_expected_date_utc + data_type: timestamp with time zone + description: "" + - name: dwh_extracted_at_utc data_type: timestamp with time zone description: "" \ No newline at end of file diff --git a/models/staging/xero/stg_xero__invoices.sql b/models/staging/xero/stg_xero__invoices.sql new file mode 100644 index 0000000..4125e6d --- /dev/null +++ b/models/staging/xero/stg_xero__invoices.sql @@ -0,0 +1,45 @@ +with + raw_invoices as (select * from {{ source("xero", "invoices") }}), + stg_xero__invoices as ( + select + {{ adapter.quote("InvoiceID") }} as id_invoice, + {{ adapter.quote("Reference") }} as reference, + {{ adapter.quote("InvoiceNumber") }} as invoice_number, + {{ adapter.quote("Date") }} as invoice_issued_date_utc, + {{ adapter.quote("DueDate") }} as invoice_due_date_utc, + {{ adapter.quote("FullyPaidOnDate") }} as was_fully_paid_on_date_utc, + {{ adapter.quote("Type") }} as invoice_type, + {{ adapter.quote("CurrencyCode") }} as invoice_currency_iso_4217, + 1 / {{ adapter.quote("CurrencyRate") }} as exchange_rate_to_gbp, + {{ adapter.quote("Total") }} as total_amount_local_curr, + {{ adapter.quote("SubTotal") }} as total_amount_wo_tax_local_curr, + {{ adapter.quote("TotalTax") }} as total_tax_local_curr, + {{ adapter.quote("AmountDue") }} as total_due_local_curr, + {{ adapter.quote("AmountPaid") }} as total_paid_local_curr, + {{ adapter.quote("Status") }} as invoice_status, + {{ adapter.quote("Contact") }} as contact, + {{ adapter.quote("Payments") }} as payments, + {{ adapter.quote("HasErrors") }} as has_errors, + {{ adapter.quote("LineItems") }} as line_items, + {{ adapter.quote("CreditNotes") }} as credit_notes, + {{ adapter.quote("Prepayments") }} as prepayments, + {{ adapter.quote("Overpayments") }} as overpayments, + {{ adapter.quote("IsDiscounted") }} as is_discounted, + {{ adapter.quote("DateString") }} as date_string, + {{ adapter.quote("DueDateString") }} as due_date_string, + {{ adapter.quote("SentToContact") }} as has_been_sent_to_contact, + {{ adapter.quote("TotalDiscount") }} as total_discount, + {{ adapter.quote("AmountCredited") }} as total_credited_local_curr, + {{ adapter.quote("HasAttachments") }} as has_attachments, + {{ adapter.quote("UpdatedDateUTC") }} as updated_at_utc, + cast({{ adapter.quote("UpdatedDateUTC") }} as date) as updated_date_utc, + {{ adapter.quote("BrandingThemeID") }} as id_branding_theme, + {{ adapter.quote("Url") }} as invoice_url, + {{ adapter.quote("LineAmountTypes") }} as line_amount_tax_inclusiveness, + {{ adapter.quote("PlannedPaymentDate") }} as payment_planned_date_utc, + {{ adapter.quote("ExpectedPaymentDate") }} as payment_expected_date_utc, + {{ adapter.quote("_airbyte_extracted_at") }} as dwh_extracted_at_utc + from raw_invoices + ) +select * +from stg_xero__invoices