From 447cb3926cc4b58bc3dfd08b41dd07d0f882f973 Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Tue, 25 Jun 2024 11:38:37 +0200 Subject: [PATCH 01/13] bring over bank transactions to int --- .../xero/int_xero__bank_transactions.sql | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 models/intermediate/xero/int_xero__bank_transactions.sql diff --git a/models/intermediate/xero/int_xero__bank_transactions.sql b/models/intermediate/xero/int_xero__bank_transactions.sql new file mode 100644 index 0000000..d972202 --- /dev/null +++ b/models/intermediate/xero/int_xero__bank_transactions.sql @@ -0,0 +1,30 @@ +with + stg_xero__bank_transactions as ( + select * from {{ ref("stg_xero__bank_transactions") }} + ) +select + id_bank_transaction, + id_prepayment, + id_overpayment, + reference, + transaction_type, + transaction_status, + transaction_at_utc, + date_string, + total_amount_local_curr, + subtotal_local_curr, + total_tax_local_curr, + transaction_currency_iso_4217, + exchange_rate_to_gbp, + line_amount_tax_inclusiveness, + full_contact_details, + line_items, + bank_account, + is_reconciled, + has_attachments, + url, + external_link_provider_name, + updated_at_utc, + updated_date_utc, + dwh_extracted_at_utc +from stg_xero__bank_transactions From 08ff92b3e9ad7af82612684e2248526555dd158e Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Tue, 25 Jun 2024 11:39:37 +0200 Subject: [PATCH 02/13] make stg naming coherent with other xero models --- models/staging/xero/stg_xero__bank_transactions.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/staging/xero/stg_xero__bank_transactions.sql b/models/staging/xero/stg_xero__bank_transactions.sql index 75d9f42..f2e4832 100644 --- a/models/staging/xero/stg_xero__bank_transactions.sql +++ b/models/staging/xero/stg_xero__bank_transactions.sql @@ -14,7 +14,7 @@ with {{ adapter.quote("DateString") }} as date_string, {{ adapter.quote("Total") }} as total_amount_local_curr, - {{ adapter.quote("SubTotal") }} as subtotal_local_curr, + {{ adapter.quote("SubTotal") }} as total_amount_wo_tax_local_curr, {{ adapter.quote("TotalTax") }} as total_tax_local_curr, {{ adapter.quote("CurrencyCode") }} as transaction_currency_iso_4217, 1 / {{ adapter.quote("CurrencyRate") }} as exchange_rate_to_gbp, From 200c324b68ba7c1f7560d20b975792d00754b949 Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Tue, 25 Jun 2024 11:41:03 +0200 Subject: [PATCH 03/13] add converted fields --- models/intermediate/xero/int_xero__bank_transactions.sql | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/models/intermediate/xero/int_xero__bank_transactions.sql b/models/intermediate/xero/int_xero__bank_transactions.sql index d972202..6065266 100644 --- a/models/intermediate/xero/int_xero__bank_transactions.sql +++ b/models/intermediate/xero/int_xero__bank_transactions.sql @@ -12,8 +12,15 @@ select transaction_at_utc, date_string, total_amount_local_curr, - subtotal_local_curr, + (total_amount_local_curr * exchange_rate_to_gbp)::numeric( + 18, 4 + ) as total_amount_in_gbp, + total_amount_wo_tax_local_curr, + (total_amount_wo_tax_local_curr * exchange_rate_to_gbp)::numeric( + 18, 4 + ) as total_amount_wo_tax_in_gbp, total_tax_local_curr, + (total_tax_local_curr * exchange_rate_to_gbp)::numeric(18, 4) as total_tax_in_gbp, transaction_currency_iso_4217, exchange_rate_to_gbp, line_amount_tax_inclusiveness, From bf54548654ea21afc568b5bf67b5aaef65a6300a Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Tue, 25 Jun 2024 11:44:00 +0200 Subject: [PATCH 04/13] table alias --- .../xero/int_xero__bank_transactions.sql | 58 ++++++++++--------- 1 file changed, 30 insertions(+), 28 deletions(-) diff --git a/models/intermediate/xero/int_xero__bank_transactions.sql b/models/intermediate/xero/int_xero__bank_transactions.sql index 6065266..d18f3e3 100644 --- a/models/intermediate/xero/int_xero__bank_transactions.sql +++ b/models/intermediate/xero/int_xero__bank_transactions.sql @@ -3,35 +3,37 @@ with select * from {{ ref("stg_xero__bank_transactions") }} ) select - id_bank_transaction, - id_prepayment, - id_overpayment, - reference, - transaction_type, - transaction_status, - transaction_at_utc, - date_string, - total_amount_local_curr, - (total_amount_local_curr * exchange_rate_to_gbp)::numeric( + bt.id_bank_transaction, + bt.id_prepayment, + bt.id_overpayment, + bt.reference, + bt.transaction_type, + bt.transaction_status, + bt.transaction_at_utc, + bt.date_string, + bt.total_amount_local_curr, + (bt.total_amount_local_curr * bt.exchange_rate_to_gbp)::numeric( 18, 4 ) as total_amount_in_gbp, - total_amount_wo_tax_local_curr, - (total_amount_wo_tax_local_curr * exchange_rate_to_gbp)::numeric( + bt.total_amount_wo_tax_local_curr, + (bt.total_amount_wo_tax_local_curr * bt.exchange_rate_to_gbp)::numeric( 18, 4 ) as total_amount_wo_tax_in_gbp, - total_tax_local_curr, - (total_tax_local_curr * exchange_rate_to_gbp)::numeric(18, 4) as total_tax_in_gbp, - transaction_currency_iso_4217, - exchange_rate_to_gbp, - line_amount_tax_inclusiveness, - full_contact_details, - line_items, - bank_account, - is_reconciled, - has_attachments, - url, - external_link_provider_name, - updated_at_utc, - updated_date_utc, - dwh_extracted_at_utc -from stg_xero__bank_transactions + bt.total_tax_local_curr, + (bt.total_tax_local_curr * bt.exchange_rate_to_gbp)::numeric( + 18, 4 + ) as total_tax_in_gbp, + bt.transaction_currency_iso_4217, + bt.exchange_rate_to_gbp, + bt.line_amount_tax_inclusiveness, + bt.full_contact_details, + bt.line_items, + bt.bank_account, + bt.is_reconciled, + bt.has_attachments, + bt.url, + bt.external_link_provider_name, + bt.updated_at_utc, + bt.updated_date_utc, + bt.dwh_extracted_at_utc +from stg_xero__bank_transactions bt From 4b0b15814e2c9723ed99e02a699df226f9f8d798 Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Tue, 25 Jun 2024 11:45:26 +0200 Subject: [PATCH 05/13] add date converted in staging --- models/staging/xero/stg_xero__bank_transactions.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/models/staging/xero/stg_xero__bank_transactions.sql b/models/staging/xero/stg_xero__bank_transactions.sql index f2e4832..dec8c6f 100644 --- a/models/staging/xero/stg_xero__bank_transactions.sql +++ b/models/staging/xero/stg_xero__bank_transactions.sql @@ -11,6 +11,7 @@ with {{ adapter.quote("Status") }} as transaction_status, {{ adapter.quote("Date") }} as transaction_at_utc, + cast({{ adapter.quote("Date") }} as date) as transaction_date_utc, {{ adapter.quote("DateString") }} as date_string, {{ adapter.quote("Total") }} as total_amount_local_curr, From 25e9d040fff9fab14ab9fd11147ef87d53c2c839 Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Tue, 25 Jun 2024 11:56:51 +0200 Subject: [PATCH 06/13] add internal exchange rates --- .../xero/int_xero__bank_transactions.sql | 23 +++++++++++++------ 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/models/intermediate/xero/int_xero__bank_transactions.sql b/models/intermediate/xero/int_xero__bank_transactions.sql index d18f3e3..1280978 100644 --- a/models/intermediate/xero/int_xero__bank_transactions.sql +++ b/models/intermediate/xero/int_xero__bank_transactions.sql @@ -1,7 +1,8 @@ with stg_xero__bank_transactions as ( select * from {{ ref("stg_xero__bank_transactions") }} - ) + ), + int_simple_exchange_rates as (select * from {{ ref("int_simple_exchange_rates") }}) select bt.id_bank_transaction, bt.id_prepayment, @@ -10,21 +11,24 @@ select bt.transaction_type, bt.transaction_status, bt.transaction_at_utc, + bt.transaction_date_utc, bt.date_string, bt.total_amount_local_curr, - (bt.total_amount_local_curr * bt.exchange_rate_to_gbp)::numeric( + (bt.total_amount_local_curr * coalesce(bt.exchange_rate_to_gbp, ser.rate))::numeric( 18, 4 ) as total_amount_in_gbp, bt.total_amount_wo_tax_local_curr, - (bt.total_amount_wo_tax_local_curr * bt.exchange_rate_to_gbp)::numeric( - 18, 4 - ) as total_amount_wo_tax_in_gbp, + ( + bt.total_amount_wo_tax_local_curr * coalesce(bt.exchange_rate_to_gbp, ser.rate) + )::numeric(18, 4) as total_amount_wo_tax_in_gbp, bt.total_tax_local_curr, - (bt.total_tax_local_curr * bt.exchange_rate_to_gbp)::numeric( + (bt.total_tax_local_curr * coalesce(bt.exchange_rate_to_gbp, ser.rate))::numeric( 18, 4 ) as total_tax_in_gbp, bt.transaction_currency_iso_4217, - bt.exchange_rate_to_gbp, + -- We try to use Xero's rate. If no rate is available, we fall back to the + -- oficial dwh rate. + coalesce(bt.exchange_rate_to_gbp, ser.rate) as exchange_rate_to_gbp, bt.line_amount_tax_inclusiveness, bt.full_contact_details, bt.line_items, @@ -37,3 +41,8 @@ select bt.updated_date_utc, bt.dwh_extracted_at_utc from stg_xero__bank_transactions bt +left join + int_simple_exchange_rates ser + on bt.transaction_currency_iso_4217 = ser.from_currency + and ser.to_currency = 'GBP' + and bt.transaction_date_utc = ser.rate_date_utc From 33d072015cb4057f12870c3971ae5837ee181e10 Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Tue, 25 Jun 2024 12:00:50 +0200 Subject: [PATCH 07/13] remove full jsons with ids --- models/intermediate/xero/int_xero__bank_transactions.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/intermediate/xero/int_xero__bank_transactions.sql b/models/intermediate/xero/int_xero__bank_transactions.sql index 1280978..5f56c77 100644 --- a/models/intermediate/xero/int_xero__bank_transactions.sql +++ b/models/intermediate/xero/int_xero__bank_transactions.sql @@ -7,6 +7,8 @@ select bt.id_bank_transaction, bt.id_prepayment, bt.id_overpayment, + bt.full_contact_details ->> 'ContactID' as id_contact, + bt.bank_account ->> 'AccountID' as id_bank_account, bt.reference, bt.transaction_type, bt.transaction_status, @@ -30,9 +32,7 @@ select -- oficial dwh rate. coalesce(bt.exchange_rate_to_gbp, ser.rate) as exchange_rate_to_gbp, bt.line_amount_tax_inclusiveness, - bt.full_contact_details, bt.line_items, - bt.bank_account, bt.is_reconciled, bt.has_attachments, bt.url, From d86b6b86276419432caf9858412806d3d5cf37d5 Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Tue, 25 Jun 2024 13:37:53 +0200 Subject: [PATCH 08/13] add tests --- models/intermediate/xero/schema.yaml | 35 ++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 models/intermediate/xero/schema.yaml diff --git a/models/intermediate/xero/schema.yaml b/models/intermediate/xero/schema.yaml new file mode 100644 index 0000000..aa1b3e3 --- /dev/null +++ b/models/intermediate/xero/schema.yaml @@ -0,0 +1,35 @@ +version: 2 + +models: + - name: int_xero__invoices + columns: + - name: id_invoice + tests: + - not_null + - unique + - name: int_xero__credit_notes + columns: + - name: id_credit_note + tests: + - not_null + - unique + - name: int_xero__invoice_line_items + columns: + - name: id_line_item + tests: + - not_null + - unique + - name: int_xero__credit_note_line_items + columns: + - name: id_line_item + tests: + - not_null + - unique + - name: int_xero__bank_transactions + columns: + - name: id_bank_transaction + tests: + - not_null + - unique + + \ No newline at end of file From 7a9c913b8af92b63b4f03c3bf3808e6fbbcbd7fd Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Tue, 25 Jun 2024 14:07:45 +0200 Subject: [PATCH 09/13] line items --- .../int_xero__bank_transaction_line_items.sql | 75 +++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 models/intermediate/xero/int_xero__bank_transaction_line_items.sql diff --git a/models/intermediate/xero/int_xero__bank_transaction_line_items.sql b/models/intermediate/xero/int_xero__bank_transaction_line_items.sql new file mode 100644 index 0000000..8aa023f --- /dev/null +++ b/models/intermediate/xero/int_xero__bank_transaction_line_items.sql @@ -0,0 +1,75 @@ +with + int_xero__bank_transactions as ( + select * from {{ ref("int_xero__bank_transactions") }} + ) + +select + bt.id_bank_transaction, + j.id_line_item, + j.quantity::numeric, + j.unit_amount::numeric, + j.line_amount_local_curr::numeric, + (j.line_amount_local_curr::numeric * bt.exchange_rate_to_gbp)::numeric( + 18, 4 + ) as line_amount_in_gbp, + case + when bt.line_amount_tax_inclusiveness = 'Inclusive' + then j.line_amount_local_curr::numeric - j.tax_amount_local_curr::numeric + when bt.line_amount_tax_inclusiveness = 'Exclusive' + then j.line_amount_local_curr::numeric + when bt.line_amount_tax_inclusiveness = 'NoTax' + then j.line_amount_local_curr::numeric + else null + end as line_amount_wo_taxes_local_curr, + case + when bt.line_amount_tax_inclusiveness = 'Inclusive' + then + (j.line_amount_local_curr::numeric * bt.exchange_rate_to_gbp)::numeric( + 18, + 4 + ) - (j.tax_amount_local_curr::numeric * bt.exchange_rate_to_gbp)::numeric( + 18, 4 + ) + when bt.line_amount_tax_inclusiveness = 'Exclusive' + then + (j.line_amount_local_curr::numeric * bt.exchange_rate_to_gbp)::numeric( + 18, 4 + ) + when bt.line_amount_tax_inclusiveness = 'NoTax' + then + (j.line_amount_local_curr::numeric * bt.exchange_rate_to_gbp)::numeric( + 18, 4 + ) + else null + end as line_amount_wo_taxes_in_gbp, + j.tax_amount_local_curr::numeric, + (j.tax_amount_local_curr::numeric * bt.exchange_rate_to_gbp)::numeric( + 18, 4 + ) as tax_amount_in_gbp, + j.tax_type, + bt.transaction_currency_iso_4217, + j.line_description, + j.id_account, + j.account_code, + j.item_code +from int_xero__bank_transactions bt +cross join + lateral( + select + (jsonb_array_elements(bt.line_items) ->> 'LineItemID') as id_line_item, + (jsonb_array_elements(bt.line_items) ->> 'AccountID') as id_account, + + (jsonb_array_elements(bt.line_items) ->> 'Quantity') as quantity, + (jsonb_array_elements(bt.line_items) ->> 'UnitAmount') as unit_amount, + ( + jsonb_array_elements(bt.line_items) ->> 'LineAmount' + ) as line_amount_local_curr, + ( + jsonb_array_elements(bt.line_items) ->> 'TaxAmount' + ) as tax_amount_local_curr, + (jsonb_array_elements(bt.line_items) ->> 'TaxType') as tax_type, + (jsonb_array_elements(bt.line_items) ->> 'Description') as line_description, + + (jsonb_array_elements(bt.line_items) ->> 'AccountCode') as account_code, + (jsonb_array_elements(bt.line_items) ->> 'ItemCode') as item_code + ) j From 8dc31f348942d2a283320fe19fbad070ef8b3be5 Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Tue, 25 Jun 2024 14:29:46 +0200 Subject: [PATCH 10/13] add accounts to staging --- models/staging/xero/_xero_sources.yml | 4 +++ models/staging/xero/schema.yml | 15 +++++++++++ models/staging/xero/stg_xero__accounts.sql | 29 ++++++++++++++++++++++ 3 files changed, 48 insertions(+) create mode 100644 models/staging/xero/stg_xero__accounts.sql diff --git a/models/staging/xero/_xero_sources.yml b/models/staging/xero/_xero_sources.yml index 88cbb79..2833340 100644 --- a/models/staging/xero/_xero_sources.yml +++ b/models/staging/xero/_xero_sources.yml @@ -23,3 +23,7 @@ sources: identifier: bank_transactions description: | Bank transactions that have been created within our accounting books. + - name: accounts + identifier: accounts + description: | + Accounts from our accounting trees. diff --git a/models/staging/xero/schema.yml b/models/staging/xero/schema.yml index b2490f8..09704d1 100644 --- a/models/staging/xero/schema.yml +++ b/models/staging/xero/schema.yml @@ -774,3 +774,18 @@ models: - name: dwh_extracted_at_utc data_type: timestamp with time zone description: "" + - name: stg_xero__accounts + description: | + Accounts in our accounting tree. + columns: + - name: id_account + data_type: character varying + description: Xero's unique identifier for the account. + tests: + - not_null + - unique + - name: account_code + data_type: character varying + description: Human readable account code. + tests: + - unique diff --git a/models/staging/xero/stg_xero__accounts.sql b/models/staging/xero/stg_xero__accounts.sql new file mode 100644 index 0000000..8aa0f53 --- /dev/null +++ b/models/staging/xero/stg_xero__accounts.sql @@ -0,0 +1,29 @@ +with + raw_accounts as (select * from {{ source("xero", "accounts") }}), + stg_xero__accounts as ( + select + {{ adapter.quote("AccountID") }} as id_account, + {{ adapter.quote("Code") }} as account_code, + {{ adapter.quote("Name") }} as account_name, + {{ adapter.quote("Type") }} as account_type, + {{ adapter.quote("Class") }} as account_class, + {{ adapter.quote("Status") }} as is_active, + {{ adapter.quote("TaxType") }} as account_tax_type, + {{ adapter.quote("Description") }} as account_description, + {{ adapter.quote("CurrencyCode") }} as account_currency_iso_4217, + {{ adapter.quote("ReportingCode") }} as reporting_code, + {{ adapter.quote("ReportingCodeName") }} as reporting_name, + {{ adapter.quote("SystemAccount") }} as system_account, + {{ adapter.quote("HasAttachments") }} as has_attachments, + {{ adapter.quote("BankAccountType") }} as bank_account_type, + {{ adapter.quote("BankAccountNumber") }} as bank_account_number, + {{ adapter.quote("ShowInExpenseClaims") }} as show_in_expense_claims, + {{ adapter.quote("EnablePaymentsToAccount") }} + as enable_payments_to_account, + {{ adapter.quote("UpdatedDateUTC") }} as updated_at_utc, + {{ adapter.quote("_airbyte_extracted_at") }} as dwh_extracted_at_utc + + from raw_accounts + ) +select * +from stg_xero__accounts From 854afffc1c034d686128628f2d92d2a3e1531a69 Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Tue, 25 Jun 2024 14:47:11 +0200 Subject: [PATCH 11/13] add account code --- .../int_xero__bank_transaction_line_items.sql | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/models/intermediate/xero/int_xero__bank_transaction_line_items.sql b/models/intermediate/xero/int_xero__bank_transaction_line_items.sql index 8aa023f..afef74e 100644 --- a/models/intermediate/xero/int_xero__bank_transaction_line_items.sql +++ b/models/intermediate/xero/int_xero__bank_transaction_line_items.sql @@ -1,11 +1,17 @@ with int_xero__bank_transactions as ( select * from {{ ref("int_xero__bank_transactions") }} - ) + ), + stg_xero__accounts as (select * from {{ ref("stg_xero__accounts") }}) select bt.id_bank_transaction, j.id_line_item, + j.line_description, + j.id_account, + j.account_code, + a.account_name, + j.item_code, j.quantity::numeric, j.unit_amount::numeric, j.line_amount_local_curr::numeric, @@ -47,11 +53,7 @@ select 18, 4 ) as tax_amount_in_gbp, j.tax_type, - bt.transaction_currency_iso_4217, - j.line_description, - j.id_account, - j.account_code, - j.item_code + bt.transaction_currency_iso_4217 from int_xero__bank_transactions bt cross join lateral( @@ -73,3 +75,4 @@ cross join (jsonb_array_elements(bt.line_items) ->> 'AccountCode') as account_code, (jsonb_array_elements(bt.line_items) ->> 'ItemCode') as item_code ) j +left join stg_xero__accounts a on j.account_code = a.account_code From 3973f0a4a6c37b7b50bf94c5af56a12f767eb915 Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Tue, 25 Jun 2024 15:06:43 +0200 Subject: [PATCH 12/13] rename field --- models/intermediate/xero/int_xero__bank_transactions.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/intermediate/xero/int_xero__bank_transactions.sql b/models/intermediate/xero/int_xero__bank_transactions.sql index 5f56c77..fefdf22 100644 --- a/models/intermediate/xero/int_xero__bank_transactions.sql +++ b/models/intermediate/xero/int_xero__bank_transactions.sql @@ -8,7 +8,7 @@ select bt.id_prepayment, bt.id_overpayment, bt.full_contact_details ->> 'ContactID' as id_contact, - bt.bank_account ->> 'AccountID' as id_bank_account, + bt.bank_account ->> 'BankAccountID' as id_bank_account, bt.reference, bt.transaction_type, bt.transaction_status, From 88e13182a5045add98396726ff1f1bb85fbbf882 Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Tue, 25 Jun 2024 16:24:39 +0200 Subject: [PATCH 13/13] convert currency after subtracting in local currency --- .../xero/int_xero__bank_transaction_line_items.sql | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/models/intermediate/xero/int_xero__bank_transaction_line_items.sql b/models/intermediate/xero/int_xero__bank_transaction_line_items.sql index afef74e..10da0fe 100644 --- a/models/intermediate/xero/int_xero__bank_transaction_line_items.sql +++ b/models/intermediate/xero/int_xero__bank_transaction_line_items.sql @@ -30,12 +30,10 @@ select case when bt.line_amount_tax_inclusiveness = 'Inclusive' then - (j.line_amount_local_curr::numeric * bt.exchange_rate_to_gbp)::numeric( - 18, - 4 - ) - (j.tax_amount_local_curr::numeric * bt.exchange_rate_to_gbp)::numeric( - 18, 4 - ) + ( + (j.line_amount_local_curr::numeric - j.tax_amount_local_curr::numeric) + * bt.exchange_rate_to_gbp + )::numeric(18, 4) when bt.line_amount_tax_inclusiveness = 'Exclusive' then (j.line_amount_local_curr::numeric * bt.exchange_rate_to_gbp)::numeric(