From e3be2708ad4424ad71e9e8ab6729f6d905730f03 Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Tue, 18 Jun 2024 11:06:27 +0200 Subject: [PATCH 1/6] add new rates to CTEs --- models/intermediate/core/int_core__verification_payments.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/models/intermediate/core/int_core__verification_payments.sql b/models/intermediate/core/int_core__verification_payments.sql index 0e7f61c..cf2ad11 100644 --- a/models/intermediate/core/int_core__verification_payments.sql +++ b/models/intermediate/core/int_core__verification_payments.sql @@ -10,7 +10,8 @@ with stg_core__payment_status as (select * from {{ ref("stg_core__payment_status") }}), int_hardcoded_historical_currency_rates as ( select * from {{ ref("int_hardcoded_historical_currency_rates") }} - ) + ), + int_simple_exchange_rates as (select * from {{ ref("int_simple_exchange_rates") }}) select vtp.id_verification_to_payment, vtp.id_payment, From 6f625ec7dbf6456daeac5907a21c29cbb74a6ddf Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Tue, 18 Jun 2024 11:13:51 +0200 Subject: [PATCH 2/6] do both rates to compare and test --- .../intermediate/core/int_core__verification_payments.sql | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/models/intermediate/core/int_core__verification_payments.sql b/models/intermediate/core/int_core__verification_payments.sql index cf2ad11..accd915 100644 --- a/models/intermediate/core/int_core__verification_payments.sql +++ b/models/intermediate/core/int_core__verification_payments.sql @@ -35,6 +35,7 @@ select p.amount as amount_in_txn_currency, p.currency, (p.amount * r.rate) as amount_in_gbp, + (p.amount * nr.rate) as new_amount_in_gbp, ps.payment_status, p.notes from stg_core__verification_to_payment vtp @@ -49,3 +50,8 @@ left join on p.paid_date_utc = r.rate_date and p.currency = r.from_currency and r.to_currency = 'GBP' +left join + int_simple_exchange_rates nr + on p.paid_date_utc = nr.rate_date_utc + and p.currency = nr.from_currency + and nr.to_currency = 'GBP' From d96c8b2abb96e10c30b0f21a1f7c278f6dff8def Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Tue, 18 Jun 2024 11:23:10 +0200 Subject: [PATCH 3/6] change which field we join on: payment date is not always informed --- models/intermediate/core/int_core__verification_payments.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/intermediate/core/int_core__verification_payments.sql b/models/intermediate/core/int_core__verification_payments.sql index accd915..9b1c9b0 100644 --- a/models/intermediate/core/int_core__verification_payments.sql +++ b/models/intermediate/core/int_core__verification_payments.sql @@ -47,11 +47,11 @@ left join left join stg_core__payment_status ps on p.id_payment_status = ps.id_payment_status left join int_hardcoded_historical_currency_rates r - on p.paid_date_utc = r.rate_date + on vtp.payment_due_date_utc = r.rate_date and p.currency = r.from_currency and r.to_currency = 'GBP' left join int_simple_exchange_rates nr - on p.paid_date_utc = nr.rate_date_utc + on vtp.payment_due_date_utc = nr.rate_date_utc and p.currency = nr.from_currency and nr.to_currency = 'GBP' From 1f9df9ea5c75a1de8a93acdc56d52657b0f1a8b1 Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Tue, 18 Jun 2024 11:24:03 +0200 Subject: [PATCH 4/6] remove old table --- .../core/int_core__verification_payments.sql | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/models/intermediate/core/int_core__verification_payments.sql b/models/intermediate/core/int_core__verification_payments.sql index 9b1c9b0..c48bb2b 100644 --- a/models/intermediate/core/int_core__verification_payments.sql +++ b/models/intermediate/core/int_core__verification_payments.sql @@ -35,7 +35,6 @@ select p.amount as amount_in_txn_currency, p.currency, (p.amount * r.rate) as amount_in_gbp, - (p.amount * nr.rate) as new_amount_in_gbp, ps.payment_status, p.notes from stg_core__verification_to_payment vtp @@ -46,12 +45,7 @@ left join on vtp.id_verification_payment_type = vpt.id_verification_payment_type left join stg_core__payment_status ps on p.id_payment_status = ps.id_payment_status left join - int_hardcoded_historical_currency_rates r - on vtp.payment_due_date_utc = r.rate_date + int_simple_exchange_rates r + on vtp.payment_due_date_utc = r.rate_date_utc and p.currency = r.from_currency and r.to_currency = 'GBP' -left join - int_simple_exchange_rates nr - on vtp.payment_due_date_utc = nr.rate_date_utc - and p.currency = nr.from_currency - and nr.to_currency = 'GBP' From 8b91babad6c2bdff1dccdd6d55165c36e5bfb1d7 Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Tue, 18 Jun 2024 11:31:59 +0200 Subject: [PATCH 5/6] remove old table from cte --- models/intermediate/core/int_core__verification_payments.sql | 3 --- 1 file changed, 3 deletions(-) diff --git a/models/intermediate/core/int_core__verification_payments.sql b/models/intermediate/core/int_core__verification_payments.sql index c48bb2b..0253066 100644 --- a/models/intermediate/core/int_core__verification_payments.sql +++ b/models/intermediate/core/int_core__verification_payments.sql @@ -8,9 +8,6 @@ with stg_core__verification as (select * from {{ ref("stg_core__verification") }}), stg_core__payment as (select * from {{ ref("stg_core__payment") }}), stg_core__payment_status as (select * from {{ ref("stg_core__payment_status") }}), - int_hardcoded_historical_currency_rates as ( - select * from {{ ref("int_hardcoded_historical_currency_rates") }} - ), int_simple_exchange_rates as (select * from {{ ref("int_simple_exchange_rates") }}) select vtp.id_verification_to_payment, From 662c7b8ba85f7b064c85259aa4f229558d656bec Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Tue, 18 Jun 2024 11:35:07 +0200 Subject: [PATCH 6/6] remove hardcoded rates and seed, remove docs --- ...nt_hardcoded_historical_currency_rates.sql | 7 -- seeds/schema.yaml | 39 ----------- seeds/stg_seed__hardcoded_currency_rates.csv | 65 ------------------- 3 files changed, 111 deletions(-) delete mode 100644 models/intermediate/cross/int_hardcoded_historical_currency_rates.sql delete mode 100644 seeds/stg_seed__hardcoded_currency_rates.csv diff --git a/models/intermediate/cross/int_hardcoded_historical_currency_rates.sql b/models/intermediate/cross/int_hardcoded_historical_currency_rates.sql deleted file mode 100644 index 8e6d81e..0000000 --- a/models/intermediate/cross/int_hardcoded_historical_currency_rates.sql +++ /dev/null @@ -1,7 +0,0 @@ -with - hardcoded_rates as (select * from {{ ref("stg_seed__hardcoded_currency_rates") }}), - dates as (select * from {{ ref("int_dates") }}) -select d.date_day as rate_date, r.* -from dates d -cross join hardcoded_rates r -where d.date_day > cast({{ var("start_date") }} as date) diff --git a/seeds/schema.yaml b/seeds/schema.yaml index 14e0f50..d23489c 100644 --- a/seeds/schema.yaml +++ b/seeds/schema.yaml @@ -41,42 +41,3 @@ seeds: min_value: 0 max_value: 8 strictly: False - - name: stg_seed__hardcoded_currency_rates - description: | - A bunch of rates between currencies, taken from our invoicing cycle in February 2024. - - These are hardcoded and so will become outdated, but it's the best we have now since we don't have a live feed for exchange rates. - config: - column_types: - from_currency: varchar(3) - to_currency: varchar(3) - rate: numeric - columns: - - name: from_currency - data_type: character varying - description: The 3 character ISO 4217 code for the base currency, in Uppercase. - tests: - - not_null - - dbt_expectations.expect_column_values_to_match_regex: - regex: "^[A-Z]{3}$" - columns: - - name: to_currency - data_type: character varying - description: The 3 character ISO 4217 code for the target currency, in Uppercase. - tests: - - not_null - - dbt_expectations.expect_column_values_to_match_regex: - regex: "^[A-Z]{3}$" - - name: rate - data_type: numeric - description: | - The rate between the two currencies. You should read it as the amount of target currency you get for each unit of base currency. - - For example: for the combination EUR-GBP, a rate of 0.8 means you get 0.8 GBP for 1 EUR. - - Rates between same-currency pairs are always one. - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_between: - min_value: 0.00000000001 - max_value: 999999999999 diff --git a/seeds/stg_seed__hardcoded_currency_rates.csv b/seeds/stg_seed__hardcoded_currency_rates.csv deleted file mode 100644 index 3d60fb0..0000000 --- a/seeds/stg_seed__hardcoded_currency_rates.csv +++ /dev/null @@ -1,65 +0,0 @@ -from_currency,to_currency,rate -EUR,EUR,1 -USD,USD,1 -GBP,GBP,1 -CAD,CAD,1 -AUD,AUD,1 -ZAR,ZAR,1 -NZD,NZD,1 -PLN,PLN,1 -EUR,USD,1.08286 -EUR,GBP,0.85555 -EUR,CAD,1.4688 -EUR,AUD,1.66453 -EUR,ZAR,20.7993 -EUR,NZD,1.77391 -EUR,PLN,4.31113 -USD,EUR,0.92335 -USD,GBP,0.79003 -USD,CAD,1.3564 -USD,AUD,1.53701 -USD,ZAR,19.2073 -USD,NZD,1.63816 -USD,PLN,3.98085 -GBP,EUR,1.16864 -GBP,USD,1.26559 -GBP,CAD,1.71664 -GBP,AUD,1.94522 -GBP,ZAR,24.3085 -GBP,NZD,2.07323 -GBP,PLN,5.03811 -NZD,EUR,0.56352 -NZD,USD,0.6103 -NZD,GBP,0.48216 -NZD,CAD,0.82781 -NZD,AUD,0.9381 -NZD,ZAR,11.7222 -NZD,PLN,2.42951 -CAD,EUR,0.68064 -CAD,USD,0.73714 -CAD,GBP,0.58237 -CAD,AUD,1.133 -CAD,ZAR,14.1585 -CAD,NZD,1.20756 -CAD,PLN,2.93446 -AUD,EUR,0.60068 -AUD,USD,0.65049 -AUD,GBP,0.51391 -AUD,CAD,0.88232 -AUD,ZAR,12.4941 -AUD,NZD,1.06572 -AUD,PLN,2.58949 -ZAR,EUR,0.04804 -ZAR,USD,0.05203 -ZAR,GBP,0.0411 -ZAR,CAD,0.07057 -ZAR,AUD,0.07997 -ZAR,NZD,0.08523 -ZAR,PLN,0.20712 -PLN,EUR,0.23184 -PLN,USD,0.25106 -PLN,GBP,0.19835 -PLN,CAD,0.34054 -PLN,AUD,0.38588 -PLN,ZAR,4.82218 -PLN,NZD,0.41128