From 906bccce0e0da453631932774c6b3f91a95b08ee Mon Sep 17 00:00:00 2001 From: Joaquin Ossa Date: Fri, 30 Aug 2024 11:33:55 +0200 Subject: [PATCH 1/6] modified model --- .../int_edeposit__verification_fees.sql | 60 +++++++++++++++++++ models/intermediate/edeposit/schema.yaml | 31 ++++++++++ 2 files changed, 91 insertions(+) create mode 100644 models/intermediate/edeposit/int_edeposit__verification_fees.sql diff --git a/models/intermediate/edeposit/int_edeposit__verification_fees.sql b/models/intermediate/edeposit/int_edeposit__verification_fees.sql new file mode 100644 index 0000000..1a30bdd --- /dev/null +++ b/models/intermediate/edeposit/int_edeposit__verification_fees.sql @@ -0,0 +1,60 @@ +{% set ok_status = ("Approved", "Flagged") %} +{% set rejected_status = "Rejected" %} +{% set rejected_fee = 0.25 %} +{% set cancellation_fee = 0.25 %} +{% set cancellation_threshold = 0.05 %} +with + int_edeposit__verifications as ( + select * from {{ ref("int_edeposit__verifications") }} + ), + edeposit_records as ( + select + id_verification, + id_user_partner, + id_booking, + is_cancelled, + creation_at_utc, + case + when verification_status in {{ ok_status }} + then nightly_fee_local * number_nights + else 0 + end as ok_status_fee, + case + when verification_status = '{{ rejected_status }}' + then {{ rejected_fee }} + else 0 + end as rejected_fee, + to_char(checkout_at_utc, 'YYYY-MM') as year_month_checkout + from int_edeposit__verifications + where version = 'V2' + ), + monthly_cancellation_threshold as ( + select + id_user_partner, + year_month_checkout, + case + when + sum(cast(is_cancelled as integer))::decimal / count(id_booking) + >= {{ cancellation_threshold }} + then true + else false + end as is_cancellation_threshold_surpassed + from edeposit_records + group by id_user_partner, year_month_checkout + ) +select + id_verification, + ok_status_fee, + rejected_fee, + case + when ct.is_cancellation_threshold_surpassed is true + then {{ cancellation_fee }} + else 0 + end as cancelled_fee +from edeposit_records er +left join + monthly_cancellation_threshold ct + on ( + er.id_user_partner = ct.id_user_partner + and er.year_month_checkout = ct.year_month_checkout + ) diff --git a/models/intermediate/edeposit/schema.yaml b/models/intermediate/edeposit/schema.yaml index b36b556..81edb50 100644 --- a/models/intermediate/edeposit/schema.yaml +++ b/models/intermediate/edeposit/schema.yaml @@ -206,3 +206,34 @@ models: - name: created_date_utc data_type: date description: "Date of creation of the verification in the system" + - name: int_edeposit__verification_fees + description: + "This table shows all fee charges per verification for E-deposit. + Cancellation fee is charged when the monthly rate of cancelled bookings over + total booking of the partner surpasses the threshold (currently set at 0.05). + Both cancellation and rejection fees are set to 0.25 though it might change" + columns: + - name: id_verification + data_type: text + description: "unique Superhog generated id for this verification" + tests: + - unique + - not_null + + - name: ok_status_fee + data_type: numeric + description: "fee charged for approved or flagged verifications" + tests: + - not_null + + - name: rejected_fee + data_type: numeric + description: "fee charged for rejected verifications" + tests: + - not_null + + - name: cancelled_fee + data_type: numeric + description: "fee charged for cancelled verifications" + tests: + - not_null From a99d4f622fc9bd5ae7d80ae4334446fa0ec85246 Mon Sep 17 00:00:00 2001 From: Joaquin Ossa Date: Tue, 3 Sep 2024 12:15:14 +0200 Subject: [PATCH 2/6] Modified model to only have fees --- .../int_edeposit__verification_fees.sql | 60 ++++++++++++++----- models/intermediate/edeposit/schema.yaml | 59 +++++++++++++++--- 2 files changed, 96 insertions(+), 23 deletions(-) diff --git a/models/intermediate/edeposit/int_edeposit__verification_fees.sql b/models/intermediate/edeposit/int_edeposit__verification_fees.sql index 1a30bdd..e7d87cf 100644 --- a/models/intermediate/edeposit/int_edeposit__verification_fees.sql +++ b/models/intermediate/edeposit/int_edeposit__verification_fees.sql @@ -7,25 +7,47 @@ with int_edeposit__verifications as ( select * from {{ ref("int_edeposit__verifications") }} ), + stg_core__edeposit_users as (select * from {{ ref("stg_core__edeposit_users") }}), + stg_xedotcom__exchange_rates as ( + select * from {{ ref("stg_xedotcom__exchange_rates") }} + ), edeposit_records as ( select - id_verification, - id_user_partner, - id_booking, - is_cancelled, - creation_at_utc, + v.id_verification, + v.id_user_partner, + v.id_booking, + v.currency, + er.rate, + v.is_cancelled, + v.creation_at_utc, case - when verification_status in {{ ok_status }} - then nightly_fee_local * number_nights + when v.verification_status in {{ ok_status }} + then v.nightly_fee_local * v.number_nights else 0 - end as ok_status_fee, + end as ok_status_fee_in_txn_currency, case - when verification_status = '{{ rejected_status }}' + when v.verification_status in {{ ok_status }} + then er.rate * v.nightly_fee_local * v.number_nights + else 0 + end as ok_status_fee_in_gbp, + case + when v.verification_status = '{{ rejected_status }}' then {{ rejected_fee }} else 0 - end as rejected_fee, - to_char(checkout_at_utc, 'YYYY-MM') as year_month_checkout - from int_edeposit__verifications + end as rejected_fee_in_txn_currency, + case + when v.verification_status = '{{ rejected_status }}' + then er.rate * {{ rejected_fee }} + else 0 + end as rejected_fee_in_gbp, + to_char(v.checkout_at_utc, 'YYYY-MM') as year_month_checkout + from int_edeposit__verifications v + inner join stg_core__edeposit_users eu on v.id_user_partner = eu.id_user_partner + inner join + stg_xedotcom__exchange_rates er + on er.from_currency = eu.currency + and er.rate_date_utc = v.checkout_date_utc + and er.to_currency = 'GBP' where version = 'V2' ), monthly_cancellation_threshold as ( @@ -44,13 +66,21 @@ with ) select id_verification, - ok_status_fee, - rejected_fee, + currency, + ok_status_fee_in_txn_currency, + ok_status_fee_in_gbp, + rejected_fee_in_txn_currency, + rejected_fee_in_gbp, case when ct.is_cancellation_threshold_surpassed is true then {{ cancellation_fee }} else 0 - end as cancelled_fee + end as cancelled_fee_in_txn_currency, + case + when ct.is_cancellation_threshold_surpassed is true + then rate * {{ cancellation_fee }} + else 0 + end as cancelled_fee_in_gbp from edeposit_records er left join monthly_cancellation_threshold ct diff --git a/models/intermediate/edeposit/schema.yaml b/models/intermediate/edeposit/schema.yaml index 81edb50..63ceeb5 100644 --- a/models/intermediate/edeposit/schema.yaml +++ b/models/intermediate/edeposit/schema.yaml @@ -211,7 +211,8 @@ models: "This table shows all fee charges per verification for E-deposit. Cancellation fee is charged when the monthly rate of cancelled bookings over total booking of the partner surpasses the threshold (currently set at 0.05). - Both cancellation and rejection fees are set to 0.25 though it might change" + Both cancellation and rejection fees are set to 0.25 though it might change. + Fees are both in the currency used by the user and in GBP" columns: - name: id_verification data_type: text @@ -220,20 +221,62 @@ models: - unique - not_null - - name: ok_status_fee - data_type: numeric - description: "fee charged for approved or flagged verifications" + - name: currency + data_type: text + description: "currency in which the transaction actually happened" tests: - not_null - - name: rejected_fee + - name: ok_status_fee_in_txn_currency data_type: numeric - description: "fee charged for rejected verifications" + description: "fee charged in used currency for approved or flagged verifications" tests: - not_null + - dbt_expectations.expect_column_values_to_be_between: + min_value: 0 + strictly: false - - name: cancelled_fee + - name: ok_status_fee_in_gbp data_type: numeric - description: "fee charged for cancelled verifications" + description: "fee charged in gbp for approved or flagged verifications" tests: - not_null + - dbt_expectations.expect_column_values_to_be_between: + min_value: 0 + strictly: false + + - name: rejected_fee_in_txn_currency + data_type: numeric + description: "fee charged in used currency for rejected verifications" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_between: + min_value: 0 + strictly: false + + - name: rejected_fee_in_gbp + data_type: numeric + description: "fee charged in gbp for rejected verifications" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_between: + min_value: 0 + strictly: false + + - name: cancelled_fee_in_txn_currency + data_type: numeric + description: "fee charged in used currency for cancelled verifications" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_between: + min_value: 0 + strictly: false + + - name: cancelled_fee_in_gbp + data_type: numeric + description: "fee charged in gbp for cancelled verifications" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_between: + min_value: 0 + strictly: false From 920733fceb9675955c444e7e08b85622fd6a930a Mon Sep 17 00:00:00 2001 From: Joaquin Ossa Date: Mon, 2 Sep 2024 11:16:51 +0200 Subject: [PATCH 3/6] Updating with Ray's comments --- models/intermediate/edeposit/schema.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/models/intermediate/edeposit/schema.yaml b/models/intermediate/edeposit/schema.yaml index 63ceeb5..b0148d0 100644 --- a/models/intermediate/edeposit/schema.yaml +++ b/models/intermediate/edeposit/schema.yaml @@ -206,6 +206,10 @@ models: - name: created_date_utc data_type: date description: "Date of creation of the verification in the system" + + - name: created_date_utc + data_type: date + description: "Date of creation of the verification in the system" - name: int_edeposit__verification_fees description: "This table shows all fee charges per verification for E-deposit. @@ -280,3 +284,4 @@ models: - dbt_expectations.expect_column_values_to_be_between: min_value: 0 strictly: false + description: "Timestamp of creation of the verification in the system" From 560e2a599454e2e75083a53bec9752e2578d1ff9 Mon Sep 17 00:00:00 2001 From: Joaquin Ossa Date: Tue, 3 Sep 2024 12:15:14 +0200 Subject: [PATCH 4/6] Modified model to only have fees --- models/intermediate/edeposit/schema.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/models/intermediate/edeposit/schema.yaml b/models/intermediate/edeposit/schema.yaml index b0148d0..e3cb139 100644 --- a/models/intermediate/edeposit/schema.yaml +++ b/models/intermediate/edeposit/schema.yaml @@ -284,4 +284,3 @@ models: - dbt_expectations.expect_column_values_to_be_between: min_value: 0 strictly: false - description: "Timestamp of creation of the verification in the system" From c24c87533648ede5c9faec3de64c8dc717d179d9 Mon Sep 17 00:00:00 2001 From: Joaquin Ossa Date: Tue, 3 Sep 2024 15:04:44 +0200 Subject: [PATCH 5/6] Fixed test error --- .../edeposit/int_edeposit__verification_fees.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/models/intermediate/edeposit/int_edeposit__verification_fees.sql b/models/intermediate/edeposit/int_edeposit__verification_fees.sql index e7d87cf..f1c2eeb 100644 --- a/models/intermediate/edeposit/int_edeposit__verification_fees.sql +++ b/models/intermediate/edeposit/int_edeposit__verification_fees.sql @@ -7,7 +7,7 @@ with int_edeposit__verifications as ( select * from {{ ref("int_edeposit__verifications") }} ), - stg_core__edeposit_users as (select * from {{ ref("stg_core__edeposit_users") }}), + stg_core__edeposit_user as (select * from {{ ref("stg_core__edeposit_user") }}), stg_xedotcom__exchange_rates as ( select * from {{ ref("stg_xedotcom__exchange_rates") }} ), @@ -16,7 +16,7 @@ with v.id_verification, v.id_user_partner, v.id_booking, - v.currency, + eu.currency, er.rate, v.is_cancelled, v.creation_at_utc, @@ -42,7 +42,7 @@ with end as rejected_fee_in_gbp, to_char(v.checkout_at_utc, 'YYYY-MM') as year_month_checkout from int_edeposit__verifications v - inner join stg_core__edeposit_users eu on v.id_user_partner = eu.id_user_partner + inner join stg_core__edeposit_user eu on v.id_user_partner = eu.id_user_partner inner join stg_xedotcom__exchange_rates er on er.from_currency = eu.currency From 7d8983e0ddf38fb9669228bb83160f84a7999a5f Mon Sep 17 00:00:00 2001 From: Joaquin Ossa Date: Tue, 3 Sep 2024 17:05:13 +0200 Subject: [PATCH 6/6] Fixed model, added created_date and checkout_date --- .../int_edeposit__verification_fees.sql | 43 ++++++++++--------- models/intermediate/edeposit/schema.yaml | 29 +++++++++++-- 2 files changed, 48 insertions(+), 24 deletions(-) diff --git a/models/intermediate/edeposit/int_edeposit__verification_fees.sql b/models/intermediate/edeposit/int_edeposit__verification_fees.sql index f1c2eeb..9a3b52f 100644 --- a/models/intermediate/edeposit/int_edeposit__verification_fees.sql +++ b/models/intermediate/edeposit/int_edeposit__verification_fees.sql @@ -8,8 +8,8 @@ with select * from {{ ref("int_edeposit__verifications") }} ), stg_core__edeposit_user as (select * from {{ ref("stg_core__edeposit_user") }}), - stg_xedotcom__exchange_rates as ( - select * from {{ ref("stg_xedotcom__exchange_rates") }} + int_daily_currency_exchange_rates as ( + select * from {{ ref("int_daily_currency_exchange_rates") }} ), edeposit_records as ( select @@ -17,9 +17,8 @@ with v.id_user_partner, v.id_booking, eu.currency, - er.rate, + cer.rate, v.is_cancelled, - v.creation_at_utc, case when v.verification_status in {{ ok_status }} then v.nightly_fee_local * v.number_nights @@ -27,7 +26,7 @@ with end as ok_status_fee_in_txn_currency, case when v.verification_status in {{ ok_status }} - then er.rate * v.nightly_fee_local * v.number_nights + then cer.rate * v.nightly_fee_local * v.number_nights else 0 end as ok_status_fee_in_gbp, case @@ -37,17 +36,17 @@ with end as rejected_fee_in_txn_currency, case when v.verification_status = '{{ rejected_status }}' - then er.rate * {{ rejected_fee }} + then cer.rate * {{ rejected_fee }} else 0 end as rejected_fee_in_gbp, to_char(v.checkout_at_utc, 'YYYY-MM') as year_month_checkout from int_edeposit__verifications v inner join stg_core__edeposit_user eu on v.id_user_partner = eu.id_user_partner - inner join - stg_xedotcom__exchange_rates er - on er.from_currency = eu.currency - and er.rate_date_utc = v.checkout_date_utc - and er.to_currency = 'GBP' + left join + int_daily_currency_exchange_rates cer + on cer.from_currency = eu.currency + and cer.rate_date_utc = v.checkout_date_utc + and cer.to_currency = 'GBP' where version = 'V2' ), monthly_cancellation_threshold as ( @@ -65,12 +64,13 @@ with group by id_user_partner, year_month_checkout ) select - id_verification, - currency, - ok_status_fee_in_txn_currency, - ok_status_fee_in_gbp, - rejected_fee_in_txn_currency, - rejected_fee_in_gbp, + v.id_verification, + v.id_booking, + er.currency, + er.ok_status_fee_in_txn_currency, + er.ok_status_fee_in_gbp, + er.rejected_fee_in_txn_currency, + er.rejected_fee_in_gbp, case when ct.is_cancellation_threshold_surpassed is true then {{ cancellation_fee }} @@ -80,9 +80,12 @@ select when ct.is_cancellation_threshold_surpassed is true then rate * {{ cancellation_fee }} else 0 - end as cancelled_fee_in_gbp -from edeposit_records er -left join + end as cancelled_fee_in_gbp, + v.created_date_utc, + v.checkout_date_utc +from int_edeposit__verifications v +inner join edeposit_records er on er.id_verification = v.id_verification +inner join monthly_cancellation_threshold ct on ( er.id_user_partner = ct.id_user_partner diff --git a/models/intermediate/edeposit/schema.yaml b/models/intermediate/edeposit/schema.yaml index e3cb139..55c6463 100644 --- a/models/intermediate/edeposit/schema.yaml +++ b/models/intermediate/edeposit/schema.yaml @@ -21,7 +21,10 @@ models: - name: id_booking data_type: text - description: "unique Superhog generated id for a booking" + description: + "unique Superhog generated id for a booking. + note that this could be duplicated and both will be charged, + it's up to the user to no generate duplicate verifications" - name: id_user_partner data_type: text @@ -207,9 +210,6 @@ models: data_type: date description: "Date of creation of the verification in the system" - - name: created_date_utc - data_type: date - description: "Date of creation of the verification in the system" - name: int_edeposit__verification_fees description: "This table shows all fee charges per verification for E-deposit. @@ -225,6 +225,15 @@ models: - unique - not_null + - name: id_booking + data_type: text + description: + "unique Superhog generated id for a booking. + note that this could be duplicated and both will be charged, + it's up to the user to no generate duplicate verifications" + tests: + - not_null + - name: currency data_type: text description: "currency in which the transaction actually happened" @@ -284,3 +293,15 @@ models: - dbt_expectations.expect_column_values_to_be_between: min_value: 0 strictly: false + + - name: checkout_date_utc + data_type: date + description: "Date of checkout for the booking" + tests: + - not_null + + - name: created_date_utc + data_type: date + description: "Date of creation of the verification in the system" + tests: + - not_null \ No newline at end of file