From 68c509050fa8988b29b8ab6744a60d5e01ead2e4 Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Tue, 17 Sep 2024 10:32:11 +0200 Subject: [PATCH 01/19] add payway to sources --- models/staging/core/_core_sources.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/models/staging/core/_core_sources.yml b/models/staging/core/_core_sources.yml index dccc25a..426a0bc 100644 --- a/models/staging/core/_core_sources.yml +++ b/models/staging/core/_core_sources.yml @@ -232,4 +232,6 @@ sources: - name: AccommodationToProductBundle identifier: AccommodationToProductBundle - name: ElectronicDepositUser - identifier: ElectronicDepositUser \ No newline at end of file + identifier: ElectronicDepositUser + - name: PayAway + identifier: PayAway From 96d18116ed4edba8885b22adf66edbb2b69cbb8e Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Tue, 17 Sep 2024 10:36:43 +0200 Subject: [PATCH 02/19] add staging model --- models/staging/core/stg_core__PayAway.sql | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 models/staging/core/stg_core__PayAway.sql diff --git a/models/staging/core/stg_core__PayAway.sql b/models/staging/core/stg_core__PayAway.sql new file mode 100644 index 0000000..115ef7e --- /dev/null +++ b/models/staging/core/stg_core__PayAway.sql @@ -0,0 +1,21 @@ +with + raw_payaway as (select * from {{ source("core", "PayAway") }}), + stg_core__payaway as ( + select + {{ adapter.quote("Id") }} as id_payaway_plan, + {{ adapter.quote("CurrencyId") }} as id_currency, + {{ adapter.quote("SuperhogUserId") }} as id_user_host, + {{ adapter.quote("StartDate") }} as start_at_utc, + {{ adapter.quote("EndDate") }} as end_at_utc, + {{ adapter.quote("EndDate") }} is null as has_no_end_date, + {{ adapter.quote("PayAwayPercentage") }} as payaway_percentage, + {{ adapter.quote("PayAwayMinimumCommission") }} + as payaway_minimum_commission_local_curr, + {{ adapter.quote("CreatedDate") }} as created_at_utc, + {{ adapter.quote("UpdatedDate") }} as updated_at_utc, + {{ adapter.quote("_airbyte_extracted_at") }} as dwh_extracted_at_utc + + from raw_payaway + ) +select * +from stg_core__payaway From 68a9eafb145a04910ed78e8c432f44eac5de8fff Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Tue, 17 Sep 2024 10:36:58 +0200 Subject: [PATCH 03/19] typo in file name --- .../staging/core/{stg_core__PayAway.sql => stg_core__payaway.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename models/staging/core/{stg_core__PayAway.sql => stg_core__payaway.sql} (100%) diff --git a/models/staging/core/stg_core__PayAway.sql b/models/staging/core/stg_core__payaway.sql similarity index 100% rename from models/staging/core/stg_core__PayAway.sql rename to models/staging/core/stg_core__payaway.sql From 624c70872fadc6194fe2ee53c0315e6b292d9da6 Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Tue, 17 Sep 2024 11:16:01 +0200 Subject: [PATCH 04/19] docs and tests --- models/staging/core/schema.yml | 126 +++++++++++++++++++++++++++++++++ 1 file changed, 126 insertions(+) diff --git a/models/staging/core/schema.yml b/models/staging/core/schema.yml index 5905c00..542beef 100644 --- a/models/staging/core/schema.yml +++ b/models/staging/core/schema.yml @@ -391,3 +391,129 @@ models: data_type: timestamp description: | Timestamp of when this data was extracted into DWH. + - name: stg_core__payaway + description: | + Contains all the PayAway plans, which are basically the settings for + host-takes-waiver plans with our host customers. All plans have a start + and end point in time, which means that any waivers that happen during + the range of plan should use the settings of this plan as a reference. + + Plans can be open ended, as in not having a specified end in time. This + just means they are indefinitely active until someone changes it. + + Plans can also have a planned end time which sits in the future. + tests: + - dbt_expectations.expect_column_pair_values_A_to_be_greater_than_B: + # The end timestamp should always be after the start timestamp + column_A: end_at_utc + column_B: start_at_utc + or_equal: False + - dbt_expectations.expect_column_pair_values_A_to_be_greater_than_B: + # The updated timestamp should always be equal or after the creation + # timestamp + column_A: updated_at_utc + column_B: created_at_utc + or_equal: True + + columns: + - name: id_payaway_plan + data_type: bigint + description: The unique id for this plan. + tests: + - unique + - not_null + + - name: id_currency + data_type: bigint + description: | + The Superhog ID of the currency that this record applies to. + tests: + - not_null + + - name: id_user_host + data_type: character varying + description: | + The Superhog ID of the host user this record applies to. + tests: + - not_null + + - name: start_at_utc + data_type: timestamp without time zone + description: | + The point in time in which this plan became active. It can never be + null. + tests: + - not_null + + - name: end_at_utc + data_type: timestamp without time zone + description: | + The point in time in which this plan will stop being active. It can be + null, which means the plan has no planned end date yet. Should this + column have a value, it should always be after the start time of the + plan. + + - name: has_no_end_date + data_type: boolean + description: | + Syntactic sugar for checking if the plan has a specified end date. + tests: + - not_null + + - name: payaway_percentage + data_type: numeric + description: | + The percentage of the Waiver payments that Superhog will keep as a + a fee. Should be between 0% and a 100%. 0% is a valid amount. + + This means that the Superhog fee is computed as: + Waiver Amount * payaway_percentage. + + If the amount that comes out of this calculation is smaller than the + amount in column payaway_minimum_commission_local_curr, then the + Superhog fee becomes payaway_minimum_commission_local_curr instead. + So, the final logic becomes: + MAX( + Waiver Amount * payaway_percentage, + payaway_minimum_commission_local_curr + ) + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_between: + min_value: 0 + max_value: 1 + strictly: false + + - name: payaway_minimum_commission_local_curr + data_type: numeric + description: | + The minimum fee that we take from each waiver payment, specified in + the currency of the guest payment (so if this record is in dollars, it + means it applies to guest payments made in dollars). We will never + charge less than this. This can be 0. + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_between: + min_value: 0 + strictly: false + + - name: created_at_utc + data_type: timestamp + description: | + Timestamp of when the pay away plan was created. + tests: + - not_null + + - name: updated_at_utc + data_type: timestamp + description: | + Timestamp of when the pay away plan to currency was last updated + tests: + - not_null + + - name: dwh_extracted_at_utc + data_type: timestamp + description: | + Timestamp of when this data was extracted into DWH. + tests: + - not_null From ac87ab4cfd4032892adc99d06d2a2ecf78fe7cf2 Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Tue, 17 Sep 2024 11:56:21 +0200 Subject: [PATCH 05/19] create file --- models/intermediate/core/int_core__payaway.sql | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 models/intermediate/core/int_core__payaway.sql diff --git a/models/intermediate/core/int_core__payaway.sql b/models/intermediate/core/int_core__payaway.sql new file mode 100644 index 0000000..d6b570b --- /dev/null +++ b/models/intermediate/core/int_core__payaway.sql @@ -0,0 +1,4 @@ +with stg_core__payaway as (select * from {{ ref("stg_core__payaway") }}) + +select * +from stg_core__payaway From 1c04c69e897cc6e49557941e8b3de24cd0c03ab0 Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Tue, 17 Sep 2024 11:59:04 +0200 Subject: [PATCH 06/19] fix comments --- models/staging/core/schema.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/models/staging/core/schema.yml b/models/staging/core/schema.yml index 542beef..2dce224 100644 --- a/models/staging/core/schema.yml +++ b/models/staging/core/schema.yml @@ -395,8 +395,11 @@ models: description: | Contains all the PayAway plans, which are basically the settings for host-takes-waiver plans with our host customers. All plans have a start - and end point in time, which means that any waivers that happen during - the range of plan should use the settings of this plan as a reference. + and end point in time. + + As of today, though, the reality is that our monthly invoicing and + settlement process with hosts only looks at the payaway plan that was + active when a month finished. Plans can be open ended, as in not having a specified end in time. This just means they are indefinitely active until someone changes it. From f3e9985d0f6d9131d1363190f5777585e9bfb06e Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Tue, 17 Sep 2024 12:16:52 +0200 Subject: [PATCH 07/19] model and docs --- .../intermediate/core/int_core__payaway.sql | 21 ++++- models/intermediate/core/schema.yml | 86 +++++++++++++++++++ 2 files changed, 104 insertions(+), 3 deletions(-) diff --git a/models/intermediate/core/int_core__payaway.sql b/models/intermediate/core/int_core__payaway.sql index d6b570b..918c54f 100644 --- a/models/intermediate/core/int_core__payaway.sql +++ b/models/intermediate/core/int_core__payaway.sql @@ -1,4 +1,19 @@ -with stg_core__payaway as (select * from {{ ref("stg_core__payaway") }}) +with + stg_core__payaway as (select * from {{ ref("stg_core__payaway") }}), + stg_core__currency as (select * from {{ ref("stg_core__currency") }}) -select * -from stg_core__payaway +-- replace currency +select + pa.id_payaway_plan, + pa.id_user_host, + pa.start_at_utc, + pa.end_at_utc, + pa.has_no_end_date, + pa.payaway_percentage, + pa.payaway_minimum_commission_local_curr, + c.iso4217_code as currency, + pa.created_at_utc, + pa.updated_at_utc, + pa.dwh_extracted_at_utc +from stg_core__payaway pa +left join stg_core__currency c on pa.id_currency = c.id_currency diff --git a/models/intermediate/core/schema.yml b/models/intermediate/core/schema.yml index 8a57965..976b09a 100644 --- a/models/intermediate/core/schema.yml +++ b/models/intermediate/core/schema.yml @@ -3252,3 +3252,89 @@ models: description: | Informative field of how many different billing countries are associated to this Deal based on the user account configuration. + - name: int_core__payaway + description: | + Contains all the PayAway plans, which are basically the settings for + host-takes-waiver plans with our host customers. All plans have a start + and end point in time, which means that any waivers that happen during + the range of plan should use the settings of this plan as a reference. + + Plans can be open ended, as in not having a specified end in time. This + just means they are indefinitely active until someone changes it. + + Plans can also have a planned end time which sits in the future. + columns: + - name: id_payaway_plan + data_type: bigint + description: "The unique id for this plan." + tests: + - not_null + - unique + + - name: id_user_host + data_type: character varying + description: "The Superhog ID of the host user this record applies to." + + - name: start_at_utc + data_type: timestamp without time zone + description: + The point in time in which this plan became active. It can never be + null. + + - name: end_at_utc + data_type: timestamp without time zone + description: + The point in time in which this plan will stop being active. It can + be null, which means the plan has no planned end date yet. Should this + column have a value, it should always be after the start time of the + plan. + + - name: has_no_end_date + data_type: boolean + description: Syntactic sugar for checking if the plan has a specified end date. + + - name: payaway_percentage + data_type: numeric + description: | + The percentage of the Waiver payments that Superhog will keep as a + a fee. Should be between 0% and a 100%. 0% is a valid amount. + + This means that the Superhog fee is computed as: + Waiver Amount * payaway_percentage. + + If the amount that comes out of this calculation is smaller than the + amount in column payaway_minimum_commission_local_curr, then the + Superhog fee becomes payaway_minimum_commission_local_curr instead. + So, the final logic becomes: + MAX( + Waiver Amount * payaway_percentage, + payaway_minimum_commission_local_curr + ) + + - name: payaway_minimum_commission_local_curr + data_type: numeric + description: + The minimum fee that we take from each waiver payment, specified in + the currency of the guest payment (so if this record is in dollars, it + means it applies to guest payments made in dollars). We will never + charge less than this. This can be 0. + + - name: currency + data_type: character varying + description: + The ISO 4217 code for the currency of this record. Must always be + filled, otherwise the records is meaningless. + tests: + - not_null + + - name: created_at_utc + data_type: timestamp without time zone + description: Timestamp of when the pay away plan was created. + + - name: updated_at_utc + data_type: timestamp without time zone + description: Timestamp of when the pay away plan to currency was last updated + + - name: dwh_extracted_at_utc + data_type: timestamp with time zone + description: Timestamp of when this data was extracted into DWH. From 049c2d3e31699720502a0a5e08ea3522b9a2ecf0 Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Tue, 17 Sep 2024 15:47:45 +0200 Subject: [PATCH 08/19] model and docs+tests --- .../core/int_core__payaway_per_month_user.sql | 51 +++++++++++++ models/intermediate/core/schema.yml | 74 +++++++++++++++++++ 2 files changed, 125 insertions(+) create mode 100644 models/intermediate/core/int_core__payaway_per_month_user.sql diff --git a/models/intermediate/core/int_core__payaway_per_month_user.sql b/models/intermediate/core/int_core__payaway_per_month_user.sql new file mode 100644 index 0000000..dceb0e0 --- /dev/null +++ b/models/intermediate/core/int_core__payaway_per_month_user.sql @@ -0,0 +1,51 @@ +with + int_core__payaway as (select * from {{ ref("int_core__payaway") }}), + int_dates as (select * from {{ ref("int_dates") }}), + active_payaway_per_month as ( + select distinct + pa.id_payaway_plan, + pa.id_user_host, + pa.start_at_utc, + pa.end_at_utc, + pa.created_at_utc, + d.month_start_date as active_in_month_start_date_utc, + d.month_end_date as active_in_month_end_date_utc + from int_core__payaway pa + inner join + int_dates d + on d.date_day + between pa.start_at_utc and coalesce(pa.end_at_utc, '2099-12-31T23:59:59Z') + -- open ended plans have null values, so we apply this to make the + -- between work + where + d.date_day < (date_trunc('month', current_date) + interval '1 month')::date + -- no sense in matching stuff in the future. The above statement returns + -- the first day of next month + ), + sorted_payaway_plans as ( + select + id_payaway_plan, + id_user_host, + start_at_utc, + end_at_utc, + created_at_utc, + active_in_month_start_date_utc, + active_in_month_end_date_utc, + row_number() over ( + partition by id_user_host, active_in_month_end_date_utc + order by created_at_utc desc + -- the latest active, created plan is the one that will be + -- valid for the month + ) as rn + from active_payaway_per_month + ) +select + id_payaway_plan, + id_user_host, + start_at_utc, + end_at_utc, + created_at_utc, + active_in_month_start_date_utc, + active_in_month_end_date_utc +from sorted_payaway_plans +where rn = 1 diff --git a/models/intermediate/core/schema.yml b/models/intermediate/core/schema.yml index 976b09a..605a36c 100644 --- a/models/intermediate/core/schema.yml +++ b/models/intermediate/core/schema.yml @@ -3184,6 +3184,80 @@ models: plans happened during that month. tests: - not_null + - name: int_core__payaway_per_month_user + description: | + This model contains the payaway plans that were considered as active + for the invoicing process each month. This is, given that more than + one plan coexist within the same month, we take the one plan that + was active at the end of the month. This is the one that should apply for + the invoicing of that month, indisctintly of the fact that there was other + plans active before. + + The time scope of the model is limited to the current month. This means + that, even though some plans will end in future dates or have no planned + end date, th + + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - id_user_host + - id_payaway_plan + - active_in_month_start_date_utc + + columns: + - name: id_payaway_plan + data_type: bigint + description: | + The unique identifier of this table, representing + the identifier of the payaway plan. + tests: + - not_null + + - name: id_user_host + data_type: string + description: | + The unique identifier of the user host that has + a price plan. + tests: + - not_null + + - name: start_at_utc + data_type: timestamp + description: | + Original timestamp of when a given plan started to be active. + tests: + - not_null + + - name: end_at_utc + data_type: timestamp + description: | + Original timestamp of when a given plan stopped being active. If it's + null, it means the plan is open ended (has no planned end date yet). + + - name: created_at_utc + data_type: timestamp + description: | + Original timestamp of when a given plan was created. + tests: + - not_null + + - name: active_in_month_start_date_utc + data_type: date + description: | + Date that refers to the first day of the month on which we will + consider a plan as active. If we're interested in retrieving the + information from June, this date will be the 1st of June. + tests: + - not_null + + - name: active_in_month_end_date_utc + data_type: date + description: | + Date that refers to the last day of the month on which we will + consider a plan as active. If we're interested in retrieving the + information from June, this date will be the 30th of June. + tests: + - not_null - name: int_core__deal description: | From a20d511b938075b74e137cb13b82861ae9900cd3 Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Tue, 17 Sep 2024 16:12:38 +0200 Subject: [PATCH 09/19] missing text --- models/intermediate/core/schema.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/models/intermediate/core/schema.yml b/models/intermediate/core/schema.yml index 605a36c..350ce8f 100644 --- a/models/intermediate/core/schema.yml +++ b/models/intermediate/core/schema.yml @@ -3195,7 +3195,8 @@ models: The time scope of the model is limited to the current month. This means that, even though some plans will end in future dates or have no planned - end date, th + end date, this table will only reflect activeness within months up to the + current month. tests: - dbt_utils.unique_combination_of_columns: From 2dd4f66189b8c3c1cae5167f8821e51eca939aea Mon Sep 17 00:00:00 2001 From: Joaquin Ossa Date: Tue, 17 Sep 2024 17:12:46 +0200 Subject: [PATCH 10/19] update int_core__guest_satisfaction_responses to version 2 of verification_payments --- .../core/int_core__guest_satisfaction_responses.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/intermediate/core/int_core__guest_satisfaction_responses.sql b/models/intermediate/core/int_core__guest_satisfaction_responses.sql index eb4eb49..e751200 100644 --- a/models/intermediate/core/int_core__guest_satisfaction_responses.sql +++ b/models/intermediate/core/int_core__guest_satisfaction_responses.sql @@ -1,6 +1,6 @@ with int_core__verification_payments as ( - select * from {{ ref("int_core__verification_payments") }} + select * from {{ ref("int_core__verification_payments", version=2) }} ), stg_core__guest_satisfaction_responses as ( select * from {{ ref("stg_core__guest_satisfaction_responses") }} From 78e7e37c5e5bac25efc77555e28f82773251ae1a Mon Sep 17 00:00:00 2001 From: Joaquin Ossa Date: Tue, 17 Sep 2024 15:07:00 +0200 Subject: [PATCH 11/19] Update model core__verification_payments to include amounts without taxes --- .../core/core__verification_payments.sql | 10 ++-- models/reporting/core/schema.yml | 54 ++++++++++++++++++- 2 files changed, 60 insertions(+), 4 deletions(-) diff --git a/models/reporting/core/core__verification_payments.sql b/models/reporting/core/core__verification_payments.sql index 5297fa0..d798780 100644 --- a/models/reporting/core/core__verification_payments.sql +++ b/models/reporting/core/core__verification_payments.sql @@ -1,6 +1,6 @@ with int_core__verification_payments as ( - select * from {{ ref("int_core__verification_payments") }} + select * from {{ ref("int_core__verification_payments", version=2) }} ) select vp.id_verification_to_payment as id_verification_to_payment, @@ -19,9 +19,13 @@ select vp.id_guest_user as id_guest_user, vp.id_verification as id_verification, vp.verification_payment_type as verification_payment_type, - vp.amount_in_txn_currency as amount_in_txn_currency, + vp.total_amount_in_txn_currency as amount_in_txn_currency, -- LEGACY + vp.total_amount_in_txn_currency as total_amount_in_txn_currency, + vp.amount_without_taxes_in_txn_currency as amount_without_taxes_in_txn_currency, vp.currency as currency, - vp.amount_in_gbp as amount_in_gbp, + vp.total_amount_in_gbp as amount_in_gbp, -- LEGACY + vp.total_amount_in_gbp as total_amount_in_gbp, + vp.amount_without_taxes_in_gbp as amount_without_taxes_in_gbp, vp.payment_status as payment_status, vp.notes as notes from int_core__verification_payments vp diff --git a/models/reporting/core/schema.yml b/models/reporting/core/schema.yml index 1b48715..359cbf8 100644 --- a/models/reporting/core/schema.yml +++ b/models/reporting/core/schema.yml @@ -29,10 +29,14 @@ models: - name: payment_due_at_utc data_type: timestamp without time zone description: The point in time at which this payment had to be paid. + tests: + - not_null - name: payment_due_date_utc data_type: date description: The date on which this payment had to be paid. + tests: + - not_null - name: payment_paid_at_utc data_type: timestamp without time zone @@ -83,30 +87,78 @@ models: - name: id_guest_user data_type: character varying description: The UUID of the guest user in the Superhog backend. + tests: + - not_null - name: id_verification data_type: bigint description: The ID of the verification that generated this payment. + tests: + - not_null - name: verification_payment_type data_type: character varying description: "The payment type. Can be one of: Waiver, Fee, Deposit, Reschedule, Cancellation." - name: amount_in_txn_currency + data_type: numeric + description: The payment amount in the currency in which the transaction actually happened. If the guest paid in Australian Dollars, this is measured in AUD. (To be decommissioned) + tests: + - dbt_expectations.expect_column_values_to_be_between: + min_value: 0 + strictly: false + + - name: total_amount_in_txn_currency data_type: numeric description: The payment amount in the currency in which the transaction actually happened. If the guest paid in Australian Dollars, this is measured in AUD. + tests: + - dbt_expectations.expect_column_values_to_be_between: + min_value: 0 + strictly: false + + - name: amount_without_taxes_in_txn_currency + data_type: numeric + description: The payment amount without taxes in the currency in which the transaction actually happened. If the guest paid in Australian Dollars, this is measured in AUD. (To be decommissioned) + tests: + - dbt_expectations.expect_column_values_to_be_between: + min_value: 0 + strictly: false - name: currency data_type: character varying description: The currency in which the transaction actually happened. If the guest paid in Australian Dollars, this is measured in AUD. + tests: + - not_null - name: amount_in_gbp data_type: numeric description: The payment amount value, converted to GBP, using the exchange rate for the day on which the payment happened. + tests: + - dbt_expectations.expect_column_values_to_be_between: + min_value: 0 + strictly: false + + - name: total_amount_in_gbp + data_type: numeric + description: The payment amount value, converted to GBP, using the exchange rate for the day on which the payment happened. + tests: + - dbt_expectations.expect_column_values_to_be_between: + min_value: 0 + strictly: false + + - name: amount_without_taxes_in_gbp + data_type: numeric + description: The payment amount value without taxes, converted to GBP, using the exchange rate for the day on which the payment happened. + tests: + - dbt_expectations.expect_column_values_to_be_between: + min_value: 0 + strictly: false - name: payment_status data_type: character varying - description: "The status of the payment. It can be one of: Paid, Refunded, Refund Failed." + description: "The status of the payment. It can be one of: Paid, Refunded, Refund Failed, Cancelled, Paid Manually, Unpaid." + tests: + - not_null - name: notes data_type: character varying From e854f2a99aadd34032466259c483b8ee7740ddca Mon Sep 17 00:00:00 2001 From: Joaquin Ossa Date: Tue, 17 Sep 2024 17:10:14 +0200 Subject: [PATCH 12/19] commit --- package-lock.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.yml b/package-lock.yml index 2b520db..d7ed31d 100644 --- a/package-lock.yml +++ b/package-lock.yml @@ -4,7 +4,7 @@ packages: - package: calogica/dbt_expectations version: 0.9.0 - package: dbt-labs/dbt_utils - version: 1.1.1 + version: 1.2.0 - package: calogica/dbt_date version: 0.8.1 -sha1_hash: 0d8e1bf7188f14d46813097c65818e3f96237266 +sha1_hash: ceec21d8037429db57330b6f23cfdc761bbb7698 From 9e62786aa9b844ade49d22600404f62e4683b0a8 Mon Sep 17 00:00:00 2001 From: Joaquin Ossa Date: Wed, 18 Sep 2024 11:26:23 +0200 Subject: [PATCH 13/19] commit2 --- models/intermediate/core/schema.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/models/intermediate/core/schema.yml b/models/intermediate/core/schema.yml index 350ce8f..222d707 100644 --- a/models/intermediate/core/schema.yml +++ b/models/intermediate/core/schema.yml @@ -932,7 +932,7 @@ models: date before a starting date. - name: int_core__verification_payments - latest_version: 1 + latest_version: 2 description: >- A simplified table that holds guest journey payments with details around when they happen, what service was being paid, what was the related @@ -1020,6 +1020,8 @@ models: versions: - v: 1 deprecation_date: 2024-10-15 00:00:00.00+00:00 + config: + alias: int_core__verification_payments - v: 2 columns: From 40bfa60016f86590c91686783f134df7c6eb0c80 Mon Sep 17 00:00:00 2001 From: Joaquin Ossa Date: Wed, 18 Sep 2024 12:41:00 +0200 Subject: [PATCH 14/19] core__verification_payments_v2 --- ...=> int_core__verification_payments_v1.sql} | 0 .../core/core__verification_payments_v1.sql | 27 +++++++ ...sql => core__verification_payments_v2.sql} | 2 - models/reporting/core/schema.yml | 76 +++++++++++-------- 4 files changed, 71 insertions(+), 34 deletions(-) rename models/intermediate/core/{int_core__verification_payments.sql => int_core__verification_payments_v1.sql} (100%) create mode 100644 models/reporting/core/core__verification_payments_v1.sql rename models/reporting/core/{core__verification_payments.sql => core__verification_payments_v2.sql} (91%) diff --git a/models/intermediate/core/int_core__verification_payments.sql b/models/intermediate/core/int_core__verification_payments_v1.sql similarity index 100% rename from models/intermediate/core/int_core__verification_payments.sql rename to models/intermediate/core/int_core__verification_payments_v1.sql diff --git a/models/reporting/core/core__verification_payments_v1.sql b/models/reporting/core/core__verification_payments_v1.sql new file mode 100644 index 0000000..afd5715 --- /dev/null +++ b/models/reporting/core/core__verification_payments_v1.sql @@ -0,0 +1,27 @@ +with + int_core__verification_payments as ( + select * from {{ ref("int_core__verification_payments", version=1) }} + ) +select + vp.id_verification_to_payment as id_verification_to_payment, + vp.id_payment as id_payment, + vp.is_refundable as is_refundable, + vp.payment_due_at_utc as payment_due_at_utc, + vp.payment_due_date_utc as payment_due_date_utc, + vp.payment_paid_at_utc as payment_paid_at_utc, + vp.payment_paid_date_utc as payment_paid_date_utc, + vp.payment_reference as payment_reference, + vp.refund_due_at_utc as refund_due_at_utc, + vp.refund_due_date_utc as refund_due_date_utc, + vp.payment_refunded_at_utc as payment_refunded_at_utc, + vp.payment_refunded_date_utc as payment_refunded_date_utc, + vp.refund_payment_reference as refund_payment_reference, + vp.id_guest_user as id_guest_user, + vp.id_verification as id_verification, + vp.verification_payment_type as verification_payment_type, + vp.amount_in_txn_currency as amount_in_txn_currency, + vp.currency as currency, + vp.amount_in_gbp as amount_in_gbp, + vp.payment_status as payment_status, + vp.notes as notes +from int_core__verification_payments vp diff --git a/models/reporting/core/core__verification_payments.sql b/models/reporting/core/core__verification_payments_v2.sql similarity index 91% rename from models/reporting/core/core__verification_payments.sql rename to models/reporting/core/core__verification_payments_v2.sql index d798780..4db5f13 100644 --- a/models/reporting/core/core__verification_payments.sql +++ b/models/reporting/core/core__verification_payments_v2.sql @@ -19,11 +19,9 @@ select vp.id_guest_user as id_guest_user, vp.id_verification as id_verification, vp.verification_payment_type as verification_payment_type, - vp.total_amount_in_txn_currency as amount_in_txn_currency, -- LEGACY vp.total_amount_in_txn_currency as total_amount_in_txn_currency, vp.amount_without_taxes_in_txn_currency as amount_without_taxes_in_txn_currency, vp.currency as currency, - vp.total_amount_in_gbp as amount_in_gbp, -- LEGACY vp.total_amount_in_gbp as total_amount_in_gbp, vp.amount_without_taxes_in_gbp as amount_without_taxes_in_gbp, vp.payment_status as payment_status, diff --git a/models/reporting/core/schema.yml b/models/reporting/core/schema.yml index 359cbf8..2b966d7 100644 --- a/models/reporting/core/schema.yml +++ b/models/reporting/core/schema.yml @@ -2,6 +2,7 @@ version: 2 models: - name: core__verification_payments + latest_version: 2 description: | Payments that have happened as part of the Guest Journey. @@ -108,22 +109,6 @@ models: min_value: 0 strictly: false - - name: total_amount_in_txn_currency - data_type: numeric - description: The payment amount in the currency in which the transaction actually happened. If the guest paid in Australian Dollars, this is measured in AUD. - tests: - - dbt_expectations.expect_column_values_to_be_between: - min_value: 0 - strictly: false - - - name: amount_without_taxes_in_txn_currency - data_type: numeric - description: The payment amount without taxes in the currency in which the transaction actually happened. If the guest paid in Australian Dollars, this is measured in AUD. (To be decommissioned) - tests: - - dbt_expectations.expect_column_values_to_be_between: - min_value: 0 - strictly: false - - name: currency data_type: character varying description: The currency in which the transaction actually happened. If the guest paid in Australian Dollars, this is measured in AUD. @@ -138,22 +123,6 @@ models: min_value: 0 strictly: false - - name: total_amount_in_gbp - data_type: numeric - description: The payment amount value, converted to GBP, using the exchange rate for the day on which the payment happened. - tests: - - dbt_expectations.expect_column_values_to_be_between: - min_value: 0 - strictly: false - - - name: amount_without_taxes_in_gbp - data_type: numeric - description: The payment amount value without taxes, converted to GBP, using the exchange rate for the day on which the payment happened. - tests: - - dbt_expectations.expect_column_values_to_be_between: - min_value: 0 - strictly: false - - name: payment_status data_type: character varying description: "The status of the payment. It can be one of: Paid, Refunded, Refund Failed, Cancelled, Paid Manually, Unpaid." @@ -164,6 +133,49 @@ models: data_type: character varying description: Free text description on the payment. Typically, contains explanations for integration issues with the payment processor. + versions: + - v: 1 + deprecation_date: 2024-10-15 00:00:00.00+00:00 + config: + alias: core__verification_payments + + - v: 2 + columns: + - name: total_amount_in_txn_currency + data_type: numeric + description: The payment amount in the currency in which the transaction actually happened. If the guest paid in Australian Dollars, this is measured in AUD. + tests: + - dbt_expectations.expect_column_values_to_be_between: + min_value: 0 + strictly: false + + - name: amount_without_taxes_in_txn_currency + data_type: numeric + description: The payment amount without taxes in the currency in which the transaction actually happened. If the guest paid in Australian Dollars, this is measured in AUD. (To be decommissioned) + tests: + - dbt_expectations.expect_column_values_to_be_between: + min_value: 0 + strictly: false + + - name: total_amount_in_gbp + data_type: numeric + description: The payment amount value, converted to GBP, using the exchange rate for the day on which the payment happened. + tests: + - dbt_expectations.expect_column_values_to_be_between: + min_value: 0 + strictly: false + + - name: amount_without_taxes_in_gbp + data_type: numeric + description: The payment amount value without taxes, converted to GBP, using the exchange rate for the day on which the payment happened. + tests: + - dbt_expectations.expect_column_values_to_be_between: + min_value: 0 + strictly: false + + - include: all + exclude: [amount_in_txn_currency, amount_in_gbp] + - name: core__bookings description: | From 985ddb8e4dec9a66878fab8872ba9d97c603e7fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oriol=20Roqu=C3=A9=20Paniagua?= Date: Wed, 18 Sep 2024 11:48:49 +0000 Subject: [PATCH 15/19] Merged PR 2887: Check-In Hero - Removing unused fields + schema changes # Description This PR closes the ticket #20046 Changes: * Removes temporary fields marked as legacy in `core__vr_check_in_cover`, after the changes in the 3 PBI Check-In Hero reports. * Additionally, I modified the schema entry to comply with the current fields available in this model. # Checklist - [X] The edited models and dependants run properly with production data. *Checked in local changing the sources to localhost after dropping the fields.* - [X] The edited models are sufficiently documented. - [X] The edited models contain PK tests, and I've ran and passed them. - [ ] I have checked for DRY opportunities with other models and docs. *N/A* - [ ] I've picked the right materialization for the affected models. *N/A* # Other - [ ] Check if a full-refresh is required after this PR is merged. Remove temporary fields Related work items: #20046 --- .../reporting/core/core__vr_check_in_cover.sql | 3 --- models/reporting/core/schema.yml | 18 ++++++++++++++---- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/models/reporting/core/core__vr_check_in_cover.sql b/models/reporting/core/core__vr_check_in_cover.sql index 3d01669..d137d9d 100644 --- a/models/reporting/core/core__vr_check_in_cover.sql +++ b/models/reporting/core/core__vr_check_in_cover.sql @@ -34,9 +34,6 @@ select amount_without_taxes_in_txn_currency as amount_without_taxes_in_txn_currency, total_amount_in_gbp as total_amount_in_gbp, amount_without_taxes_in_gbp as amount_without_taxes_in_gbp, - total_amount_in_txn_currency as amount_in_txn_currency, -- LEGACY - total_amount_in_gbp as amount_in_gbp, -- LEGACY - checkin_cover_limit_amount_local_curr as checkin_cover_limit_amount_local_curr, checkin_cover_limit_amount_in_gbp as checkin_cover_limit_amount_in_gbp from core__vr_check_in_cover diff --git a/models/reporting/core/schema.yml b/models/reporting/core/schema.yml index 1b48715..b67e6fa 100644 --- a/models/reporting/core/schema.yml +++ b/models/reporting/core/schema.yml @@ -323,19 +323,29 @@ models: description: | The date at which the guest finished the guest journey. - - name: amount_in_txn_currency + - name: total_amount_in_txn_currency data_type: numeric description: | The amount paid by the guest for the check-in cover, in the currency - in which the payment actually took place. + in which the payment actually took place. Tax inclusive. + + - name: amount_without_taxes_in_txn_currency + data_type: numeric + description: | + The equivalent amount paid by the guest for the check-in cover, in the currency + in which the payment actually took place, without taxes. - name: currency data_type: character varying description: The currency used by the guest for the payment. - - name: amount_in_gbp + - name: total_amount_in_gbp data_type: numeric - description: The amount paid by the guest, converted into GBP. + description: The amount paid by the guest, converted into GBP. Tax inclusive. + + - name: amount_without_taxes_in_gbp + data_type: numeric + description: The equivalent amount paid by the guest, converted into GBP and without taxes. - name: payment_status data_type: character varying From 5620ecbe682235a7ed855d755eae11426376a1fc Mon Sep 17 00:00:00 2001 From: uri Date: Wed, 18 Sep 2024 13:52:55 +0200 Subject: [PATCH 16/19] Remove .sql in the xero__bank_transaction_denom_mart --- models/reporting/xero/schema.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/reporting/xero/schema.yml b/models/reporting/xero/schema.yml index 417efc1..7df3c2e 100644 --- a/models/reporting/xero/schema.yml +++ b/models/reporting/xero/schema.yml @@ -1105,7 +1105,7 @@ models: The transaction currency is defined at the Bank Transaction level, and the values you see in this field should always be the same as the currency of the Bank Transaction this line item belongs to. - - name: xero__bank_transaction_denom_mart.sql + - name: xero__bank_transaction_denom_mart description: | This model is a denormalized mart, which only exists for presentation purposes in a PBI report. From 923bfa70919bf552304150e8cc3ec9af7cdbe708 Mon Sep 17 00:00:00 2001 From: Joaquin Ossa Date: Wed, 18 Sep 2024 14:20:48 +0200 Subject: [PATCH 17/19] Update exposures to pull from version 1 --- models/reporting/core/schema.yml | 2 +- models/reporting/exposures.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/models/reporting/core/schema.yml b/models/reporting/core/schema.yml index 2b966d7..3e05e26 100644 --- a/models/reporting/core/schema.yml +++ b/models/reporting/core/schema.yml @@ -192,7 +192,7 @@ models: description: The unique, Superhog generated id for this booking. tests: - unique - - not_null + - not_nullgit p - name: id_user_guest data_type: character varying diff --git a/models/reporting/exposures.yml b/models/reporting/exposures.yml index 79118ff..ce23470 100644 --- a/models/reporting/exposures.yml +++ b/models/reporting/exposures.yml @@ -12,7 +12,7 @@ exposures: depends_on: - ref('dates') - - ref('core__verification_payments') + - ref('core__verification_payments', version=1) owner: name: Pablo Martin From 7ceeabbe7b27449cf2c2cd22dbec662a66c343d9 Mon Sep 17 00:00:00 2001 From: Joaquin Ossa Date: Wed, 18 Sep 2024 14:22:33 +0200 Subject: [PATCH 18/19] Revert "commit" This reverts commit e854f2a99aadd34032466259c483b8ee7740ddca. --- package-lock.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.yml b/package-lock.yml index d7ed31d..2b520db 100644 --- a/package-lock.yml +++ b/package-lock.yml @@ -4,7 +4,7 @@ packages: - package: calogica/dbt_expectations version: 0.9.0 - package: dbt-labs/dbt_utils - version: 1.2.0 + version: 1.1.1 - package: calogica/dbt_date version: 0.8.1 -sha1_hash: ceec21d8037429db57330b6f23cfdc761bbb7698 +sha1_hash: 0d8e1bf7188f14d46813097c65818e3f96237266 From feaedb2a06bd37555217ee0fb645c8f5a07b070d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oriol=20Roqu=C3=A9=20Paniagua?= Date: Thu, 19 Sep 2024 07:00:15 +0000 Subject: [PATCH 19/19] Merged PR 2911: Bugfix core__bookings failing dbt test # Description Fixes issue # Checklist - [ ] The edited models and dependants run properly with production data. - [ ] The edited models are sufficiently documented. - [ ] The edited models contain PK tests, and I've ran and passed them. - [ ] I have checked for DRY opportunities with other models and docs. - [ ] I've picked the right materialization for the affected models. # Other - [ ] Check if a full-refresh is required after this PR is merged. --- models/reporting/core/schema.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/reporting/core/schema.yml b/models/reporting/core/schema.yml index a0c04af..2665c9e 100644 --- a/models/reporting/core/schema.yml +++ b/models/reporting/core/schema.yml @@ -192,7 +192,7 @@ models: description: The unique, Superhog generated id for this booking. tests: - unique - - not_nullgit p + - not_null - name: id_user_guest data_type: character varying