From c95f551acfe75157339d62d98addd7047be1b3e9 Mon Sep 17 00:00:00 2001 From: Joaquin Ossa Date: Tue, 17 Dec 2024 19:09:54 +0100 Subject: [PATCH 1/3] s&p update models --- macros/tests/at_least_one_null.sql | 10 ++ macros/tests/is_first_day_of_month.sql | 3 + macros/tests/is_last_day_of_month.sql | 9 ++ ...een_and_protect__verification_requests.sql | 23 +++- .../screen_and_protect/schema.yml | 100 ++++++++++++++++-- .../reporting/screen_and_protect/schema.yml | 20 ++-- ...een_and_protect__verification_requests.sql | 6 +- models/staging/screen_and_protect/schema.yml | 20 ++-- ...een_and_protect__verification_requests.sql | 6 +- 9 files changed, 158 insertions(+), 39 deletions(-) create mode 100644 macros/tests/at_least_one_null.sql create mode 100644 macros/tests/is_first_day_of_month.sql create mode 100644 macros/tests/is_last_day_of_month.sql diff --git a/macros/tests/at_least_one_null.sql b/macros/tests/at_least_one_null.sql new file mode 100644 index 0000000..9723af1 --- /dev/null +++ b/macros/tests/at_least_one_null.sql @@ -0,0 +1,10 @@ +{% test at_least_one_null(model, columns) %} + + select * + from {{ model }} + where + {% for column in columns %} + {{ column }} is not null {% if not loop.last %} and {% endif %} + {% endfor %} +-- If all columns are not null, the row violates the test +{% endtest %} diff --git a/macros/tests/is_first_day_of_month.sql b/macros/tests/is_first_day_of_month.sql new file mode 100644 index 0000000..4895063 --- /dev/null +++ b/macros/tests/is_first_day_of_month.sql @@ -0,0 +1,3 @@ +{% test is_first_day_of_month(model, column_name) %} + select * from {{ model }} where extract(day from {{ column_name }}) != 1 +{% endtest %} diff --git a/macros/tests/is_last_day_of_month.sql b/macros/tests/is_last_day_of_month.sql new file mode 100644 index 0000000..b87fd16 --- /dev/null +++ b/macros/tests/is_last_day_of_month.sql @@ -0,0 +1,9 @@ +{% test is_last_day_of_month(model, column_name) %} + select * + from {{ model }} + where + {{ column_name }} + != date_trunc('MONTH', {{ column_name }}) + + interval '1 MONTH' + - interval '1 DAY' +{% endtest %} diff --git a/models/intermediate/screen_and_protect/int_screen_and_protect__verification_requests.sql b/models/intermediate/screen_and_protect/int_screen_and_protect__verification_requests.sql index 05374e7..45c8991 100644 --- a/models/intermediate/screen_and_protect/int_screen_and_protect__verification_requests.sql +++ b/models/intermediate/screen_and_protect/int_screen_and_protect__verification_requests.sql @@ -25,6 +25,23 @@ select vr.pet_protection, vr.verification_status, vr.verification_status_reason, + (au.json_document_user_data ->> 'PriceIncrease')::decimal as price_increase, + (au.json_document_user_data ->> 'PriceIncreaseStartDate')::date + as price_increase_start_date_utc, + (au.json_document_user_data ->> 'MonthlyVolumeDiscount')::decimal + as monthly_volume_discount, + (au.json_document_user_data ->> 'ThresholdApprovedBookingVolume')::integer + as threshold_approved_booking_volume, + (au.json_document_user_data ->> 'MonthlyVolumeDiscountStartDate')::date + as monthly_volume_discount_start_date_utc, + (au.json_document_user_data ->> 'MonthlyVolumeDiscountEndDate')::date + as monthly_volume_discount_end_date_utc, + (au.json_document_user_data ->> 'MonthlyGeneralDiscount')::decimal + as monthly_general_discount, + (au.json_document_user_data ->> 'MonthlyGeneralDiscountStartDate')::date + as monthly_general_discount_start_date_utc, + (au.json_document_user_data ->> 'MonthlyGeneralDiscountEndDate')::date + as monthly_general_discount_end_date_utc, vr.email_flag, vr.phone_flag, vr.watch_list, @@ -53,9 +70,9 @@ select vr.status_updated_date_utc, vr.updated_at_utc, vr.updated_date_utc, - vr.cosmos_creation_at_utc, - vr.cosmos_creation_date_utc, - vr.created_date_utc + vr.creation_at_utc, + vr.creation_date_utc, + vr.cosmos_created_date_utc from stg_screen_and_protect__verification_requests vr inner join stg_core__apim_user au on vr.id_user_partner = au.id_apim_user inner join diff --git a/models/intermediate/screen_and_protect/schema.yml b/models/intermediate/screen_and_protect/schema.yml index 6806202..9ca3e1f 100644 --- a/models/intermediate/screen_and_protect/schema.yml +++ b/models/intermediate/screen_and_protect/schema.yml @@ -6,6 +6,11 @@ models: Records of verification requests from the Screen and Protect API. The table tracks verification requests, their outcomes, and related metadata about guests, listings, and partners. + tests: + - at_least_one_null: + columns: + - monthly_volume_discount + - monthly_general_discount columns: - name: id_verification data_type: text @@ -95,6 +100,81 @@ models: - "FLAGGED" - "REJECTED" + - name: price_increase + data_type: numeric + description: The percentage or value of the price increase + applied to the user's account. + tests: + - dbt_expectations.expect_column_values_to_be_between: + min_value: 0 + strictly: true + + - name: price_increase_start_date_utc + data_type: date + description: The date when the price increase becomes effective. + This is the first day of the month. + tests: + - is_first_day_of_month + + - name: monthly_volume_discount + data_type: numeric + description: The discount percentage or value offered based on the + volume of bookings achieved within a month. + No user can have more than one discount per month. + tests: + - dbt_expectations.expect_column_values_to_be_between: + min_value: 0 + max_value: 1 + strictly: true + + - name: threshold_approved_booking_volume + data_type: numeric + description: The minimum number of bookings required to qualify for + the monthly volume discount. + tests: + - dbt_expectations.expect_column_values_to_be_between: + min_value: 0 + strictly: true + + - name: monthly_volume_discount_start_date_utc + data_type: date + description: The date when the monthly volume discount period begins. + This is the first day of the month. + tests: + - is_first_day_of_month + + - name: monthly_volume_discount_end_date_utc + data_type: date + description: The date when the monthly volume discount period ends. + This is the last day of the month. + tests: + - is_last_day_of_month + + - name: monthly_general_discount + data_type: numeric + description: The general discount percentage or value applied to all + bookings within the applicable period. + No user can have more than one discount per month. + tests: + - dbt_expectations.expect_column_values_to_be_between: + min_value: 0 + max_value: 1 + strictly: true + + - name: monthly_general_discount_start_date_utc + data_type: date + description: The date when the general discount period begins. + This is the first day of the month. + tests: + - is_first_day_of_month + + - name: monthly_general_discount_end_date_utc + data_type: date + description: The date when the general discount period ends. + This is the last day of the month. + tests: + - is_last_day_of_month + - name: verification_status_reason data_type: text description: Reason for the verification status. @@ -225,23 +305,23 @@ models: tests: - not_null - - name: cosmos_creation_at_utc + - name: creation_at_utc data_type: timestamp without time zone description: | - Timestamp of when the verification request was created in Cosmos DB. + Timestamp of when the reservation was created. tests: - not_null - - name: cosmos_creation_date_utc + - name: creation_date_utc + data_type: date + description: | + Date of when the reservation was created. + tests: + - not_null + + - name: cosmos_created_date_utc data_type: date description: | Date of when the verification request was created in Cosmos DB. tests: - not_null - - - name: created_date_utc - data_type: date - description: | - Date when the reservation was created. - tests: - - not_null diff --git a/models/reporting/screen_and_protect/schema.yml b/models/reporting/screen_and_protect/schema.yml index 2d52292..7817753 100644 --- a/models/reporting/screen_and_protect/schema.yml +++ b/models/reporting/screen_and_protect/schema.yml @@ -225,23 +225,23 @@ models: tests: - not_null - - name: cosmos_creation_at_utc + - name: creation_at_utc data_type: timestamp without time zone description: | - Timestamp of when the verification request was created in Cosmos DB. + Timestamp of when the reservation was created. tests: - not_null - - name: cosmos_creation_date_utc + - name: creation_date_utc + data_type: date + description: | + Date of when the reservation was created. + tests: + - not_null + + - name: cosmos_created_date_utc data_type: date description: | Date of when the verification request was created in Cosmos DB. tests: - not_null - - - name: created_date_utc - data_type: date - description: | - Date when the reservation was created. - tests: - - not_null diff --git a/models/reporting/screen_and_protect/screen_and_protect__verification_requests.sql b/models/reporting/screen_and_protect/screen_and_protect__verification_requests.sql index 8161c43..ca0165e 100644 --- a/models/reporting/screen_and_protect/screen_and_protect__verification_requests.sql +++ b/models/reporting/screen_and_protect/screen_and_protect__verification_requests.sql @@ -45,7 +45,7 @@ select status_updated_date_utc as status_updated_date_utc, updated_at_utc as updated_at_utc, updated_date_utc as updated_date_utc, - cosmos_creation_at_utc as cosmos_creation_at_utc, - cosmos_creation_date_utc as cosmos_creation_date_utc, - created_date_utc as created_date_utc + creation_at_utc as cosmos_creation_at_utc, + creation_date_utc as cosmos_creation_date_utc, + cosmos_created_date_utc as created_date_utc from int_screen_and_protect__verification_requests diff --git a/models/staging/screen_and_protect/schema.yml b/models/staging/screen_and_protect/schema.yml index ca2527e..c7cf271 100644 --- a/models/staging/screen_and_protect/schema.yml +++ b/models/staging/screen_and_protect/schema.yml @@ -214,27 +214,27 @@ models: tests: - not_null - - name: cosmos_creation_at_utc + - name: creation_at_utc data_type: timestamp without time zone description: | - Timestamp of when the verification request was created in Cosmos DB. + Timestamp of when the reservation was created. tests: - not_null - - name: cosmos_creation_date_utc + - name: creation_date_utc + data_type: date + description: | + Date of when the reservation was created. + tests: + - not_null + + - name: cosmos_created_date_utc data_type: date description: | Date of when the verification request was created in Cosmos DB. tests: - not_null - - name: created_date_utc - data_type: date - description: | - Date when the reservation was created. - tests: - - not_null - - name: cosmos_db_timestamp_utc data_type: timestamp with time zone description: Internal Cosmos DB timestamp of the last record update. diff --git a/models/staging/screen_and_protect/stg_screen_and_protect__verification_requests.sql b/models/staging/screen_and_protect/stg_screen_and_protect__verification_requests.sql index 8fb9a3e..cbaf7a2 100644 --- a/models/staging/screen_and_protect/stg_screen_and_protect__verification_requests.sql +++ b/models/staging/screen_and_protect/stg_screen_and_protect__verification_requests.sql @@ -86,11 +86,11 @@ with ({{ adapter.quote("documents") }} ->> 'UpdatedDate')::date as updated_date_utc, ({{ adapter.quote("documents") }} ->> 'CreationDate')::timestamp - as cosmos_creation_at_utc, + as creation_at_utc, ({{ adapter.quote("documents") }} ->> 'CreationDate')::date - as cosmos_creation_date_utc, + as creation_date_utc, ({{ adapter.quote("documents") }} ->> 'CreatedDate')::date - as created_date_utc, + as cosmos_created_date_utc, to_timestamp( (({{ adapter.quote("documents") }} ->> '_ts'))::integer ) as cosmos_db_timestamp_utc From 24aa9b70211092e3bcfa8e902265fea46a63744f Mon Sep 17 00:00:00 2001 From: Joaquin Ossa Date: Wed, 18 Dec 2024 10:52:52 +0100 Subject: [PATCH 2/3] Helped with readability on test --- macros/tests/is_last_day_of_month.sql | 9 +++++---- .../int_screen_and_protect__verification_requests.sql | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/macros/tests/is_last_day_of_month.sql b/macros/tests/is_last_day_of_month.sql index b87fd16..37fa732 100644 --- a/macros/tests/is_last_day_of_month.sql +++ b/macros/tests/is_last_day_of_month.sql @@ -2,8 +2,9 @@ select * from {{ model }} where - {{ column_name }} - != date_trunc('MONTH', {{ column_name }}) - + interval '1 MONTH' - - interval '1 DAY' + {{ column_name }} != ( + date_trunc('MONTH', {{ column_name }}) + + interval '1 MONTH' + - interval '1 DAY' + ) {% endtest %} diff --git a/models/intermediate/screen_and_protect/int_screen_and_protect__verification_requests.sql b/models/intermediate/screen_and_protect/int_screen_and_protect__verification_requests.sql index 45c8991..6a63c00 100644 --- a/models/intermediate/screen_and_protect/int_screen_and_protect__verification_requests.sql +++ b/models/intermediate/screen_and_protect/int_screen_and_protect__verification_requests.sql @@ -7,7 +7,7 @@ with stg_core__apim_user as (select * from {{ ref("stg_core__apim_user") }}), stg_core__apim_user_type as (select * from {{ ref("stg_core__apim_user_type") }}) select - -- note that these ids are not the same as the ones found in Core DWH + -- note that these ids are not the same as the ones found in Core -- they are completely unrelated vr.id_verification, vr.id_booking, From 0cebdcf160ac6a5149eca2f1fd10e5dd3435408a Mon Sep 17 00:00:00 2001 From: Joaquin Ossa Date: Wed, 18 Dec 2024 11:46:54 +0100 Subject: [PATCH 3/3] new approach for S&P users data extraction --- .../int_core__screen_and_protect_users.sql | 46 ++++++ models/intermediate/core/schema.yml | 147 ++++++++++++++++++ ...een_and_protect__verification_requests.sql | 44 ++---- .../screen_and_protect/schema.yml | 4 +- 4 files changed, 209 insertions(+), 32 deletions(-) create mode 100644 models/intermediate/core/int_core__screen_and_protect_users.sql diff --git a/models/intermediate/core/int_core__screen_and_protect_users.sql b/models/intermediate/core/int_core__screen_and_protect_users.sql new file mode 100644 index 0000000..e3e4a2f --- /dev/null +++ b/models/intermediate/core/int_core__screen_and_protect_users.sql @@ -0,0 +1,46 @@ +{% set api_name = "SCREENANDPROTECTAPI" %} + +with + stg_core__apim_user as (select * from {{ ref("stg_core__apim_user") }}), + stg_core__apim_user_type as (select * from {{ ref("stg_core__apim_user_type") }}) +select + -- note that these ids are not the same as the ones found in Core + -- they are completely unrelated + au.id_apim_user, + au.id_apim_user_type, + au.json_document_user_data ->> 'DealId' as id_deal, + au.json_document_user_data ->> 'AccountType' as account_type, + au.json_document_user_data ->> 'ClientMarkup' as client_markup, + au.json_document_user_data ->> 'TechEmailAddress' as tech_email_address, + au.json_document_user_data ->> 'InvoicingEmailAddress' as invoicing_email_address, + case + when (au.json_document_user_data ->> 'FlaggedNotProtected') = 'false' + then true + else false + end as is_protected, + (au.json_document_user_data ->> 'PriceIncrease')::decimal as price_increase, + (au.json_document_user_data ->> 'PriceIncreaseStartDate')::date + as price_increase_start_date_utc, + (au.json_document_user_data ->> 'MonthlyVolumeDiscount')::decimal + as monthly_volume_discount, + (au.json_document_user_data ->> 'ThresholdApprovedBookingVolume')::integer + as threshold_approved_booking_volume, + (au.json_document_user_data ->> 'MonthlyVolumeDiscountStartDate')::date + as monthly_volume_discount_start_date_utc, + (au.json_document_user_data ->> 'MonthlyVolumeDiscountEndDate')::date + as monthly_volume_discount_end_date_utc, + (au.json_document_user_data ->> 'MonthlyGeneralDiscount')::decimal + as monthly_general_discount, + (au.json_document_user_data ->> 'MonthlyGeneralDiscountStartDate')::date + as monthly_general_discount_start_date_utc, + (au.json_document_user_data ->> 'MonthlyGeneralDiscountEndDate')::date + as monthly_general_discount_end_date_utc, + au.created_at_utc, + au.created_date_utc, + au.updated_at_utc, + au.updated_date_utc +from stg_core__apim_user au +inner join + stg_core__apim_user_type aut + on au.id_apim_user_type = aut.id_apim_user_type + and upper(aut.user_type_name) = '{{ api_name }}' diff --git a/models/intermediate/core/schema.yml b/models/intermediate/core/schema.yml index cbe7b13..69fd4ae 100644 --- a/models/intermediate/core/schema.yml +++ b/models/intermediate/core/schema.yml @@ -4427,3 +4427,150 @@ models: description: | Check-in Cover fees revenue paid without taxes in GBP. Can be null. + + - name: int_core__screen_and_protect_users + description: | + "Contains information about Screen & Protect API users. + It includes all user details related with the service." + columns: + - name: id_apim_user + data_type: character varying + description: "Identifier of the User. Acts as primary + key for this table." + tests: + - not_null + - unique + + - name: id_apim_user_type + data_type: bigint + description: "Identifier of the type of user." + + - name: id_deal + data_type: text + description: "" + + - name: account_type + data_type: text + description: "" + + - name: client_markup + data_type: text + description: "" + + - name: tech_email_address + data_type: text + description: "" + + - name: invoicing_email_address + data_type: text + description: "" + + - name: is_protected + data_type: boolean + description: | + Indicates if the user's bookings are protected or not. + tests: + - not_null + + - name: price_increase + data_type: numeric + description: The percentage or value of the price increase + applied to the user's account. + tests: + - dbt_expectations.expect_column_values_to_be_between: + min_value: 0 + strictly: true + + - name: price_increase_start_date_utc + data_type: date + description: The date when the price increase becomes effective. + This is the first day of the month. + tests: + - is_first_day_of_month + + - name: monthly_volume_discount + data_type: numeric + description: The discount percentage or value offered based on the + volume of bookings achieved within a month. + No user can have more than one discount per month. + tests: + - dbt_expectations.expect_column_values_to_be_between: + min_value: 0 + max_value: 100 + strictly: true + + - name: threshold_approved_booking_volume + data_type: numeric + description: The minimum number of bookings required to qualify for + the monthly volume discount. + tests: + - dbt_expectations.expect_column_values_to_be_between: + min_value: 0 + strictly: true + + - name: monthly_volume_discount_start_date_utc + data_type: date + description: The date when the monthly volume discount period begins. + This is the first day of the month. + tests: + - is_first_day_of_month + + - name: monthly_volume_discount_end_date_utc + data_type: date + description: The date when the monthly volume discount period ends. + This is the last day of the month. + tests: + - is_last_day_of_month + + - name: monthly_general_discount + data_type: numeric + description: The general discount percentage or value applied to all + bookings within the applicable period. + No user can have more than one discount per month. + tests: + - dbt_expectations.expect_column_values_to_be_between: + min_value: 0 + max_value: 100 + strictly: true + + - name: monthly_general_discount_start_date_utc + data_type: date + description: The date when the general discount period begins. + This is the first day of the month. + tests: + - is_first_day_of_month + + - name: monthly_general_discount_end_date_utc + data_type: date + description: The date when the general discount period ends. + This is the last day of the month. + tests: + - is_last_day_of_month + + - name: created_at_utc + data_type: timestamp + description: | + Timestamp of when this user was created. + tests: + - not_null + + - name: created_date_utc + data_type: date + description: | + Date of when this user was created. + tests: + - not_null + + - name: updated_at_utc + data_type: timestamp + description: | + Timestamp of when this user was last updated. + tests: + - not_null + + - name: updated_date_utc + data_type: date + description: | + Date of when this user was last updated. + tests: + - not_null diff --git a/models/intermediate/screen_and_protect/int_screen_and_protect__verification_requests.sql b/models/intermediate/screen_and_protect/int_screen_and_protect__verification_requests.sql index 6a63c00..3fbc582 100644 --- a/models/intermediate/screen_and_protect/int_screen_and_protect__verification_requests.sql +++ b/models/intermediate/screen_and_protect/int_screen_and_protect__verification_requests.sql @@ -1,11 +1,10 @@ -{% set api_name = "SCREENANDPROTECTAPI" %} - with stg_screen_and_protect__verification_requests as ( select * from {{ ref("stg_screen_and_protect__verification_requests") }} ), - stg_core__apim_user as (select * from {{ ref("stg_core__apim_user") }}), - stg_core__apim_user_type as (select * from {{ ref("stg_core__apim_user_type") }}) + int_core__screen_and_protect_users as ( + select * from {{ ref("int_core__screen_and_protect_users") }} + ) select -- note that these ids are not the same as the ones found in Core -- they are completely unrelated @@ -13,11 +12,7 @@ select vr.id_booking, vr.id_user_partner, vr.id_accommodation, - case - when (au.json_document_user_data ->> 'FlaggedNotProtected') = 'false' - then true - else false - end as is_protected, + spu.is_protected, vr.protection_type, vr.protection_starting_level, vr.protection_basic_amount, @@ -25,23 +20,15 @@ select vr.pet_protection, vr.verification_status, vr.verification_status_reason, - (au.json_document_user_data ->> 'PriceIncrease')::decimal as price_increase, - (au.json_document_user_data ->> 'PriceIncreaseStartDate')::date - as price_increase_start_date_utc, - (au.json_document_user_data ->> 'MonthlyVolumeDiscount')::decimal - as monthly_volume_discount, - (au.json_document_user_data ->> 'ThresholdApprovedBookingVolume')::integer - as threshold_approved_booking_volume, - (au.json_document_user_data ->> 'MonthlyVolumeDiscountStartDate')::date - as monthly_volume_discount_start_date_utc, - (au.json_document_user_data ->> 'MonthlyVolumeDiscountEndDate')::date - as monthly_volume_discount_end_date_utc, - (au.json_document_user_data ->> 'MonthlyGeneralDiscount')::decimal - as monthly_general_discount, - (au.json_document_user_data ->> 'MonthlyGeneralDiscountStartDate')::date - as monthly_general_discount_start_date_utc, - (au.json_document_user_data ->> 'MonthlyGeneralDiscountEndDate')::date - as monthly_general_discount_end_date_utc, + spu.price_increase, + spu.price_increase_start_date_utc, + spu.monthly_volume_discount, + spu.threshold_approved_booking_volume, + spu.monthly_volume_discount_start_date_utc, + spu.monthly_volume_discount_end_date_utc, + spu.monthly_general_discount, + spu.monthly_general_discount_start_date_utc, + spu.monthly_general_discount_end_date_utc, vr.email_flag, vr.phone_flag, vr.watch_list, @@ -74,8 +61,5 @@ select vr.creation_date_utc, vr.cosmos_created_date_utc from stg_screen_and_protect__verification_requests vr -inner join stg_core__apim_user au on vr.id_user_partner = au.id_apim_user inner join - stg_core__apim_user_type aut - on au.id_apim_user_type = aut.id_apim_user_type - and upper(aut.user_type_name) = '{{ api_name }}' + int_core__screen_and_protect_users spu on vr.id_user_partner = spu.id_apim_user diff --git a/models/intermediate/screen_and_protect/schema.yml b/models/intermediate/screen_and_protect/schema.yml index 9ca3e1f..acf0425 100644 --- a/models/intermediate/screen_and_protect/schema.yml +++ b/models/intermediate/screen_and_protect/schema.yml @@ -124,7 +124,7 @@ models: tests: - dbt_expectations.expect_column_values_to_be_between: min_value: 0 - max_value: 1 + max_value: 100 strictly: true - name: threshold_approved_booking_volume @@ -158,7 +158,7 @@ models: tests: - dbt_expectations.expect_column_values_to_be_between: min_value: 0 - max_value: 1 + max_value: 100 strictly: true - name: monthly_general_discount_start_date_utc