From c95f551acfe75157339d62d98addd7047be1b3e9 Mon Sep 17 00:00:00 2001 From: Joaquin Ossa Date: Tue, 17 Dec 2024 19:09:54 +0100 Subject: [PATCH] s&p update models --- macros/tests/at_least_one_null.sql | 10 ++ macros/tests/is_first_day_of_month.sql | 3 + macros/tests/is_last_day_of_month.sql | 9 ++ ...een_and_protect__verification_requests.sql | 23 +++- .../screen_and_protect/schema.yml | 100 ++++++++++++++++-- .../reporting/screen_and_protect/schema.yml | 20 ++-- ...een_and_protect__verification_requests.sql | 6 +- models/staging/screen_and_protect/schema.yml | 20 ++-- ...een_and_protect__verification_requests.sql | 6 +- 9 files changed, 158 insertions(+), 39 deletions(-) create mode 100644 macros/tests/at_least_one_null.sql create mode 100644 macros/tests/is_first_day_of_month.sql create mode 100644 macros/tests/is_last_day_of_month.sql diff --git a/macros/tests/at_least_one_null.sql b/macros/tests/at_least_one_null.sql new file mode 100644 index 0000000..9723af1 --- /dev/null +++ b/macros/tests/at_least_one_null.sql @@ -0,0 +1,10 @@ +{% test at_least_one_null(model, columns) %} + + select * + from {{ model }} + where + {% for column in columns %} + {{ column }} is not null {% if not loop.last %} and {% endif %} + {% endfor %} +-- If all columns are not null, the row violates the test +{% endtest %} diff --git a/macros/tests/is_first_day_of_month.sql b/macros/tests/is_first_day_of_month.sql new file mode 100644 index 0000000..4895063 --- /dev/null +++ b/macros/tests/is_first_day_of_month.sql @@ -0,0 +1,3 @@ +{% test is_first_day_of_month(model, column_name) %} + select * from {{ model }} where extract(day from {{ column_name }}) != 1 +{% endtest %} diff --git a/macros/tests/is_last_day_of_month.sql b/macros/tests/is_last_day_of_month.sql new file mode 100644 index 0000000..b87fd16 --- /dev/null +++ b/macros/tests/is_last_day_of_month.sql @@ -0,0 +1,9 @@ +{% test is_last_day_of_month(model, column_name) %} + select * + from {{ model }} + where + {{ column_name }} + != date_trunc('MONTH', {{ column_name }}) + + interval '1 MONTH' + - interval '1 DAY' +{% endtest %} diff --git a/models/intermediate/screen_and_protect/int_screen_and_protect__verification_requests.sql b/models/intermediate/screen_and_protect/int_screen_and_protect__verification_requests.sql index 05374e7..45c8991 100644 --- a/models/intermediate/screen_and_protect/int_screen_and_protect__verification_requests.sql +++ b/models/intermediate/screen_and_protect/int_screen_and_protect__verification_requests.sql @@ -25,6 +25,23 @@ select vr.pet_protection, vr.verification_status, vr.verification_status_reason, + (au.json_document_user_data ->> 'PriceIncrease')::decimal as price_increase, + (au.json_document_user_data ->> 'PriceIncreaseStartDate')::date + as price_increase_start_date_utc, + (au.json_document_user_data ->> 'MonthlyVolumeDiscount')::decimal + as monthly_volume_discount, + (au.json_document_user_data ->> 'ThresholdApprovedBookingVolume')::integer + as threshold_approved_booking_volume, + (au.json_document_user_data ->> 'MonthlyVolumeDiscountStartDate')::date + as monthly_volume_discount_start_date_utc, + (au.json_document_user_data ->> 'MonthlyVolumeDiscountEndDate')::date + as monthly_volume_discount_end_date_utc, + (au.json_document_user_data ->> 'MonthlyGeneralDiscount')::decimal + as monthly_general_discount, + (au.json_document_user_data ->> 'MonthlyGeneralDiscountStartDate')::date + as monthly_general_discount_start_date_utc, + (au.json_document_user_data ->> 'MonthlyGeneralDiscountEndDate')::date + as monthly_general_discount_end_date_utc, vr.email_flag, vr.phone_flag, vr.watch_list, @@ -53,9 +70,9 @@ select vr.status_updated_date_utc, vr.updated_at_utc, vr.updated_date_utc, - vr.cosmos_creation_at_utc, - vr.cosmos_creation_date_utc, - vr.created_date_utc + vr.creation_at_utc, + vr.creation_date_utc, + vr.cosmos_created_date_utc from stg_screen_and_protect__verification_requests vr inner join stg_core__apim_user au on vr.id_user_partner = au.id_apim_user inner join diff --git a/models/intermediate/screen_and_protect/schema.yml b/models/intermediate/screen_and_protect/schema.yml index 6806202..9ca3e1f 100644 --- a/models/intermediate/screen_and_protect/schema.yml +++ b/models/intermediate/screen_and_protect/schema.yml @@ -6,6 +6,11 @@ models: Records of verification requests from the Screen and Protect API. The table tracks verification requests, their outcomes, and related metadata about guests, listings, and partners. + tests: + - at_least_one_null: + columns: + - monthly_volume_discount + - monthly_general_discount columns: - name: id_verification data_type: text @@ -95,6 +100,81 @@ models: - "FLAGGED" - "REJECTED" + - name: price_increase + data_type: numeric + description: The percentage or value of the price increase + applied to the user's account. + tests: + - dbt_expectations.expect_column_values_to_be_between: + min_value: 0 + strictly: true + + - name: price_increase_start_date_utc + data_type: date + description: The date when the price increase becomes effective. + This is the first day of the month. + tests: + - is_first_day_of_month + + - name: monthly_volume_discount + data_type: numeric + description: The discount percentage or value offered based on the + volume of bookings achieved within a month. + No user can have more than one discount per month. + tests: + - dbt_expectations.expect_column_values_to_be_between: + min_value: 0 + max_value: 1 + strictly: true + + - name: threshold_approved_booking_volume + data_type: numeric + description: The minimum number of bookings required to qualify for + the monthly volume discount. + tests: + - dbt_expectations.expect_column_values_to_be_between: + min_value: 0 + strictly: true + + - name: monthly_volume_discount_start_date_utc + data_type: date + description: The date when the monthly volume discount period begins. + This is the first day of the month. + tests: + - is_first_day_of_month + + - name: monthly_volume_discount_end_date_utc + data_type: date + description: The date when the monthly volume discount period ends. + This is the last day of the month. + tests: + - is_last_day_of_month + + - name: monthly_general_discount + data_type: numeric + description: The general discount percentage or value applied to all + bookings within the applicable period. + No user can have more than one discount per month. + tests: + - dbt_expectations.expect_column_values_to_be_between: + min_value: 0 + max_value: 1 + strictly: true + + - name: monthly_general_discount_start_date_utc + data_type: date + description: The date when the general discount period begins. + This is the first day of the month. + tests: + - is_first_day_of_month + + - name: monthly_general_discount_end_date_utc + data_type: date + description: The date when the general discount period ends. + This is the last day of the month. + tests: + - is_last_day_of_month + - name: verification_status_reason data_type: text description: Reason for the verification status. @@ -225,23 +305,23 @@ models: tests: - not_null - - name: cosmos_creation_at_utc + - name: creation_at_utc data_type: timestamp without time zone description: | - Timestamp of when the verification request was created in Cosmos DB. + Timestamp of when the reservation was created. tests: - not_null - - name: cosmos_creation_date_utc + - name: creation_date_utc + data_type: date + description: | + Date of when the reservation was created. + tests: + - not_null + + - name: cosmos_created_date_utc data_type: date description: | Date of when the verification request was created in Cosmos DB. tests: - not_null - - - name: created_date_utc - data_type: date - description: | - Date when the reservation was created. - tests: - - not_null diff --git a/models/reporting/screen_and_protect/schema.yml b/models/reporting/screen_and_protect/schema.yml index 2d52292..7817753 100644 --- a/models/reporting/screen_and_protect/schema.yml +++ b/models/reporting/screen_and_protect/schema.yml @@ -225,23 +225,23 @@ models: tests: - not_null - - name: cosmos_creation_at_utc + - name: creation_at_utc data_type: timestamp without time zone description: | - Timestamp of when the verification request was created in Cosmos DB. + Timestamp of when the reservation was created. tests: - not_null - - name: cosmos_creation_date_utc + - name: creation_date_utc + data_type: date + description: | + Date of when the reservation was created. + tests: + - not_null + + - name: cosmos_created_date_utc data_type: date description: | Date of when the verification request was created in Cosmos DB. tests: - not_null - - - name: created_date_utc - data_type: date - description: | - Date when the reservation was created. - tests: - - not_null diff --git a/models/reporting/screen_and_protect/screen_and_protect__verification_requests.sql b/models/reporting/screen_and_protect/screen_and_protect__verification_requests.sql index 8161c43..ca0165e 100644 --- a/models/reporting/screen_and_protect/screen_and_protect__verification_requests.sql +++ b/models/reporting/screen_and_protect/screen_and_protect__verification_requests.sql @@ -45,7 +45,7 @@ select status_updated_date_utc as status_updated_date_utc, updated_at_utc as updated_at_utc, updated_date_utc as updated_date_utc, - cosmos_creation_at_utc as cosmos_creation_at_utc, - cosmos_creation_date_utc as cosmos_creation_date_utc, - created_date_utc as created_date_utc + creation_at_utc as cosmos_creation_at_utc, + creation_date_utc as cosmos_creation_date_utc, + cosmos_created_date_utc as created_date_utc from int_screen_and_protect__verification_requests diff --git a/models/staging/screen_and_protect/schema.yml b/models/staging/screen_and_protect/schema.yml index ca2527e..c7cf271 100644 --- a/models/staging/screen_and_protect/schema.yml +++ b/models/staging/screen_and_protect/schema.yml @@ -214,27 +214,27 @@ models: tests: - not_null - - name: cosmos_creation_at_utc + - name: creation_at_utc data_type: timestamp without time zone description: | - Timestamp of when the verification request was created in Cosmos DB. + Timestamp of when the reservation was created. tests: - not_null - - name: cosmos_creation_date_utc + - name: creation_date_utc + data_type: date + description: | + Date of when the reservation was created. + tests: + - not_null + + - name: cosmos_created_date_utc data_type: date description: | Date of when the verification request was created in Cosmos DB. tests: - not_null - - name: created_date_utc - data_type: date - description: | - Date when the reservation was created. - tests: - - not_null - - name: cosmos_db_timestamp_utc data_type: timestamp with time zone description: Internal Cosmos DB timestamp of the last record update. diff --git a/models/staging/screen_and_protect/stg_screen_and_protect__verification_requests.sql b/models/staging/screen_and_protect/stg_screen_and_protect__verification_requests.sql index 8fb9a3e..cbaf7a2 100644 --- a/models/staging/screen_and_protect/stg_screen_and_protect__verification_requests.sql +++ b/models/staging/screen_and_protect/stg_screen_and_protect__verification_requests.sql @@ -86,11 +86,11 @@ with ({{ adapter.quote("documents") }} ->> 'UpdatedDate')::date as updated_date_utc, ({{ adapter.quote("documents") }} ->> 'CreationDate')::timestamp - as cosmos_creation_at_utc, + as creation_at_utc, ({{ adapter.quote("documents") }} ->> 'CreationDate')::date - as cosmos_creation_date_utc, + as creation_date_utc, ({{ adapter.quote("documents") }} ->> 'CreatedDate')::date - as created_date_utc, + as cosmos_created_date_utc, to_timestamp( (({{ adapter.quote("documents") }} ->> '_ts'))::integer ) as cosmos_db_timestamp_utc