From c209e670ca57ca269803011093bf7c528369a075 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oriol=20Roqu=C3=A9=20Paniagua?= Date: Fri, 15 Nov 2024 10:04:02 +0000 Subject: [PATCH] Merged PR 3539: Normalise service names and create booking to service table # Description Goal of the PR is start modelising the Services and Revenue for New Dash/New Pricing. Be aware that this is a first model and it's not by all means the definitive nor complete version. Changes: * Services name normalisation: It's a bit painful because we usually had two names: 'MyService' and 'My Service'. The initial setup I built was considering the first, but in some new integrated tables it only has the second. So I just standardised everything to be in upper case and with spaces such as 'MY SERVICE'. Might make more sense later on. This affects many existing models. * Compute a new Service Business Type field in `stg_core__product_service`: Main reason is that effectively Platform scope in New Pricing have business differences in the services types but this does not exist in the backend. I do it at staging level with a dedicated accepted values test to ensure any new update is captured and notified. You can read more about the business logic in this [Notion page](https://www.notion.so/knowyourguest-superhog/All-Services-Definitions-333fba1fbed54715b6b1c3b5133f0d4f). * Create a new model `int_core__booking_to_service`: this model joins the information of BookingViewToService with BookingView and Booking tables to retrieve a more consolidated version of what we will needed downstream for the future table of Booking Service Detail and the Aggregated version (not included in this PR). This table likely can be enriched in future PRs. At the moment it just contains the bare minimum fields needed for the lines/aggregated versions. (NB: Booking information is needed, for instance, for the services which unit price need to be applied for the number of nights of the booking) # Checklist - [X] The edited models and dependants run properly with production data. - [X] The edited models are sufficiently documented. - [X] The edited models contain PK tests, and I've ran and passed them. - [X] I have checked for DRY opportunities with other models and docs. - [X] I've picked the right materialization for the affected models. # Other - [X] Check if a full-refresh is required after this PR is merged. **stg_core__booking_view_to_service needs full refresh** Related work items: #20809 --- ..._core__accommodation_to_product_bundle.sql | 3 +- .../int_core__booking_to_product_bundle.sql | 2 +- .../core/int_core__booking_to_service.sql | 31 ++++ .../core/int_core__new_dash_user_overview.sql | 2 +- .../int_core__product_service_to_price.sql | 1 + .../core/int_core__protection_plan_cover.sql | 1 + .../int_core__protection_plan_to_price.sql | 1 + models/intermediate/core/schema.yml | 143 +++++++++++++++++- models/staging/core/schema.yml | 15 +- .../stg_core__booking_view_to_service.sql | 2 +- .../core/stg_core__product_service.sql | 34 ++++- .../core/stg_core__protection_plan.sql | 4 +- .../core/stg_core__user_product_bundle.sql | 4 +- 13 files changed, 224 insertions(+), 19 deletions(-) create mode 100644 models/intermediate/core/int_core__booking_to_service.sql diff --git a/models/intermediate/core/int_core__accommodation_to_product_bundle.sql b/models/intermediate/core/int_core__accommodation_to_product_bundle.sql index 9625857..01e7931 100644 --- a/models/intermediate/core/int_core__accommodation_to_product_bundle.sql +++ b/models/intermediate/core/int_core__accommodation_to_product_bundle.sql @@ -11,7 +11,7 @@ select atpb.id_accommodation, atpb.id_user_product_bundle, upb.id_user_host, - upb.product_bundle_name as user_product_bundle_name, + upb.product_bundle_display_name as user_product_bundle_name, atpb.starts_at_utc as original_starts_at_utc, atpb.ends_at_utc as original_ends_at_utc, /* @@ -39,4 +39,3 @@ from stg_core__accommodation_to_product_bundle as atpb inner join int_core__user_product_bundle as upb on atpb.id_user_product_bundle = upb.id_user_product_bundle - diff --git a/models/intermediate/core/int_core__booking_to_product_bundle.sql b/models/intermediate/core/int_core__booking_to_product_bundle.sql index 9f741e4..7faa8c3 100644 --- a/models/intermediate/core/int_core__booking_to_product_bundle.sql +++ b/models/intermediate/core/int_core__booking_to_product_bundle.sql @@ -11,7 +11,7 @@ select btpb.id_booking, btpb.id_user_product_bundle, upb.id_user_host, - upb.product_bundle_name as user_product_bundle_name, + upb.product_bundle_display_name as user_product_bundle_name, btpb.created_at_utc, btpb.created_date_utc, btpb.updated_at_utc, diff --git a/models/intermediate/core/int_core__booking_to_service.sql b/models/intermediate/core/int_core__booking_to_service.sql new file mode 100644 index 0000000..de1484b --- /dev/null +++ b/models/intermediate/core/int_core__booking_to_service.sql @@ -0,0 +1,31 @@ +{{ config(materialized="table", unique_key=["id_booking_view_to_service"]) }} +with + stg_core__booking_view_to_service as ( + select * from {{ ref("stg_core__booking_view_to_service") }} + ), + stg_core__booking_view as (select * from {{ ref("stg_core__booking_view") }}), + stg_core__booking as (select * from {{ ref("stg_core__booking") }}), + int_core__user_host as (select * from {{ ref("int_core__user_host") }}) +select + bv.id_booking, + bvts.id_booking_view_to_service as id_booking_service_detail, + bvts.id_verification, + bvts.id_product_service, + bvts.id_protection_plan, + bvts.service_status, + bvts.service_name, + bvts.service_protection_amount, + bvts.created_at_utc as service_detail_created_at_utc, + bvts.updated_at_utc as service_detail_updated_at_utc, + b.created_at_utc as booking_created_at_utc, + b.check_in_at_utc as booking_check_in_at_utc, + b.check_out_at_utc as booking_check_out_at_utc, + b.id_user_host, + uh.account_currency_iso4217 as host_currency_code, + case + when uh.account_currency_iso4217 is null then true else false + end as is_missing_host_currency_code +from stg_core__booking_view_to_service bvts +inner join stg_core__booking_view bv on bvts.id_booking_view = bv.id_booking_view +left join stg_core__booking b on bv.id_booking = b.id_booking +left join int_core__user_host uh on b.id_user_host = uh.id_user_host diff --git a/models/intermediate/core/int_core__new_dash_user_overview.sql b/models/intermediate/core/int_core__new_dash_user_overview.sql index 3366ed0..98e5ccf 100644 --- a/models/intermediate/core/int_core__new_dash_user_overview.sql +++ b/models/intermediate/core/int_core__new_dash_user_overview.sql @@ -1,4 +1,4 @@ -{% set product_bundles_without_paid_service = "('BasicScreening')" %} +{% set product_bundles_without_paid_service = "('BASIC SCREENING')" %} with int_core__user_product_bundle as ( select * from {{ ref("int_core__user_product_bundle") }} diff --git a/models/intermediate/core/int_core__product_service_to_price.sql b/models/intermediate/core/int_core__product_service_to_price.sql index 6204a29..48e5d82 100644 --- a/models/intermediate/core/int_core__product_service_to_price.sql +++ b/models/intermediate/core/int_core__product_service_to_price.sql @@ -14,6 +14,7 @@ select pstp.amount_local_curr, pstp.product_service_price_name, ps.product_service_name, + ps.product_service_display_name, pstp.starts_at_utc, pstp.ends_at_utc, pstp.has_no_end_date, diff --git a/models/intermediate/core/int_core__protection_plan_cover.sql b/models/intermediate/core/int_core__protection_plan_cover.sql index 2aa8623..4753cb6 100644 --- a/models/intermediate/core/int_core__protection_plan_cover.sql +++ b/models/intermediate/core/int_core__protection_plan_cover.sql @@ -13,6 +13,7 @@ select ppc.maximum_protection_cover_local_curr, ppc.protection_plan_cover_name, pp.protection_name, + pp.protection_display_name, ppc.created_at_utc, ppc.created_date_utc, ppc.updated_at_utc, diff --git a/models/intermediate/core/int_core__protection_plan_to_price.sql b/models/intermediate/core/int_core__protection_plan_to_price.sql index 261776f..c6326a7 100644 --- a/models/intermediate/core/int_core__protection_plan_to_price.sql +++ b/models/intermediate/core/int_core__protection_plan_to_price.sql @@ -14,6 +14,7 @@ select pptp.amount_local_curr, pptp.protection_plan_price_name, pp.protection_name, + pp.protection_display_name, pptp.starts_at_utc, pptp.ends_at_utc, pptp.has_no_end_date, diff --git a/models/intermediate/core/schema.yml b/models/intermediate/core/schema.yml index a3432b4..efa6aa3 100644 --- a/models/intermediate/core/schema.yml +++ b/models/intermediate/core/schema.yml @@ -1994,7 +1994,7 @@ models: - name: product_bundle_name data_type: string description: | - The CamelCase name of the Product Bundle. + The name of the Product Bundle. - name: product_bundle_display_name data_type: string @@ -2150,7 +2150,7 @@ models: - name: product_bundle_name data_type: string description: | - The CamelCase name of the Product Bundle. + The name of the Product Bundle. - name: created_at_utc data_type: timestamp @@ -2241,7 +2241,7 @@ models: - name: product_bundle_name data_type: string description: | - The CamelCase name of the Product Bundle. + The name of the Product Bundle. - name: original_starts_at_utc data_type: timestamp @@ -2818,14 +2818,14 @@ models: - name: product_bundle_name data_type: string description: | - The CamelCase name of the product bundle. + The name of the product bundle. tests: - not_null - name: product_service_name data_type: string description: | - The CamelCase name of the product service. + The name of the product service. tests: - not_null @@ -3149,3 +3149,136 @@ models: Timestamp of when this data was extracted into DWH. tests: - not_null + + - name: int_core__booking_to_service + description: | + This model contains the information of which booking has a certain + service applied. It's a subset of all bookings since it only + applies to bookings that come from hosts that have been migrated into + the New Dash or New Pricing. + + tests: + - dbt_expectations.expect_column_pair_values_A_to_be_greater_than_B: + column_A: booking_check_out_at_utc + column_B: booking_check_in_at_utc + or_equal: True + - dbt_expectations.expect_column_pair_values_A_to_be_greater_than_B: + column_A: service_detail_created_at_utc + column_B: booking_created_at_utc + or_equal: True + - dbt_expectations.expect_column_pair_values_A_to_be_greater_than_B: + column_A: service_detail_updated_at_utc + column_B: service_detail_created_at_utc + or_equal: True + + columns: + - name: id_booking + data_type: bigint + description: | + The identifier of the booking. Acts as Foreign Key to Booking table. + Cannot be null. + tests: + - not_null + + - name: id_booking_service_detail + data_type: bigint + description: | + The identifier of the booking that has a certain service applied. + The same booking can have multiple services applied and would + have distinct id_booking_service_detail. + tests: + - not_null + - unique + + - name: id_verification + data_type: bigint + description: | + The identifier of the verification. It acts as Foreign Key to Verification + table. It can be null. + + - name: id_product_service + data_type: bigint + description: | + The identifier of the product service. It acts as Foreign Key to Product + Service table. It can be null. + + - name: id_protection_plan + data_type: bigint + description: | + The identifier of the protection plan, aka protection service. It acts as + Foreign Key to Product Plan table. It can be null. + + - name: service_status + data_type: string + description: | + The status of the service applied within the booking. Cannot be null. + tests: + - not_null + + - name: service_name + data_type: string + description: | + The name of the service applied within the booking. Cannot be null. + tests: + - not_null + + - name: service_protection_amount + data_type: string + description: | + The amount protected given for the service applied within the booking. + Cannot be null. + tests: + - not_null + + - name: service_detail_created_at_utc + data_type: timestamp + description: | + Timestamp of when this Service row was created in the Backend. + tests: + - not_null + + - name: service_detail_updated_at_utc + data_type: timestamp + description: | + Timestamp of when this Service row was last updated in the Backend. + tests: + - not_null + + - name: booking_created_at_utc + data_type: timestamp + description: | + Timestamp of when the Booking row was created in the Backend. + tests: + - not_null + + - name: booking_check_in_at_utc + data_type: timestamp + description: | + Timestamp of the Check-in of the Booking. + tests: + - not_null + + - name: booking_check_out_at_utc + data_type: timestamp + description: | + Timestamp of the Check-out of the Booking. + tests: + - not_null + + - name: id_user_host + data_type: string + description: | + Unique identifier of the user that acts as Host. + tests: + - not_null + + - name: host_currency_code + data_type: string + description: | + Iso 4217 currency code for the account of the Host. + It can be null. + + - name: is_missing_host_currency_code + data_type: boolean + description: | + Flag to identify if the host is missing the currency code. diff --git a/models/staging/core/schema.yml b/models/staging/core/schema.yml index 4133f18..9b504b8 100644 --- a/models/staging/core/schema.yml +++ b/models/staging/core/schema.yml @@ -522,11 +522,11 @@ models: - name: product_service_name data_type: string description: | - CamelCase name of the product service. Uniquely identifies the product service. + Name of the product service. Uniquely identifies the product service. tests: - unique - not_null - - name: product_service_full_name + - name: product_service_display_name data_type: string description: | A more readable way to display the product service. It's mainly product_service_name @@ -556,6 +556,17 @@ models: tests: - not_null - unique + - name: service_business_type + data_type: string + description: | + Specifies the business type of the service, in essence, whether it is a Screening service + or a Deposit Management service. + tests: + - not_null + - accepted_values: + values: + - "SCREENING" + - "DEPOSIT_MANAGEMENT" - name: dwh_extracted_at_utc data_type: timestamp description: | diff --git a/models/staging/core/stg_core__booking_view_to_service.sql b/models/staging/core/stg_core__booking_view_to_service.sql index 8e76f72..8032a83 100644 --- a/models/staging/core/stg_core__booking_view_to_service.sql +++ b/models/staging/core/stg_core__booking_view_to_service.sql @@ -11,7 +11,7 @@ with {{ adapter.quote("ProductServiceId") }} as id_product_service, {{ adapter.quote("ProtectionPlanId") }} as id_protection_plan, - {{ adapter.quote("ServiceName") }} as service_name, + upper({{ adapter.quote("ServiceName") }}) as service_name, {{ adapter.quote("ProtectionAmount") }} as service_protection_amount, upper({{ adapter.quote("Status") }}) as service_status, diff --git a/models/staging/core/stg_core__product_service.sql b/models/staging/core/stg_core__product_service.sql index 9b282ae..7faf742 100644 --- a/models/staging/core/stg_core__product_service.sql +++ b/models/staging/core/stg_core__product_service.sql @@ -4,8 +4,8 @@ with select {{ adapter.quote("Id") }} as id_product_service, - {{ adapter.quote("Name") }} as product_service_name, - {{ adapter.quote("FullName") }} as product_service_full_name, + upper({{ adapter.quote("Name") }}) as product_service_name, + upper({{ adapter.quote("FullName") }}) as product_service_display_name, {{ adapter.quote("Description") }} as product_service_description, {{ adapter.quote("ProductServiceFlag") }} as product_service_binary_tier, @@ -13,5 +13,33 @@ with from raw_product_service ) -select * +select + id_product_service, + case + when + id_product_service in ( + + 1, -- 'BASIC SCREENING' + 2, -- 'SEX OFFENDERS CHECK' + 3, -- 'ID VERIFICATION' + 4 -- 'SCREENING PLUS' + ) + then 'SCREENING' + when + id_product_service in ( + + 5, -- 'BASIC DAMAGE DEPOSIT' + 6, -- 'DAMAGE DEPOSIT PLUS' + 7, -- 'BASIC WAIVER' + 8, -- 'WAIVER PLUS' + 9 -- 'WAIVER PRO' + ) + then 'DEPOSIT_MANAGEMENT' + else null + end as service_business_type, + product_service_name, + product_service_display_name, + product_service_description, + product_service_binary_tier, + dwh_extracted_at_utc from stg_core__product_service diff --git a/models/staging/core/stg_core__protection_plan.sql b/models/staging/core/stg_core__protection_plan.sql index ec45a21..0211a9f 100644 --- a/models/staging/core/stg_core__protection_plan.sql +++ b/models/staging/core/stg_core__protection_plan.sql @@ -3,8 +3,8 @@ with select {{ adapter.quote("Id") }} as id, - {{ adapter.quote("Name") }} as protection_name, - {{ adapter.quote("DisplayName") }} as protection_display_name, + upper({{ adapter.quote("Name") }}) as protection_name, + upper({{ adapter.quote("DisplayName") }}) as protection_display_name, {{ adapter.quote("RequiredProductServices") }} as required_product_services_binary_tier diff --git a/models/staging/core/stg_core__user_product_bundle.sql b/models/staging/core/stg_core__user_product_bundle.sql index 4c949f7..fcf8280 100644 --- a/models/staging/core/stg_core__user_product_bundle.sql +++ b/models/staging/core/stg_core__user_product_bundle.sql @@ -9,8 +9,8 @@ with {{ adapter.quote("ProductBundleId") }} as id_product_bundle, {{ adapter.quote("ProtectionPlanId") }} as id_protection_plan, - {{ adapter.quote("Name") }} as product_bundle_name, - {{ adapter.quote("DisplayName") }} as product_bundle_display_name, + upper({{ adapter.quote("Name") }}) as product_bundle_name, + upper({{ adapter.quote("DisplayName") }}) as product_bundle_display_name, {{ adapter.quote("DisplayOnFrontEnd") }} as display_on_front_end, {{ adapter.quote("ChosenProductServices") }} as chosen_product_services,