Merged PR 3869: S&P models update

# Description

This PR is mainly to update the `int_screen_and_protect__verification_requests` model to include some data from Wilbur needed for the invoice.
This includes fields like `price_discounts` or `price_increases` all of which have a start and end date which should be the first and last date of any month correspondingly for when the discount/increase is being applied. For this I added some custom tests to make sure that the dates are correct, as well as another test to check that the `general_discount` and the `volume_discount` are not active at the same time, is one or the other (let me know if you think it is a bit overkill).

On the other hand I modified the `created_date` and `creation_date` fields on all the verification_request models for S&P which where incorrectly classified as cosmos_creation_date or reservation_created_date and it was the other way around. (I am aware that this will affect the PBI report, currently there is no real data so it is not an issue)

# Checklist

- [x] The edited models and dependants run properly with production data.
- [x] The edited models are sufficiently documented.
- [x] The edited models contain PK tests, and I've ran and passed them.
- [x] I have checked for DRY opportunities with other models and docs.
- [x] I've picked the right materialization for the affected models.

# Other

- [ ] Check if a full-refresh is required after this PR is merged.

s&p update models

Related work items: #25360

macros/tests/at_least_one_null.sql

@@ -0,0 +1,10 @@
+{% test at_least_one_null(model, columns) %}
+
+    select *
+    from {{ model }}
+    where
+        {% for column in columns %}
+            {{ column }} is not null {% if not loop.last %} and {% endif %}
+        {% endfor %}
+-- If all columns are not null, the row violates the test
+{% endtest %}

macros/tests/is_first_day_of_month.sql

@@ -0,0 +1,3 @@
+{% test is_first_day_of_month(model, column_name) %}
+    select * from {{ model }} where extract(day from {{ column_name }}) != 1
+{% endtest %}

macros/tests/is_last_day_of_month.sql

@@ -0,0 +1,10 @@
+{% test is_last_day_of_month(model, column_name) %}
+    select *
+    from {{ model }}
+    where
+        {{ column_name }} != (
+            date_trunc('MONTH', {{ column_name }})
+            + interval '1 MONTH'
+            - interval '1 DAY'
+        )
+{% endtest %}

models/intermediate/core/int_core__screen_and_protect_users.sql

@@ -0,0 +1,46 @@
+{% set api_name = "SCREENANDPROTECTAPI" %}
+
+with
+    stg_core__apim_user as (select * from {{ ref("stg_core__apim_user") }}),
+    stg_core__apim_user_type as (select * from {{ ref("stg_core__apim_user_type") }})
+select
+    -- note that these ids are not the same as the ones found in Core
+    -- they are completely unrelated
+    au.id_apim_user,
+    au.id_apim_user_type,
+    au.json_document_user_data ->> 'DealId' as id_deal,
+    au.json_document_user_data ->> 'AccountType' as account_type,
+    au.json_document_user_data ->> 'ClientMarkup' as client_markup,
+    au.json_document_user_data ->> 'TechEmailAddress' as tech_email_address,
+    au.json_document_user_data ->> 'InvoicingEmailAddress' as invoicing_email_address,
+    case
+        when (au.json_document_user_data ->> 'FlaggedNotProtected') = 'false'
+        then true
+        else false
+    end as is_protected,
+    (au.json_document_user_data ->> 'PriceIncrease')::decimal as price_increase,
+    (au.json_document_user_data ->> 'PriceIncreaseStartDate')::date
+    as price_increase_start_date_utc,
+    (au.json_document_user_data ->> 'MonthlyVolumeDiscount')::decimal
+    as monthly_volume_discount,
+    (au.json_document_user_data ->> 'ThresholdApprovedBookingVolume')::integer
+    as threshold_approved_booking_volume,
+    (au.json_document_user_data ->> 'MonthlyVolumeDiscountStartDate')::date
+    as monthly_volume_discount_start_date_utc,
+    (au.json_document_user_data ->> 'MonthlyVolumeDiscountEndDate')::date
+    as monthly_volume_discount_end_date_utc,
+    (au.json_document_user_data ->> 'MonthlyGeneralDiscount')::decimal
+    as monthly_general_discount,
+    (au.json_document_user_data ->> 'MonthlyGeneralDiscountStartDate')::date
+    as monthly_general_discount_start_date_utc,
+    (au.json_document_user_data ->> 'MonthlyGeneralDiscountEndDate')::date
+    as monthly_general_discount_end_date_utc,
+    au.created_at_utc,
+    au.created_date_utc,
+    au.updated_at_utc,
+    au.updated_date_utc
+from stg_core__apim_user au
+inner join
+    stg_core__apim_user_type aut
+    on au.id_apim_user_type = aut.id_apim_user_type
+    and upper(aut.user_type_name) = '{{ api_name }}'

models/intermediate/core/schema.yml

@@ -4427,3 +4427,150 @@ models:
         description: |
           Check-in Cover fees revenue paid without taxes in GBP.
           Can be null.
+
+  - name: int_core__screen_and_protect_users
+    description: |
+      "Contains information about Screen & Protect API users.
+      It includes all user details related with the service."
+    columns:
+      - name: id_apim_user
+        data_type: character varying
+        description: "Identifier of the User. Acts as primary
+          key for this table."
+        tests:
+          - not_null
+          - unique
+
+      - name: id_apim_user_type
+        data_type: bigint
+        description: "Identifier of the type of user."
+
+      - name: id_deal
+        data_type: text
+        description: ""
+
+      - name: account_type
+        data_type: text
+        description: ""
+
+      - name: client_markup
+        data_type: text
+        description: ""
+
+      - name: tech_email_address
+        data_type: text
+        description: ""
+
+      - name: invoicing_email_address
+        data_type: text
+        description: ""
+
+      - name: is_protected
+        data_type: boolean
+        description: |
+          Indicates if the user's bookings are protected or not.
+        tests:
+          - not_null
+
+      - name: price_increase
+        data_type: numeric
+        description: The percentage or value of the price increase
+          applied to the user's account.
+        tests:
+          - dbt_expectations.expect_column_values_to_be_between:
+              min_value: 0
+              strictly: true
+
+      - name: price_increase_start_date_utc
+        data_type: date
+        description: The date when the price increase becomes effective.
+          This is the first day of the month.
+        tests:
+          - is_first_day_of_month
+
+      - name: monthly_volume_discount
+        data_type: numeric
+        description: The discount percentage or value offered based on the
+          volume of bookings achieved within a month.
+          No user can have more than one discount per month.
+        tests:
+          - dbt_expectations.expect_column_values_to_be_between:
+              min_value: 0
+              max_value: 100
+              strictly: true
+
+      - name: threshold_approved_booking_volume
+        data_type: numeric
+        description: The minimum number of bookings required to qualify for
+          the monthly volume discount.
+        tests:
+          - dbt_expectations.expect_column_values_to_be_between:
+              min_value: 0
+              strictly: true
+
+      - name: monthly_volume_discount_start_date_utc
+        data_type: date
+        description: The date when the monthly volume discount period begins.
+          This is the first day of the month.
+        tests:
+          - is_first_day_of_month
+
+      - name: monthly_volume_discount_end_date_utc
+        data_type: date
+        description: The date when the monthly volume discount period ends.
+          This is the last day of the month.
+        tests:
+          - is_last_day_of_month
+
+      - name: monthly_general_discount
+        data_type: numeric
+        description: The general discount percentage or value applied to all
+          bookings within the applicable period.
+          No user can have more than one discount per month.
+        tests:
+          - dbt_expectations.expect_column_values_to_be_between:
+              min_value: 0
+              max_value: 100
+              strictly: true
+
+      - name: monthly_general_discount_start_date_utc
+        data_type: date
+        description: The date when the general discount period begins.
+          This is the first day of the month.
+        tests:
+          - is_first_day_of_month
+
+      - name: monthly_general_discount_end_date_utc
+        data_type: date
+        description: The date when the general discount period ends.
+          This is the last day of the month.
+        tests:
+          - is_last_day_of_month
+
+      - name: created_at_utc
+        data_type: timestamp
+        description: |
+          Timestamp of when this user was created.
+        tests:
+          - not_null
+
+      - name: created_date_utc
+        data_type: date
+        description: |
+          Date of when this user was created.
+        tests:
+          - not_null
+
+      - name: updated_at_utc
+        data_type: timestamp
+        description: |
+          Timestamp of when this user was last updated.
+        tests:
+          - not_null
+
+      - name: updated_date_utc
+        data_type: date
+        description: |
+          Date of when this user was last updated.
+        tests:
+          - not_null

models/intermediate/screen_and_protect/int_screen_and_protect__verification_requests.sql

@@ -1,23 +1,18 @@
-{% set api_name = "SCREENANDPROTECTAPI" %}
-
 with
     stg_screen_and_protect__verification_requests as (
         select * from {{ ref("stg_screen_and_protect__verification_requests") }}
     ),
-    stg_core__apim_user as (select * from {{ ref("stg_core__apim_user") }}),
+    int_core__screen_and_protect_users as (
-    stg_core__apim_user_type as (select * from {{ ref("stg_core__apim_user_type") }})
+        select * from {{ ref("int_core__screen_and_protect_users") }}
+    )
 select
-    -- note that these ids are not the same as the ones found in Core DWH
+    -- note that these ids are not the same as the ones found in Core
     -- they are completely unrelated
     vr.id_verification,
     vr.id_booking,
     vr.id_user_partner,
     vr.id_accommodation,
-    case
+    spu.is_protected,
-        when (au.json_document_user_data ->> 'FlaggedNotProtected') = 'false'
-        then true
-        else false
-    end as is_protected,
     vr.protection_type,
     vr.protection_starting_level,
     vr.protection_basic_amount,
@@ -25,6 +20,15 @@ select
     vr.pet_protection,
     vr.verification_status,
     vr.verification_status_reason,
+    spu.price_increase,
+    spu.price_increase_start_date_utc,
+    spu.monthly_volume_discount,
+    spu.threshold_approved_booking_volume,
+    spu.monthly_volume_discount_start_date_utc,
+    spu.monthly_volume_discount_end_date_utc,
+    spu.monthly_general_discount,
+    spu.monthly_general_discount_start_date_utc,
+    spu.monthly_general_discount_end_date_utc,
     vr.email_flag,
     vr.phone_flag,
     vr.watch_list,
@@ -53,12 +57,9 @@ select
     vr.status_updated_date_utc,
     vr.updated_at_utc,
     vr.updated_date_utc,
-    vr.cosmos_creation_at_utc,
+    vr.creation_at_utc,
-    vr.cosmos_creation_date_utc,
+    vr.creation_date_utc,
-    vr.created_date_utc
+    vr.cosmos_created_date_utc
 from stg_screen_and_protect__verification_requests vr
-inner join stg_core__apim_user au on vr.id_user_partner = au.id_apim_user
 inner join
-    stg_core__apim_user_type aut
+    int_core__screen_and_protect_users spu on vr.id_user_partner = spu.id_apim_user
-    on au.id_apim_user_type = aut.id_apim_user_type
-    and upper(aut.user_type_name) = '{{ api_name }}'

models/intermediate/screen_and_protect/schema.yml

@@ -6,6 +6,11 @@ models:
       Records of verification requests from the Screen and Protect API. The 
       table tracks verification requests, their outcomes, and related metadata 
       about guests, listings, and partners.
+    tests:
+      - at_least_one_null:
+          columns:
+            - monthly_volume_discount
+            - monthly_general_discount
     columns:
       - name: id_verification
         data_type: text
@@ -95,6 +100,81 @@ models:
                 - "FLAGGED"
                 - "REJECTED"
 
+      - name: price_increase
+        data_type: numeric
+        description: The percentage or value of the price increase
+          applied to the user's account.
+        tests:
+          - dbt_expectations.expect_column_values_to_be_between:
+              min_value: 0
+              strictly: true
+
+      - name: price_increase_start_date_utc
+        data_type: date
+        description: The date when the price increase becomes effective.
+          This is the first day of the month.
+        tests:
+          - is_first_day_of_month
+
+      - name: monthly_volume_discount
+        data_type: numeric
+        description: The discount percentage or value offered based on the
+          volume of bookings achieved within a month.
+          No user can have more than one discount per month.
+        tests:
+          - dbt_expectations.expect_column_values_to_be_between:
+              min_value: 0
+              max_value: 100
+              strictly: true
+
+      - name: threshold_approved_booking_volume
+        data_type: numeric
+        description: The minimum number of bookings required to qualify for
+          the monthly volume discount.
+        tests:
+          - dbt_expectations.expect_column_values_to_be_between:
+              min_value: 0
+              strictly: true
+
+      - name: monthly_volume_discount_start_date_utc
+        data_type: date
+        description: The date when the monthly volume discount period begins.
+          This is the first day of the month.
+        tests:
+          - is_first_day_of_month
+
+      - name: monthly_volume_discount_end_date_utc
+        data_type: date
+        description: The date when the monthly volume discount period ends.
+          This is the last day of the month.
+        tests:
+          - is_last_day_of_month
+
+      - name: monthly_general_discount
+        data_type: numeric
+        description: The general discount percentage or value applied to all
+          bookings within the applicable period.
+          No user can have more than one discount per month.
+        tests:
+          - dbt_expectations.expect_column_values_to_be_between:
+              min_value: 0
+              max_value: 100
+              strictly: true
+
+      - name: monthly_general_discount_start_date_utc
+        data_type: date
+        description: The date when the general discount period begins.
+          This is the first day of the month.
+        tests:
+          - is_first_day_of_month
+
+      - name: monthly_general_discount_end_date_utc
+        data_type: date
+        description: The date when the general discount period ends.
+          This is the last day of the month.
+        tests:
+          - is_last_day_of_month
+
       - name: verification_status_reason
         data_type: text
         description: Reason for the verification status.
@@ -225,23 +305,23 @@ models:
         tests:
           - not_null
 
-      - name: cosmos_creation_at_utc
+      - name: creation_at_utc
         data_type: timestamp without time zone
         description: |
-          Timestamp of when the verification request was created in Cosmos DB.
+          Timestamp of when the reservation was created.
         tests:
           - not_null
 
-      - name: cosmos_creation_date_utc
+      - name: creation_date_utc
+        data_type: date
+        description: |
+          Date of when the reservation was created.
+        tests:
+          - not_null
+
+      - name: cosmos_created_date_utc
         data_type: date
         description: |
           Date of when the verification request was created in Cosmos DB.
         tests:
           - not_null
-
-      - name: created_date_utc
-        data_type: date
-        description: |
-          Date when the reservation was created.
-        tests:
-          - not_null

models/reporting/screen_and_protect/schema.yml

@@ -225,23 +225,23 @@ models:
         tests:
           - not_null
 
-      - name: cosmos_creation_at_utc
+      - name: creation_at_utc
         data_type: timestamp without time zone
         description: |
-          Timestamp of when the verification request was created in Cosmos DB.
+          Timestamp of when the reservation was created.
         tests:
           - not_null
 
-      - name: cosmos_creation_date_utc
+      - name: creation_date_utc
+        data_type: date
+        description: |
+          Date of when the reservation was created.
+        tests:
+          - not_null
+
+      - name: cosmos_created_date_utc
         data_type: date
         description: |
           Date of when the verification request was created in Cosmos DB.
         tests:
           - not_null
-
-      - name: created_date_utc
-        data_type: date
-        description: |
-          Date when the reservation was created.
-        tests:
-          - not_null

models/reporting/screen_and_protect/screen_and_protect__verification_requests.sql

@@ -45,7 +45,7 @@ select
     status_updated_date_utc as status_updated_date_utc,
     updated_at_utc as updated_at_utc,
     updated_date_utc as updated_date_utc,
-    cosmos_creation_at_utc as cosmos_creation_at_utc,
+    creation_at_utc as cosmos_creation_at_utc,
-    cosmos_creation_date_utc as cosmos_creation_date_utc,
+    creation_date_utc as cosmos_creation_date_utc,
-    created_date_utc as created_date_utc
+    cosmos_created_date_utc as created_date_utc
 from int_screen_and_protect__verification_requests

models/staging/screen_and_protect/schema.yml

@@ -214,27 +214,27 @@ models:
         tests:
           - not_null
 
-      - name: cosmos_creation_at_utc
+      - name: creation_at_utc
         data_type: timestamp without time zone
         description: |
-          Timestamp of when the verification request was created in Cosmos DB.
+          Timestamp of when the reservation was created.
         tests:
           - not_null
 
-      - name: cosmos_creation_date_utc
+      - name: creation_date_utc
+        data_type: date
+        description: |
+          Date of when the reservation was created.
+        tests:
+          - not_null
+
+      - name: cosmos_created_date_utc
         data_type: date
         description: |
           Date of when the verification request was created in Cosmos DB.
         tests:
           - not_null
 
-      - name: created_date_utc
-        data_type: date
-        description: |
-          Date when the reservation was created.
-        tests:
-          - not_null
-
       - name: cosmos_db_timestamp_utc
         data_type: timestamp with time zone
         description: Internal Cosmos DB timestamp of the last record update.

models/staging/screen_and_protect/stg_screen_and_protect__verification_requests.sql

@@ -86,11 +86,11 @@ with
             ({{ adapter.quote("documents") }} ->> 'UpdatedDate')::date
             as updated_date_utc,
             ({{ adapter.quote("documents") }} ->> 'CreationDate')::timestamp
-            as cosmos_creation_at_utc,
+            as creation_at_utc,
             ({{ adapter.quote("documents") }} ->> 'CreationDate')::date
-            as cosmos_creation_date_utc,
+            as creation_date_utc,
             ({{ adapter.quote("documents") }} ->> 'CreatedDate')::date
-            as created_date_utc,
+            as cosmos_created_date_utc,
             to_timestamp(
                 (({{ adapter.quote("documents") }} ->> '_ts'))::integer
             ) as cosmos_db_timestamp_utc