new approach for S&P users data extraction

models/intermediate/core/int_core__screen_and_protect_users.sql

@@ -0,0 +1,46 @@
+{% set api_name = "SCREENANDPROTECTAPI" %}
+
+with
+    stg_core__apim_user as (select * from {{ ref("stg_core__apim_user") }}),
+    stg_core__apim_user_type as (select * from {{ ref("stg_core__apim_user_type") }})
+select
+    -- note that these ids are not the same as the ones found in Core
+    -- they are completely unrelated
+    au.id_apim_user,
+    au.id_apim_user_type,
+    au.json_document_user_data ->> 'DealId' as id_deal,
+    au.json_document_user_data ->> 'AccountType' as account_type,
+    au.json_document_user_data ->> 'ClientMarkup' as client_markup,
+    au.json_document_user_data ->> 'TechEmailAddress' as tech_email_address,
+    au.json_document_user_data ->> 'InvoicingEmailAddress' as invoicing_email_address,
+    case
+        when (au.json_document_user_data ->> 'FlaggedNotProtected') = 'false'
+        then true
+        else false
+    end as is_protected,
+    (au.json_document_user_data ->> 'PriceIncrease')::decimal as price_increase,
+    (au.json_document_user_data ->> 'PriceIncreaseStartDate')::date
+    as price_increase_start_date_utc,
+    (au.json_document_user_data ->> 'MonthlyVolumeDiscount')::decimal
+    as monthly_volume_discount,
+    (au.json_document_user_data ->> 'ThresholdApprovedBookingVolume')::integer
+    as threshold_approved_booking_volume,
+    (au.json_document_user_data ->> 'MonthlyVolumeDiscountStartDate')::date
+    as monthly_volume_discount_start_date_utc,
+    (au.json_document_user_data ->> 'MonthlyVolumeDiscountEndDate')::date
+    as monthly_volume_discount_end_date_utc,
+    (au.json_document_user_data ->> 'MonthlyGeneralDiscount')::decimal
+    as monthly_general_discount,
+    (au.json_document_user_data ->> 'MonthlyGeneralDiscountStartDate')::date
+    as monthly_general_discount_start_date_utc,
+    (au.json_document_user_data ->> 'MonthlyGeneralDiscountEndDate')::date
+    as monthly_general_discount_end_date_utc,
+    au.created_at_utc,
+    au.created_date_utc,
+    au.updated_at_utc,
+    au.updated_date_utc
+from stg_core__apim_user au
+inner join
+    stg_core__apim_user_type aut
+    on au.id_apim_user_type = aut.id_apim_user_type
+    and upper(aut.user_type_name) = '{{ api_name }}'

models/intermediate/core/schema.yml

@@ -4427,3 +4427,150 @@ models:
         description: |
           Check-in Cover fees revenue paid without taxes in GBP.
           Can be null.
+
+  - name: int_core__screen_and_protect_users
+    description: |
+      "Contains information about Screen & Protect API users.
+      It includes all user details related with the service."
+    columns:
+      - name: id_apim_user
+        data_type: character varying
+        description: "Identifier of the User. Acts as primary
+          key for this table."
+        tests:
+          - not_null
+          - unique
+
+      - name: id_apim_user_type
+        data_type: bigint
+        description: "Identifier of the type of user."
+
+      - name: id_deal
+        data_type: text
+        description: ""
+
+      - name: account_type
+        data_type: text
+        description: ""
+
+      - name: client_markup
+        data_type: text
+        description: ""
+
+      - name: tech_email_address
+        data_type: text
+        description: ""
+
+      - name: invoicing_email_address
+        data_type: text
+        description: ""
+
+      - name: is_protected
+        data_type: boolean
+        description: |
+          Indicates if the user's bookings are protected or not.
+        tests:
+          - not_null
+
+      - name: price_increase
+        data_type: numeric
+        description: The percentage or value of the price increase
+          applied to the user's account.
+        tests:
+          - dbt_expectations.expect_column_values_to_be_between:
+              min_value: 0
+              strictly: true
+
+      - name: price_increase_start_date_utc
+        data_type: date
+        description: The date when the price increase becomes effective.
+          This is the first day of the month.
+        tests:
+          - is_first_day_of_month
+
+      - name: monthly_volume_discount
+        data_type: numeric
+        description: The discount percentage or value offered based on the
+          volume of bookings achieved within a month.
+          No user can have more than one discount per month.
+        tests:
+          - dbt_expectations.expect_column_values_to_be_between:
+              min_value: 0
+              max_value: 100
+              strictly: true
+
+      - name: threshold_approved_booking_volume
+        data_type: numeric
+        description: The minimum number of bookings required to qualify for
+          the monthly volume discount.
+        tests:
+          - dbt_expectations.expect_column_values_to_be_between:
+              min_value: 0
+              strictly: true
+
+      - name: monthly_volume_discount_start_date_utc
+        data_type: date
+        description: The date when the monthly volume discount period begins.
+          This is the first day of the month.
+        tests:
+          - is_first_day_of_month
+
+      - name: monthly_volume_discount_end_date_utc
+        data_type: date
+        description: The date when the monthly volume discount period ends.
+          This is the last day of the month.
+        tests:
+          - is_last_day_of_month
+
+      - name: monthly_general_discount
+        data_type: numeric
+        description: The general discount percentage or value applied to all
+          bookings within the applicable period.
+          No user can have more than one discount per month.
+        tests:
+          - dbt_expectations.expect_column_values_to_be_between:
+              min_value: 0
+              max_value: 100
+              strictly: true
+
+      - name: monthly_general_discount_start_date_utc
+        data_type: date
+        description: The date when the general discount period begins.
+          This is the first day of the month.
+        tests:
+          - is_first_day_of_month
+
+      - name: monthly_general_discount_end_date_utc
+        data_type: date
+        description: The date when the general discount period ends.
+          This is the last day of the month.
+        tests:
+          - is_last_day_of_month
+
+      - name: created_at_utc
+        data_type: timestamp
+        description: |
+          Timestamp of when this user was created.
+        tests:
+          - not_null
+
+      - name: created_date_utc
+        data_type: date
+        description: |
+          Date of when this user was created.
+        tests:
+          - not_null
+
+      - name: updated_at_utc
+        data_type: timestamp
+        description: |
+          Timestamp of when this user was last updated.
+        tests:
+          - not_null
+
+      - name: updated_date_utc
+        data_type: date
+        description: |
+          Date of when this user was last updated.
+        tests:
+          - not_null

models/intermediate/screen_and_protect/int_screen_and_protect__verification_requests.sql

@@ -1,11 +1,10 @@
-{% set api_name = "SCREENANDPROTECTAPI" %}
-
 with
     stg_screen_and_protect__verification_requests as (
         select * from {{ ref("stg_screen_and_protect__verification_requests") }}
     ),
-    stg_core__apim_user as (select * from {{ ref("stg_core__apim_user") }}),
+    int_core__screen_and_protect_users as (
-    stg_core__apim_user_type as (select * from {{ ref("stg_core__apim_user_type") }})
+        select * from {{ ref("int_core__screen_and_protect_users") }}
+    )
 select
     -- note that these ids are not the same as the ones found in Core
     -- they are completely unrelated
@@ -13,11 +12,7 @@ select
     vr.id_booking,
     vr.id_user_partner,
     vr.id_accommodation,
-    case
+    spu.is_protected,
-        when (au.json_document_user_data ->> 'FlaggedNotProtected') = 'false'
-        then true
-        else false
-    end as is_protected,
     vr.protection_type,
     vr.protection_starting_level,
     vr.protection_basic_amount,
@@ -25,23 +20,15 @@ select
     vr.pet_protection,
     vr.verification_status,
     vr.verification_status_reason,
-    (au.json_document_user_data ->> 'PriceIncrease')::decimal as price_increase,
+    spu.price_increase,
-    (au.json_document_user_data ->> 'PriceIncreaseStartDate')::date
+    spu.price_increase_start_date_utc,
-    as price_increase_start_date_utc,
+    spu.monthly_volume_discount,
-    (au.json_document_user_data ->> 'MonthlyVolumeDiscount')::decimal
+    spu.threshold_approved_booking_volume,
-    as monthly_volume_discount,
+    spu.monthly_volume_discount_start_date_utc,
-    (au.json_document_user_data ->> 'ThresholdApprovedBookingVolume')::integer
+    spu.monthly_volume_discount_end_date_utc,
-    as threshold_approved_booking_volume,
+    spu.monthly_general_discount,
-    (au.json_document_user_data ->> 'MonthlyVolumeDiscountStartDate')::date
+    spu.monthly_general_discount_start_date_utc,
-    as monthly_volume_discount_start_date_utc,
+    spu.monthly_general_discount_end_date_utc,
-    (au.json_document_user_data ->> 'MonthlyVolumeDiscountEndDate')::date
-    as monthly_volume_discount_end_date_utc,
-    (au.json_document_user_data ->> 'MonthlyGeneralDiscount')::decimal
-    as monthly_general_discount,
-    (au.json_document_user_data ->> 'MonthlyGeneralDiscountStartDate')::date
-    as monthly_general_discount_start_date_utc,
-    (au.json_document_user_data ->> 'MonthlyGeneralDiscountEndDate')::date
-    as monthly_general_discount_end_date_utc,
     vr.email_flag,
     vr.phone_flag,
     vr.watch_list,
@@ -74,8 +61,5 @@ select
     vr.creation_date_utc,
     vr.cosmos_created_date_utc
 from stg_screen_and_protect__verification_requests vr
-inner join stg_core__apim_user au on vr.id_user_partner = au.id_apim_user
 inner join
-    stg_core__apim_user_type aut
+    int_core__screen_and_protect_users spu on vr.id_user_partner = spu.id_apim_user
-    on au.id_apim_user_type = aut.id_apim_user_type
-    and upper(aut.user_type_name) = '{{ api_name }}'

models/intermediate/screen_and_protect/schema.yml

@@ -124,7 +124,7 @@ models:
         tests:
           - dbt_expectations.expect_column_values_to_be_between:
               min_value: 0
-              max_value: 1
+              max_value: 100
               strictly: true
 
       - name: threshold_approved_booking_volume
@@ -158,7 +158,7 @@ models:
         tests:
           - dbt_expectations.expect_column_values_to_be_between:
               min_value: 0
-              max_value: 1
+              max_value: 100
               strictly: true
 
       - name: monthly_general_discount_start_date_utc