Merged PR 2694: Basic model changes for edeposit

# Description

Basic model changes for edeposit

# Checklist

- [x] The edited models and dependants run properly with production data.
- [x] The edited models are sufficiently documented.
- [x] The edited models contain PK tests, and I've ran and passed them.
- [x] I have checked for DRY opportunities with other models and docs.
- [x] I've picked the right materialization for the affected models.

# Other

- [ ] Check if a full-refresh is required after this PR is merged.

Related work items: #20125

models/intermediate/core/int_edeposit__agg_fee_per_user.sql

@@ -1,62 +0,0 @@
-{% set ok_status = ("Approved", "Flagged") %}
-{% set rejected_status = "Rejected" %}
-{% set rejected_fee = 0.25 %}
-{% set cancellation_fee = 0.25 %}
-{% set cancellation_threshold = 0.05 %}
-with
-    int_edeposit__verifications as (
-        select * from {{ ref("int_edeposit__verifications") }}
-    ),
-    edeposit_records as (
-        select
-            id_booking,
-            is_cancelled,
-            channel,
-            cast(checkin_at_utc as date) as checkin_date_utc,
-            cast(checkout_at_utc as date) as checkout_date_utc,
-            creation_at_utc,
-            verification_status,
-            id_user_host,
-            case
-                when verification_status in {{ ok_status }}
-                then cast(nightly_fee_local as float) * number_nights
-                else 0
-            end as ok_status_fee,
-            case
-                when verification_status = '{{ rejected_status }}'
-                then {{ rejected_fee }}
-                else 0
-            end as rejected_fee,
-            to_char(creation_at_utc, 'YYYY-MM') as year_month_created,
-            to_char(checkout_at_utc, 'YYYY-MM') as year_month_checkout
-        from int_edeposit__verifications
-        where version = 'V2'  -- This version V2 indicates records for e-deposit
-    )
-select
-    year_month_created,
-    year_month_checkout,
-    id_user_host,
-    count(distinct id_booking) as bookings_per_month,
-    sum(cast(is_cancelled as integer)) as cancelled_per_month,
-    sum(cast(is_cancelled as integer))::decimal
-    / count(distinct id_booking) as cancelled_ratio,
-    case
-        when
-            sum(cast(is_cancelled as integer))::decimal / count(distinct id_booking)
-            >= {{ cancellation_threshold }}
-        then sum(cast(is_cancelled as integer)) * {{ cancellation_fee }}
-        else 0
-    end as sum_cancelled_fee,
-    sum(ok_status_fee) as sum_ok_status_fee,
-    sum(rejected_fee) as sum_rejected_fee,
-    case
-        when
-            sum(cast(is_cancelled as integer))::decimal / count(distinct id_booking)
-            >= {{ cancellation_threshold }}
-        then sum(cast(is_cancelled as integer)) * {{ cancellation_fee }}
-        else 0
-    end
-    + sum(ok_status_fee)
-    + sum(rejected_fee) as total_revenue
-from edeposit_records
-group by year_month_created, year_month_checkout, id_user_host

models/intermediate/core/schema.yaml

@@ -2781,233 +2781,4 @@ models:
       - name: has_bookings_with_product_bundle_with_paid_service
         data_type: integer
         description: |
-          Integer-based flag version of total_bookings_with_product_bundle_with_paid_service.
+          Integer-based flag version of total_bookings_with_product_bundle_with_paid_service.
-  
-  - name: int_edeposit__verifications
-    description: 
-      "This table holds records on verifications for e-deposit bookings.
-      It contains details on validations checked on the guests, guest information
-      and some booking details like checkin-checkout date or the status of the verification.
-      The id values found here are completely unrelated to the ones found in Core DWH."
-    columns:
-      - name: id_verification
-        data_type: text
-        description: "unique Superhog generated id for this verification"        
-        tests:
-          - unique
-          - not_null
-
-      - name: id_booking
-        data_type: text
-        description: "unique Superhog generated id for a booking"
-
-      - name: id_user_host
-        data_type: text
-        description: "unique Superhog generated id for host"
-
-      - name: id_accommodation
-        data_type: text
-        description: "unique Superhog generated id for a listing"
-
-      - name: version
-        data_type: text
-        description: 
-          "value to identify if it is Guesty (V1) or E-deposit (V2)"
-        tests:
-          - accepted_values:
-                values:
-                - V1
-                - V2
-
-      - name: verification_source
-        data_type: text
-        description: 
-          "source of the verification for the booking"
-        tests:
-          - accepted_values:
-                values:
-                - Guesty
-                - Edeposit
-
-      - name: verification_status
-        data_type: text
-        description: "status of the verification"
-
-      - name: verification_status_reason
-        data_type: text
-        description: "short explanation for status"
-
-      - name: nightly_fee_local
-        data_type: text
-        description: "fee charged per night"
-
-      - name: number_nights
-        data_type: integer
-        description: "number of nights for the booking"
-
-      - name: email_flag
-        data_type: text
-        description: ""
-
-      - name: phone_flag
-        data_type: text
-        description: ""
-
-      - name: watch_list
-        data_type: text
-        description: ""
-
-      - name: channel
-        data_type: text
-        description: ""
-
-      - name: checkin_at_utc
-        data_type: timestamp without time zone
-        description: "Timestamp of checkin for the booking"
-
-      - name: checkout_at_utc
-        data_type: timestamp without time zone
-        description: "Timestamp of checkout for the booking"
-
-      - name: is_cancelled
-        data_type: boolean
-        description: ""
-
-      - name: cancelled_at_utc
-        data_type: timestamp without time zone
-        description: "Timestamp of cancellation of the booking"
-
-      - name: user_email
-        data_type: text
-        description: ""
-
-      - name: guest_email
-        data_type: text
-        description: ""
-
-      - name: guest_last_name
-        data_type: text
-        description: ""
-
-      - name: guest_first_name
-        data_type: text
-        description: ""
-
-      - name: guest_telephone
-        data_type: text
-        description: ""
-
-      - name: company_name
-        data_type: text
-        description: ""
-
-      - name: property_manager_name
-        data_type: text
-        description: ""
-
-      - name: property_manager_email
-        data_type: text
-        description: ""
-
-      - name: listing_name
-        data_type: text
-        description: ""
-
-      - name: listing_town
-        data_type: text
-        description: ""
-
-      - name: listing_country
-        data_type: text
-        description: ""
-
-      - name: listing_postcode
-        data_type: text
-        description: ""
-
-      - name: pets_allowed
-        data_type: boolean
-        description: ""
-
-      - name: level_of_protection_amount
-        data_type: integer
-        description: ""
-
-      - name: level_of_protection_currency
-        data_type: text
-        description: ""
-
-      - name: status_updated_at_utc
-        data_type: timestamp without time zone
-        description: "Timestamp when status was last updated"
-
-      - name: updated_at_utc
-        data_type: timestamp without time zone
-        description: "Timestamp of last updated"
-
-      - name: creation_at_utc
-        data_type: timestamp without time zone
-        description: ""
-
-      - name: created_at_utc
-        data_type: timestamp without time zone
-        description: ""
-
-  - name: int_edeposit__agg_fee_per_user
-    description: 
-      "This table holds detailed data on revenue generated through e-deposit verifications.
-      Each record provides insights into booking activities per user, including the number 
-      of bookings, cancellations, and associated fees within specific months. Each record 
-      captures data for bookings created in a particular month along with their corresponding 
-      checkout month, allowing for a comprehensive view of the booking lifecycle and associated 
-      revenues within those periods."
-    tests: 
-      - dbt_utils.unique_combination_of_columns:
-          combination_of_columns:
-            - year_month_created
-            - year_month_checkout
-            - id_user_host
-    columns:
-      - name: year_month_created
-        data_type: text
-        description: "first day of month of created date"
-
-      - name: year_month_checkout
-        data_type: text
-        description: "first day of month of check-out date"
-
-      - name: id_user_host
-        data_type: text
-        description: "unique id value for user host"
-
-      - name: bookings_per_month
-        data_type: bigint
-        description: "total number of bookings"
-
-      - name: cancelled_per_month
-        data_type: bigint
-        description: "number of cancelled bookings"
-
-      - name: cancelled_ratio
-        data_type: numeric
-        description: "ratio of cancelled bookings over total bookings"
-
-      - name: sum_cancelled_fee
-        data_type: numeric
-        description: "sum of fees charged for cancelled bookings
-          (currency-less)"
-
-      - name: sum_ok_status_fee
-        data_type: double precision
-        description: |
-          "sum of fees charged for bookings with status 'Approved' or 'Flagged'
-          (currency-less)"
-
-      - name: sum_rejected_fee
-        data_type: numeric
-        description: "sum of fees charged for rejected bookings
-          (currency-less)"
-
-      - name: total_revenue
-        data_type: double precision
-        description: "total sum of fees charged (currency-less)"

models/intermediate/edeposit/int_edeposit__verifications.sql

@@ -5,16 +5,15 @@ with
 select
     -- note that these ids are not the same as the ones found in Core DWH
     -- they are completely unrelated
-    id as id_verification,
+    id_verification,
     id_booking,
-    id_user as id_user_host,
+    id_user as id_user_partner,
     id_listing as id_accommodation,
     version,  -- V1 for Guesty and V2 for E-deposit
     case
         when version = 'V1' then 'Guesty' when version = 'V2' then 'Edeposit' else null
     end as verification_source,
     verification_status,
-    verification_status_reason,
     nightly_fee_local,
     cast(checkout_at_utc as date) - cast(checkin_at_utc as date) as number_nights,
     email_flag,
@@ -22,9 +21,12 @@ select
     watch_list,
     channel,
     checkin_at_utc,
+    cast(checkin_at_utc as date) as checkin_date_utc,
     checkout_at_utc,
+    cast(checkout_at_utc as date) as checkout_date_utc,
     is_cancelled,
     cancelled_at_utc,
+    cast(cancelled_at_utc as date) as cancelled_date_utc,
     user_email,
     guest_email,
     guest_last_name,
@@ -41,7 +43,11 @@ select
     level_of_protection_amount,
     level_of_protection_currency,
     status_updated_at_utc,
+    cast(status_updated_at_utc as date) as status_updated_date_utc,
     updated_at_utc,
+    cast(updated_at_utc as date) as updated_date_utc,
     creation_at_utc,
-    created_at_utc
+    cast(creation_at_utc as date) as creation_date_utc,
+    created_at_utc,
+    cast(created_at_utc as date) as created_date_utc
 from stg_edeposit__verifications

models/intermediate/edeposit/schema.yaml

@@ -0,0 +1,208 @@
+version: 2
+
+models:
+  - name: int_edeposit__verifications
+    description: 
+      "This table holds records on verifications for e-deposit bookings.
+      It contains details on validations checked on the guests, guest information
+      and some booking details like checkin-checkout date or the status of the verification.
+      The id values found here are completely unrelated to the ones found in Core DWH.
+      
+      Note that id_verifications and booking_id should normally be 1 to 1.
+      Though there are exception, the API will accept a duplicate booking and the users
+      will be charged for it. A duplicate would return a unique id_verification."
+    columns:
+      - name: id_verification
+        data_type: text
+        description: "unique Superhog generated id for this verification"        
+        tests:
+          - unique
+          - not_null
+
+      - name: id_booking
+        data_type: text
+        description: "unique Superhog generated id for a booking"
+
+      - name: id_user_partner
+        data_type: text
+        description: "unique Superhog generated id for partner"
+        tests:
+          - not_null
+
+      - name: id_accommodation
+        data_type: text
+        description: "unique Superhog generated id for a listing"
+
+      - name: version
+        data_type: text
+        description: 
+          "value to identify if it is Guesty (V1) or E-deposit (V2)"
+        tests:
+          - accepted_values:
+                values:
+                - V1
+                - V2
+
+      - name: verification_source
+        data_type: text
+        description: 
+          "source of the verification for the booking"
+        tests:
+          - accepted_values:
+                values:
+                - Guesty
+                - Edeposit
+
+      - name: verification_status
+        data_type: text
+        description: "status of the verification"
+
+      - name: nightly_fee_local
+        data_type: double precision
+        description: "fee charged per night"
+
+      - name: number_nights
+        data_type: integer
+        description: "number of nights for the booking"
+
+      - name: email_flag
+        data_type: text
+        description: "screening result for email"
+
+      - name: phone_flag
+        data_type: text
+        description: "screening result for phone"
+
+      - name: watch_list
+        data_type: text
+        description: "screening result of the guest"
+
+      - name: channel
+        data_type: text
+        description: ""
+
+      - name: checkin_at_utc
+        data_type: timestamp without time zone
+        description: "Timestamp of checkin for the booking"
+
+      - name: checkin_date_utc
+        data_type: date
+        description: "Date of checkin for the booking"
+
+      - name: checkout_at_utc
+        data_type: timestamp without time zone
+        description: "Timestamp of checkout for the booking"
+
+      - name: checkout_date_utc
+        data_type: date
+        description: "Date of checkout for the booking"
+
+      - name: is_cancelled
+        data_type: boolean
+        description: ""
+
+      - name: cancelled_at_utc
+        data_type: timestamp without time zone
+        description: "Timestamp of cancellation of the booking"
+
+      - name: cancelled_date_utc
+        data_type: date
+        description: "Date of cancellation for the booking"
+
+      - name: user_email
+        data_type: text
+        description: ""
+
+      - name: guest_email
+        data_type: text
+        description: ""
+
+      - name: guest_last_name
+        data_type: text
+        description: ""
+
+      - name: guest_first_name
+        data_type: text
+        description: ""
+
+      - name: guest_telephone
+        data_type: text
+        description: ""
+
+      - name: company_name
+        data_type: text
+        description: ""
+
+      - name: property_manager_name
+        data_type: text
+        description: ""
+
+      - name: property_manager_email
+        data_type: text
+        description: ""
+
+      - name: listing_name
+        data_type: text
+        description: ""
+
+      - name: listing_town
+        data_type: text
+        description: ""
+
+      - name: listing_country
+        data_type: text
+        description: ""
+
+      - name: listing_postcode
+        data_type: text
+        description: ""
+
+      - name: pets_allowed
+        data_type: boolean
+        description: ""
+
+      - name: level_of_protection_amount
+        data_type: integer
+        description: ""
+
+      - name: level_of_protection_currency
+        data_type: text
+        description: ""
+
+      - name: status_updated_at_utc
+        data_type: timestamp without time zone
+        description: "Timestamp when status was last updated"
+
+      - name: status_updated_date_utc
+        data_type: date
+        description: "Date of last status update of the verification"
+
+      - name: updated_at_utc
+        data_type: timestamp without time zone
+        description: "Timestamp of last updated of the verification"
+
+      - name: updated_date_utc
+        data_type: date
+        description: "Date of last update of the verification"
+
+      - name: athena_creation_at_utc
+        data_type: timestamp without time zone
+        description: 
+          "Athena timestamp referring to when the booking was created. 
+          It's provided by Guesty, but is not mandatory. 
+          In case of doubt use created_at_utc or created_date_utc fields"
+
+      - name: athena_creation_date_utc
+        data_type: date
+        description:  
+          "Athena date referring to when the booking was created. 
+          It's provided by Guesty, but is not mandatory. 
+          In case of doubt use created_at_utc or created_date_utc fields"
+
+      - name: created_at_utc
+        data_type: timestamp without time zone
+        description: "Timestamp of creation of the verification in the system"
+
+      - name: created_date_utc
+        data_type: date
+        description: "Date of creation of the verification in the system"

models/staging/edeposit/schema.yml

@@ -6,7 +6,7 @@ models:
       "Records of each transaction that happens in the edeposit API. Records are
        mutable and can get updated."
     columns:
-      - name: id
+      - name: id_verification
         data_type: character varying
         description: "Unique id for the specific transaction."
         tests:

models/staging/edeposit/stg_edeposit__verifications.sql

@@ -27,14 +27,16 @@ with
     ),
     stg_edeposit__verifications as (
         select
-            {{ adapter.quote("documents") }} ->> 'id' as id,
+            {{ adapter.quote("documents") }} ->> 'id' as id_verification,
             {{ adapter.quote("documents") }} ->> 'BookingId' as id_booking,
             {{ adapter.quote("documents") }} ->> 'userId' as id_user,
             {{ adapter.quote("documents") }} ->> 'ListingId' as id_listing,
 
             {{ adapter.quote("documents") }} ->> 'Version' as "version",
 
-            {{ adapter.quote("documents") }} ->> 'NightlyFee' as "nightly_fee_local",
+            cast(
+                {{ adapter.quote("documents") }} ->> 'NightlyFee' as decimal(19, 4)
+            ) as "nightly_fee_local",
 
             {{ adapter.quote("documents") }} ->> 'Status' as verification_status,
             {{ adapter.quote("documents") }}