From c89dd02799b45658a66921b61d296d459e790885 Mon Sep 17 00:00:00 2001 From: Joaquin Ossa Date: Fri, 21 Feb 2025 16:25:41 +0100 Subject: [PATCH 01/19] commit wip --- .../int_resolutions__incidents.sql | 86 +++++ models/intermediate/resolutions/schema.yml | 363 ++++++++++++++++++ 2 files changed, 449 insertions(+) create mode 100644 models/intermediate/resolutions/int_resolutions__incidents.sql create mode 100644 models/intermediate/resolutions/schema.yml diff --git a/models/intermediate/resolutions/int_resolutions__incidents.sql b/models/intermediate/resolutions/int_resolutions__incidents.sql new file mode 100644 index 0000000..60d23fe --- /dev/null +++ b/models/intermediate/resolutions/int_resolutions__incidents.sql @@ -0,0 +1,86 @@ +with stg_resolutions__incidents as (select * from {{ ref("stg_resolutions__incidents") }}) + +select + -- Basic Incident Details + id_incident, + id_user, + id_verification, + current_status_name, + is_submission_complete, + current_agent_name, + read_only_for_customers, + status_history_logs, + document_version, + task_execution_logs, + created_at_date, + created_date_date, + updated_at_date, + updated_date_date, + + -- Host Details + id_user_host, + host_user_claim_logs, + host_account_name, + + -- Host Contact Details + host_email, + host_last_name, + host_first_name, + host_phone_code, + host_phone_number, + host_phone_number_with_code, + + -- Guest Details + id_user_guest, + guest_email, + guest_last_name, + guest_first_name, + guest_phone_code, + guest_phone_number, + guest_phone_number_with_code, + + -- Guest Deposit Details + is_guest_deposit_retained, + is_guest_deposit_collected, + deposit_retained_amount_in_txn_currency, + deposit_retained_currency, + + -- Guest Involvements + has_guest_contributed_to_cost, + has_host_taken_preventative_steps, + guest_contribution_amount_in_txn_currency, + guest_contribution_currency, + guest_contacted_about_damage, + guest_contacted_evidence_files, + guest_preventative_steps_details, + + -- Accommodation Details + id_accommodation, + accommodation_url, + accommodation_name, + are_pets_allowed, + + -- Booking Details + check_in_at_utc, + check_in_date_utc, + check_out_at_utc, + check_out_date_utc, + id_booking, + + booking_status, + id_reservation, + booking_details, + number_of_guests, + booking_services, + booking_protection, + booking_platform_used, + booking_platform_reference, + + -- Damage Report + before_damage_evidence, + original_invoice_evidence, + damage_incident_details, + has_confirmed_loss, + damage_report_items, + +from raw_incident diff --git a/models/intermediate/resolutions/schema.yml b/models/intermediate/resolutions/schema.yml new file mode 100644 index 0000000..e896661 --- /dev/null +++ b/models/intermediate/resolutions/schema.yml @@ -0,0 +1,363 @@ +version: 2 + +models: + - name: int_resolutions__incidents + description: | + "Contains all incidents stored by the resolutions team in Cosmos DB. + This includes data about the incident, the user claiming the incident, the booking, + the accommodation, the host and the guest." + columns: + - name: id_incident + data_type: text + description: "Unique identifier for the incident." + data_tests: + - unique + - not_null + + - name: id_user + data_type: text + description: "Id of the user who created the incident." + data_tests: + - not_null + + - name: id_verification + data_type: text + description: "Superhog unique validation ID, coming from other APIs." + + - name: current_status_name + data_type: text + description: "current status of the incident." + data_tests: + - not_null + + - name: is_submission_complete + data_type: boolean + description: "Flag to indicate if the submission is complete." + data_tests: + - not_null + + - name: current_agent_name + data_type: text + description: "Name of the agent who is currently handling the incident." + + - name: read_only_for_customers + data_type: boolean + description: "Flag to indicate if the incident is read only for customers." + + - name: status_history_logs + data_type: text + description: "Log of the status history." + + - name: document_version + data_type: text + description: "Reported document resCentre, Api, manual form" + + - name: task_execution_logs + data_type: text + description: "" + + - name: created_at_date + data_type: timestamp without time zone + description: "Timestamp when the incident was created." + data_tests: + - not_null + + - name: created_date_date + data_type: date + description: "Date when the incident was created." + data_tests: + - not_null + + - name: updated_at_date + data_type: timestamp without time zone + description: "Timestamp when the incident was last updated." + data_tests: + - not_null + + - name: updated_date_date + data_type: date + description: "Date when the incident was last updated." + data_tests: + - not_null + + - name: id_user_host + data_type: text + description: "Unique Id that identifies the host. + Can be from SH or can be external from the client." + data_tests: + - not_null + + - name: host_user_claim_logs + data_type: text + description: "" + + - name: host_account_name + data_type: text + description: "Name of the host account." + data_tests: + - not_null + + - name: host_email + data_type: text + description: "Email of the host." + data_tests: + - not_null + + - name: host_last_name + data_type: text + description: "Last name of the host." + data_tests: + - not_null + + - name: host_first_name + data_type: text + description: "First name of the host." + data_tests: + - not_null + + - name: host_phone_code + data_type: text + description: "Phone code of the host." + + - name: host_phone_number + data_type: text + description: "Phone number of the host." + + - name: host_phone_number_with_code + data_type: text + description: "Phone number of the host with the code." + + - name: id_user_guest + data_type: text + description: "Superhog code that uniquely identifies a single guest." + data_tests: + - not_null + + - name: guest_email + data_type: text + description: "Email of the guest." + data_tests: + - not_null + + - name: guest_last_name + data_type: text + description: "Last name of the guest." + data_tests: + - not_null + + - name: guest_first_name + data_type: text + description: "First name of the guest." + data_tests: + - not_null + + - name: guest_phone_code + data_type: text + description: "Phone code of the guest." + + - name: guest_phone_number + data_type: text + description: "Phone number of the guest." + + - name: guest_phone_number_with_code + data_type: text + description: "Phone number of the guest with the code." + + - name: is_guest_deposit_retained + data_type: boolean + description: "In case a deposit has been collected from the guest, + whether it has been retained or not." + + - name: is_guest_deposit_collected + data_type: boolean + description: "If the host has collected a deposit from the guest." + + - name: deposit_retained_amount_in_txn_currency + data_type: text + description: "Amount of the deposit retained in local currency." + + - name: deposit_retained_currency + data_type: text + description: "Currency of the deposit retained." + + - name: has_guest_contributed_to_cost + data_type: boolean + description: "If the guest has agreed to contribute to the cost when + talking to the host" + + - name: has_host_taken_preventative_steps + data_type: boolean + description: "If the host has taken any action to prevent the damage + or further damage once identified" + + - name: guest_contribution_amount_in_txn_currency + data_type: text + description: "Amount of the guest contribution, in case they did, + in local currency." + + - name: guest_contribution_currency + data_type: text + description: "Currency of the guest contribution." + + - name: guest_contacted_about_damage + data_type: boolean + description: "If the host has contacted the guest about the damage caused" + + - name: guest_contacted_evidence_files + data_type: text + description: "" + + - name: guest_preventative_steps_details + data_type: text + description: "Description of any action to prevent the damage or further + damage once identified" + + - name: id_accommodation + data_type: text + description: "Superhog code that uniquely identifies a single accommodation." + data_tests: + - not_null + + - name: accommodation_url + data_type: text + description: "Accommodation URL from any distribution channel, + preferable direct website." + + - name: accommodation_name + data_type: text + description: "Accommodation name." + + - name: accommodation_address + data_type: text + description: "Address of the accommodation." + + - name: are_pets_allowed + data_type: boolean + description: "Flag to indicate if pets are allowed in the accommodation." + + - name: comment_logs + data_type: text + description: "" + + - name: has_underlying_insurance + data_type: boolean + description: "If the property has underlying insurance" + + - name: insurance_claim_submitted + data_type: boolean + description: "If an insurance claim has been submitted to an insurance company" + + - name: insurance_evidence_files + data_type: text + description: "" + + - name: check_in_at_utc + data_type: timestamp without time zone + description: "Timestamp of the check-in date in UTC of the booking." + data_tests: + - not_null + + - name: check_in_date_utc + data_type: date + description: "Date of the check-in date in UTC of the booking." + data_tests: + - not_null + + - name: check_out_at_utc + data_type: timestamp without time zone + description: "Timestamp of the check-out date in UTC of the booking." + data_tests: + - not_null + + - name: check_out_date_utc + data_type: date + description: "Date of the check-out date in UTC of the booking." + data_tests: + - not_null + + - name: id_booking + data_type: text + description: "Superhog unique booking ID" + data_tests: + - not_null + + - name: booking_status + data_type: text + description: "Status of the booking." + + - name: id_reservation + data_type: text + description: "External unique ID" + + - name: booking_details + data_type: text + description: "" + + - name: number_of_guests + data_type: text + description: "Number of guests in the booking." + + - name: booking_services + data_type: text + description: "List of services included in the booking." + + - name: booking_protection + data_type: text + description: "" + + - name: booking_platform_used + data_type: text + description: "The booking platform used" + + - name: booking_platform_reference + data_type: text + description: "" + + - name: agent_history + data_type: text + description: "" + + - name: before_damage_evidence + data_type: text + description: "" + + - name: original_invoice_evidence + data_type: text + description: "" + + - name: damage_incident_details + data_type: text + description: "Description of what happened in general terms" + + - name: has_confirmed_loss + data_type: boolean + description: "Confirmation that the information they have provided in + the report is true to the best of their knowledge." + + - name: damage_report_items + data_type: text + description: "List of items that were damaged." + + - name: third_party_claim_submitted + data_type: boolean + description: "If the host is seeking compensation from another platform" + + - name: third_party_claim_from + data_type: text + description: "For example, Aircover from Airbnb." + + - name: third_party_claim_amount_in_txn_currency + data_type: text + description: "Claim amount in local currency if the host is seeking + compensation from another platform." + + - name: third_party_claim_currency + data_type: text + description: "Currency of the claim amount if the host is seeking + compensation from another platform." + + - name: cosmos_db_timestamp_utc + data_type: timestamp + description: Internal Cosmos DB timestamp of the last record update. + data_tests: + - not_null From 7551f6e5159d861f56e64ebd346c9e7245f36b04 Mon Sep 17 00:00:00 2001 From: Joaquin Ossa Date: Fri, 21 Feb 2025 16:26:07 +0100 Subject: [PATCH 02/19] commit wip --- .../intermediate/resolutions/int_resolutions__incidents.sql | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/models/intermediate/resolutions/int_resolutions__incidents.sql b/models/intermediate/resolutions/int_resolutions__incidents.sql index 60d23fe..5555c05 100644 --- a/models/intermediate/resolutions/int_resolutions__incidents.sql +++ b/models/intermediate/resolutions/int_resolutions__incidents.sql @@ -1,4 +1,7 @@ -with stg_resolutions__incidents as (select * from {{ ref("stg_resolutions__incidents") }}) +with + stg_resolutions__incidents as ( + select * from {{ ref("stg_resolutions__incidents") }} + ) select -- Basic Incident Details From 47d9b6f292ce471a29b9f87445a437eca4b9b4ce Mon Sep 17 00:00:00 2001 From: Joaquin Ossa Date: Fri, 21 Feb 2025 18:11:53 +0100 Subject: [PATCH 03/19] commit wip --- .../int_resolutions__incidents.sql | 41 +++++++++++++++++-- 1 file changed, 38 insertions(+), 3 deletions(-) diff --git a/models/intermediate/resolutions/int_resolutions__incidents.sql b/models/intermediate/resolutions/int_resolutions__incidents.sql index 5555c05..99912bf 100644 --- a/models/intermediate/resolutions/int_resolutions__incidents.sql +++ b/models/intermediate/resolutions/int_resolutions__incidents.sql @@ -1,11 +1,28 @@ with stg_resolutions__incidents as ( select * from {{ ref("stg_resolutions__incidents") }} + ), + comments_timeline as ( + select + i.id_incident, + ( + select (comment ->> 'CreatedDate')::timestamp + from jsonb_array_elements(i.comment_logs::jsonb) comment + order by (comment ->> 'CreatedDate')::timestamp + limit 1 + ) as first_comment_date, + ( + select (comment ->> 'CreatedDate')::timestamp + from jsonb_array_elements(i.comment_logs::jsonb) comment + order by (comment ->> 'CreatedDate')::timestamp desc + limit 1 + ) as last_comment_date + from stg_resolutions__incidents i ) select -- Basic Incident Details - id_incident, + i.id_incident, id_user, id_verification, current_status_name, @@ -20,6 +37,10 @@ select updated_at_date, updated_date_date, + -- Resolution Details + first_comment_date, + last_comment_date, + -- Host Details id_user_host, host_user_claim_logs, @@ -69,7 +90,6 @@ select check_out_at_utc, check_out_date_utc, id_booking, - booking_status, id_reservation, booking_details, @@ -86,4 +106,19 @@ select has_confirmed_loss, damage_report_items, -from raw_incident + -- Calculator + protection_name, + was_overriden, + waiver_or_deposit_name, + guest_verification_status, + lower_protection_limit_usd, + upper_protection_limit_usd, + calculated_payout_amount_in_txn_currency, + calculated_payout_currency, + calculated_payout_amount_in_usd, + calculated_guest_charge_amount_in_txn_currency, + calculated_guest_charge_currency, + calculated_guest_charge_amount_in_usd + +from stg_resolutions__incidents i +left join comments_timeline ct on i.id_incident = ct.id_incident From 29755961e6a98628fde9f07d2dc1663248ec8fd6 Mon Sep 17 00:00:00 2001 From: Joaquin Ossa Date: Fri, 21 Feb 2025 19:16:28 +0100 Subject: [PATCH 04/19] commit wip --- .../int_resolutions__incidents.sql | 30 +++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/models/intermediate/resolutions/int_resolutions__incidents.sql b/models/intermediate/resolutions/int_resolutions__incidents.sql index 99912bf..e6d5c62 100644 --- a/models/intermediate/resolutions/int_resolutions__incidents.sql +++ b/models/intermediate/resolutions/int_resolutions__incidents.sql @@ -2,7 +2,7 @@ with stg_resolutions__incidents as ( select * from {{ ref("stg_resolutions__incidents") }} ), - comments_timeline as ( + logs_timeline as ( select i.id_incident, ( @@ -16,8 +16,34 @@ with from jsonb_array_elements(i.comment_logs::jsonb) comment order by (comment ->> 'CreatedDate')::timestamp desc limit 1 - ) as last_comment_date + ) as last_comment_date, + ( + select count(*) from jsonb_array_elements(i.comment_logs::jsonb) comment + ) as comment_count, + ( + select count(*) + from jsonb_array_elements(i.damage_report_items::jsonb) items + ) as number_damaged_items, + from stg_resolutions__incidents i + ), + damage_reported_items as ( + select + i.id_incident, + jsonb_array_elements(i.documents -> 'DamageReport' -> 'Items') as item + from stg_resolutions__incidents i + ) + damage_report_amounts as ( + select + id_incident, + sum((corrective_measures ->> 'Amount')::numeric) as asked_repair_amount, + sum( + (corrective_measures ->> 'SettlementAmount')::numeric + ) as accepted_amount, + from + parsed_data, + jsonb_array_elements(item -> 'CorrectiveMeasures') as corrective_measures + group by id_incident ) select From 9fa62e0bf256d832c8f97a10dc03a28c23a4a8ce Mon Sep 17 00:00:00 2001 From: Joaquin Ossa Date: Fri, 21 Feb 2025 19:59:30 +0100 Subject: [PATCH 05/19] commit wip --- .../int_resolutions__incidents.sql | 72 ++++++++++++------- 1 file changed, 48 insertions(+), 24 deletions(-) diff --git a/models/intermediate/resolutions/int_resolutions__incidents.sql b/models/intermediate/resolutions/int_resolutions__incidents.sql index e6d5c62..240953a 100644 --- a/models/intermediate/resolutions/int_resolutions__incidents.sql +++ b/models/intermediate/resolutions/int_resolutions__incidents.sql @@ -4,44 +4,52 @@ with ), logs_timeline as ( select - i.id_incident, + id_incident, ( select (comment ->> 'CreatedDate')::timestamp - from jsonb_array_elements(i.comment_logs::jsonb) comment + from jsonb_array_elements(comment_logs::jsonb) comment order by (comment ->> 'CreatedDate')::timestamp limit 1 - ) as first_comment_date, + ) as first_comment_at_utc, ( select (comment ->> 'CreatedDate')::timestamp - from jsonb_array_elements(i.comment_logs::jsonb) comment + from jsonb_array_elements(comment_logs::jsonb) comment order by (comment ->> 'CreatedDate')::timestamp desc limit 1 - ) as last_comment_date, + ) as last_comment_at_utc, ( - select count(*) from jsonb_array_elements(i.comment_logs::jsonb) comment + select count(*) from jsonb_array_elements(comment_logs::jsonb) comment ) as comment_count, ( - select count(*) - from jsonb_array_elements(i.damage_report_items::jsonb) items - ) as number_damaged_items, - - from stg_resolutions__incidents i + select (status_step ->> 'CreatedDate')::timestamp + from jsonb_array_elements(status_history_logs::jsonb) status_step + order by (status_step ->> 'CreatedDate')::timestamp + limit 1 + ) as first_status_at_utc, + ( + select (status_step ->> 'CreatedDate')::timestamp + from jsonb_array_elements(status_history_logs::jsonb) status_step + order by (status_step ->> 'CreatedDate')::timestamp desc + limit 1 + ) as last_status_at_utc + from stg_resolutions__incidents ), damage_reported_items as ( - select - i.id_incident, - jsonb_array_elements(i.documents -> 'DamageReport' -> 'Items') as item - from stg_resolutions__incidents i - ) - damage_report_amounts as ( + select id_incident, jsonb_array_elements(damage_report_items::jsonb) as item -- No need for 'Items' if already an array + from stg_resolutions__incidents + ), + damage_report as ( select id_incident, - sum((corrective_measures ->> 'Amount')::numeric) as asked_repair_amount, + count(distinct item ->> 'Id') as number_damaged_items, -- Counting unique damaged items sum( - (corrective_measures ->> 'SettlementAmount')::numeric - ) as accepted_amount, + coalesce((corrective_measures ->> 'Amount')::numeric, 0) + ) as asked_repair_amount, + sum( + coalesce((corrective_measures ->> 'SettlementAmount')::numeric, 0) + ) as accepted_amount from - parsed_data, + damage_reported_items, jsonb_array_elements(item -> 'CorrectiveMeasures') as corrective_measures group by id_incident ) @@ -64,8 +72,23 @@ select updated_date_date, -- Resolution Details - first_comment_date, - last_comment_date, + lt.first_comment_at_utc, + lt.last_comment_at_utc, + lt.comment_count, + lt.first_status_at_utc, + lt.last_status_at_utc, + case + when is_submission_complete + then + ( + extract(epoch from (lt.last_status_at_utc - lt.first_status_at_utc)) + / 3600 + )::decimal(19, 4) + else null + end as resolution_time_in_hours, + dr.number_damaged_items, + dr.asked_repair_amount, + dr.accepted_amount, -- Host Details id_user_host, @@ -147,4 +170,5 @@ select calculated_guest_charge_amount_in_usd from stg_resolutions__incidents i -left join comments_timeline ct on i.id_incident = ct.id_incident +left join logs_timeline lt on i.id_incident = lt.id_incident +left join damage_report dr on i.id_incident = dr.id_incident From 3f224d8fed65db4d0cd5e16a7c44acba854ed98b Mon Sep 17 00:00:00 2001 From: Joaquin Ossa Date: Sat, 22 Feb 2025 12:42:20 +0100 Subject: [PATCH 06/19] commit wip --- .../int_resolutions__incidents.sql | 137 ++++----- models/intermediate/resolutions/schema.yml | 282 ++++++------------ 2 files changed, 154 insertions(+), 265 deletions(-) diff --git a/models/intermediate/resolutions/int_resolutions__incidents.sql b/models/intermediate/resolutions/int_resolutions__incidents.sql index 240953a..7698e61 100644 --- a/models/intermediate/resolutions/int_resolutions__incidents.sql +++ b/models/intermediate/resolutions/int_resolutions__incidents.sql @@ -31,7 +31,11 @@ with from jsonb_array_elements(status_history_logs::jsonb) status_step order by (status_step ->> 'CreatedDate')::timestamp desc limit 1 - ) as last_status_at_utc + ) as last_status_at_utc, + ( + select count(*) + from jsonb_array_elements(host_user_claim_logs::jsonb) comment + ) as claims_count from stg_resolutions__incidents ), damage_reported_items as ( @@ -41,13 +45,16 @@ with damage_report as ( select id_incident, - count(distinct item ->> 'Id') as number_damaged_items, -- Counting unique damaged items + count(distinct item ->> 'Id') as number_damaged_items, + sum( + coalesce((corrective_measures ->> 'OriginalAmount')::numeric, 0) + ) as original_value_amount_in_txn_currency, sum( coalesce((corrective_measures ->> 'Amount')::numeric, 0) - ) as asked_repair_amount, + ) as asked_repair_amount_in_txn_currency, sum( coalesce((corrective_measures ->> 'SettlementAmount')::numeric, 0) - ) as accepted_amount + ) as accepted_amount_in_txn_currency from damage_reported_items, jsonb_array_elements(item -> 'CorrectiveMeasures') as corrective_measures @@ -62,10 +69,7 @@ select current_status_name, is_submission_complete, current_agent_name, - read_only_for_customers, - status_history_logs, document_version, - task_execution_logs, created_at_date, created_date_date, updated_at_date, @@ -87,75 +91,11 @@ select else null end as resolution_time_in_hours, dr.number_damaged_items, - dr.asked_repair_amount, - dr.accepted_amount, + dr.original_value_amount_in_txn_currency, + dr.asked_repair_amount_in_txn_currency, + dr.accepted_amount_in_txn_currency, - -- Host Details - id_user_host, - host_user_claim_logs, - host_account_name, - - -- Host Contact Details - host_email, - host_last_name, - host_first_name, - host_phone_code, - host_phone_number, - host_phone_number_with_code, - - -- Guest Details - id_user_guest, - guest_email, - guest_last_name, - guest_first_name, - guest_phone_code, - guest_phone_number, - guest_phone_number_with_code, - - -- Guest Deposit Details - is_guest_deposit_retained, - is_guest_deposit_collected, - deposit_retained_amount_in_txn_currency, - deposit_retained_currency, - - -- Guest Involvements - has_guest_contributed_to_cost, - has_host_taken_preventative_steps, - guest_contribution_amount_in_txn_currency, - guest_contribution_currency, - guest_contacted_about_damage, - guest_contacted_evidence_files, - guest_preventative_steps_details, - - -- Accommodation Details - id_accommodation, - accommodation_url, - accommodation_name, - are_pets_allowed, - - -- Booking Details - check_in_at_utc, - check_in_date_utc, - check_out_at_utc, - check_out_date_utc, - id_booking, - booking_status, - id_reservation, - booking_details, - number_of_guests, - booking_services, - booking_protection, - booking_platform_used, - booking_platform_reference, - - -- Damage Report - before_damage_evidence, - original_invoice_evidence, - damage_incident_details, - has_confirmed_loss, - damage_report_items, - - -- Calculator + -- Resolutions calculator protection_name, was_overriden, waiver_or_deposit_name, @@ -169,6 +109,53 @@ select calculated_guest_charge_currency, calculated_guest_charge_amount_in_usd + -- Host Details + id_user_host, + lt.claims_count, + host_account_name, + + -- Host Contact Details + host_email, + host_last_name, + host_first_name, + host_phone_code, + host_phone_number, + + -- Guest Details + id_user_guest, + guest_email, + guest_last_name, + guest_first_name, + guest_phone_code, + guest_phone_number, + + -- Guest Deposit Details + is_guest_deposit_retained, + is_guest_deposit_collected, + deposit_retained_amount_in_txn_currency, + deposit_retained_currency, + + -- Guest Involvements + has_guest_contributed_to_cost, + has_host_taken_preventative_steps, + guest_contribution_amount_in_txn_currency, + guest_contribution_currency, + guest_contacted_about_damage, + + -- Accommodation Details + id_accommodation, + accommodation_name, + + -- Booking Details + check_in_at_utc, + check_in_date_utc, + check_out_at_utc, + check_out_date_utc, + id_booking, + booking_status, + id_reservation, + booking_protection + from stg_resolutions__incidents i left join logs_timeline lt on i.id_incident = lt.id_incident left join damage_report dr on i.id_incident = dr.id_incident diff --git a/models/intermediate/resolutions/schema.yml b/models/intermediate/resolutions/schema.yml index e896661..b804a12 100644 --- a/models/intermediate/resolutions/schema.yml +++ b/models/intermediate/resolutions/schema.yml @@ -2,23 +2,15 @@ version: 2 models: - name: int_resolutions__incidents - description: | - "Contains all incidents stored by the resolutions team in Cosmos DB. - This includes data about the incident, the user claiming the incident, the booking, - the accommodation, the host and the guest." + description: "" columns: - name: id_incident data_type: text description: "Unique identifier for the incident." - data_tests: - - unique - - not_null - name: id_user data_type: text description: "Id of the user who created the incident." - data_tests: - - not_null - name: id_verification data_type: text @@ -27,93 +19,142 @@ models: - name: current_status_name data_type: text description: "current status of the incident." - data_tests: - - not_null - name: is_submission_complete data_type: boolean description: "Flag to indicate if the submission is complete." - data_tests: - - not_null - name: current_agent_name data_type: text description: "Name of the agent who is currently handling the incident." - - name: read_only_for_customers - data_type: boolean - description: "Flag to indicate if the incident is read only for customers." - - - name: status_history_logs - data_type: text - description: "Log of the status history." - - name: document_version data_type: text description: "Reported document resCentre, Api, manual form" - - name: task_execution_logs - data_type: text - description: "" - - name: created_at_date data_type: timestamp without time zone description: "Timestamp when the incident was created." - data_tests: - - not_null - name: created_date_date data_type: date description: "Date when the incident was created." - data_tests: - - not_null - name: updated_at_date data_type: timestamp without time zone description: "Timestamp when the incident was last updated." - data_tests: - - not_null - name: updated_date_date data_type: date description: "Date when the incident was last updated." - data_tests: - - not_null + + - name: first_comment_at_utc + data_type: timestamp without time zone + description: "" + + - name: last_comment_at_utc + data_type: timestamp without time zone + description: "" + + - name: comment_count + data_type: bigint + description: "" + + - name: first_status_at_utc + data_type: timestamp without time zone + description: "" + + - name: last_status_at_utc + data_type: timestamp without time zone + description: "" + + - name: resolution_time_in_hours + data_type: numeric + description: "" + + - name: number_damaged_items + data_type: bigint + description: "" + + - name: original_value_amount_in_txn_currency + data_type: numeric + description: "" + + - name: asked_repair_amount_in_txn_currency + data_type: numeric + description: "" + + - name: accepted_amount_in_txn_currency + data_type: numeric + description: "" + + - name: protection_name + data_type: text + description: "Protection name." + + - name: was_overriden + data_type: boolean + description: "Flag to indicate if the proposed settlement value from the calculator was overriden." + + - name: waiver_or_deposit_name + data_type: text + description: "Waiver or deposit name." + + - name: guest_verification_status + data_type: text + description: "Guest verification status for this booking." + + - name: lower_protection_limit_usd + data_type: text + description: "Lower protection limit in USD." + + - name: upper_protection_limit_usd + data_type: text + description: "Upper protection limit in USD." + + - name: calculated_payout_amount_in_txn_currency + data_type: text + description: "Calculated payout amount in local currency." + + - name: calculated_payout_currency + data_type: text + description: "Currency of the calculated payout amount." + + - name: calculated_payout_amount_in_usd + data_type: text + description: "Calculated payout amount in USD." + + - name: calculated_guest_charge_amount_in_txn_currency + data_type: text + description: "Calculated guest charge amount in local currency." + + - name: calculated_guest_charge_currency + data_type: text + description: "Currency of the calculated guest charge amount." - name: id_user_host data_type: text - description: "Unique Id that identifies the host. - Can be from SH or can be external from the client." - data_tests: - - not_null + description: "Unique Id that identifies the host. Can be from SH or can be external from the client." - - name: host_user_claim_logs - data_type: text + - name: claims_count + data_type: bigint description: "" - name: host_account_name data_type: text description: "Name of the host account." - data_tests: - - not_null - name: host_email data_type: text description: "Email of the host." - data_tests: - - not_null - name: host_last_name data_type: text description: "Last name of the host." - data_tests: - - not_null - name: host_first_name data_type: text description: "First name of the host." - data_tests: - - not_null - name: host_phone_code data_type: text @@ -123,33 +164,21 @@ models: data_type: text description: "Phone number of the host." - - name: host_phone_number_with_code - data_type: text - description: "Phone number of the host with the code." - - name: id_user_guest data_type: text description: "Superhog code that uniquely identifies a single guest." - data_tests: - - not_null - name: guest_email data_type: text description: "Email of the guest." - data_tests: - - not_null - name: guest_last_name data_type: text description: "Last name of the guest." - data_tests: - - not_null - name: guest_first_name data_type: text description: "First name of the guest." - data_tests: - - not_null - name: guest_phone_code data_type: text @@ -159,14 +188,9 @@ models: data_type: text description: "Phone number of the guest." - - name: guest_phone_number_with_code - data_type: text - description: "Phone number of the guest with the code." - - name: is_guest_deposit_retained data_type: boolean - description: "In case a deposit has been collected from the guest, - whether it has been retained or not." + description: "In case a deposit has been collected from the guest, whether it has been retained or not." - name: is_guest_deposit_collected data_type: boolean @@ -182,18 +206,15 @@ models: - name: has_guest_contributed_to_cost data_type: boolean - description: "If the guest has agreed to contribute to the cost when - talking to the host" + description: "If the guest has agreed to contribute to the cost when talking to the host" - name: has_host_taken_preventative_steps data_type: boolean - description: "If the host has taken any action to prevent the damage - or further damage once identified" + description: "If the host has taken any action to prevent the damage or further damage once identified" - name: guest_contribution_amount_in_txn_currency data_type: text - description: "Amount of the guest contribution, in case they did, - in local currency." + description: "Amount of the guest contribution, in case they did, in local currency." - name: guest_contribution_currency data_type: text @@ -203,83 +224,33 @@ models: data_type: boolean description: "If the host has contacted the guest about the damage caused" - - name: guest_contacted_evidence_files - data_type: text - description: "" - - - name: guest_preventative_steps_details - data_type: text - description: "Description of any action to prevent the damage or further - damage once identified" - - name: id_accommodation data_type: text description: "Superhog code that uniquely identifies a single accommodation." - data_tests: - - not_null - - - name: accommodation_url - data_type: text - description: "Accommodation URL from any distribution channel, - preferable direct website." - name: accommodation_name data_type: text description: "Accommodation name." - - name: accommodation_address - data_type: text - description: "Address of the accommodation." - - - name: are_pets_allowed - data_type: boolean - description: "Flag to indicate if pets are allowed in the accommodation." - - - name: comment_logs - data_type: text - description: "" - - - name: has_underlying_insurance - data_type: boolean - description: "If the property has underlying insurance" - - - name: insurance_claim_submitted - data_type: boolean - description: "If an insurance claim has been submitted to an insurance company" - - - name: insurance_evidence_files - data_type: text - description: "" - - name: check_in_at_utc data_type: timestamp without time zone description: "Timestamp of the check-in date in UTC of the booking." - data_tests: - - not_null - name: check_in_date_utc data_type: date description: "Date of the check-in date in UTC of the booking." - data_tests: - - not_null - name: check_out_at_utc data_type: timestamp without time zone description: "Timestamp of the check-out date in UTC of the booking." - data_tests: - - not_null - name: check_out_date_utc data_type: date description: "Date of the check-out date in UTC of the booking." - data_tests: - - not_null - name: id_booking data_type: text description: "Superhog unique booking ID" - data_tests: - - not_null - name: booking_status data_type: text @@ -289,75 +260,6 @@ models: data_type: text description: "External unique ID" - - name: booking_details - data_type: text - description: "" - - - name: number_of_guests - data_type: text - description: "Number of guests in the booking." - - - name: booking_services - data_type: text - description: "List of services included in the booking." - - name: booking_protection data_type: text description: "" - - - name: booking_platform_used - data_type: text - description: "The booking platform used" - - - name: booking_platform_reference - data_type: text - description: "" - - - name: agent_history - data_type: text - description: "" - - - name: before_damage_evidence - data_type: text - description: "" - - - name: original_invoice_evidence - data_type: text - description: "" - - - name: damage_incident_details - data_type: text - description: "Description of what happened in general terms" - - - name: has_confirmed_loss - data_type: boolean - description: "Confirmation that the information they have provided in - the report is true to the best of their knowledge." - - - name: damage_report_items - data_type: text - description: "List of items that were damaged." - - - name: third_party_claim_submitted - data_type: boolean - description: "If the host is seeking compensation from another platform" - - - name: third_party_claim_from - data_type: text - description: "For example, Aircover from Airbnb." - - - name: third_party_claim_amount_in_txn_currency - data_type: text - description: "Claim amount in local currency if the host is seeking - compensation from another platform." - - - name: third_party_claim_currency - data_type: text - description: "Currency of the claim amount if the host is seeking - compensation from another platform." - - - name: cosmos_db_timestamp_utc - data_type: timestamp - description: Internal Cosmos DB timestamp of the last record update. - data_tests: - - not_null From 2f484054dfc0bef31ab5fbdef306ada6b34fe9b4 Mon Sep 17 00:00:00 2001 From: Joaquin Ossa Date: Sun, 23 Feb 2025 14:50:51 +0100 Subject: [PATCH 07/19] commit wip --- .../int_resolutions__incidents.sql | 26 ++++++++++++------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/models/intermediate/resolutions/int_resolutions__incidents.sql b/models/intermediate/resolutions/int_resolutions__incidents.sql index 7698e61..3cb4eb2 100644 --- a/models/intermediate/resolutions/int_resolutions__incidents.sql +++ b/models/intermediate/resolutions/int_resolutions__incidents.sql @@ -41,24 +41,32 @@ with damage_reported_items as ( select id_incident, jsonb_array_elements(damage_report_items::jsonb) as item -- No need for 'Items' if already an array from stg_resolutions__incidents + where damage_report_items is not null ), damage_report as ( select - id_incident, - count(distinct item ->> 'Id') as number_damaged_items, + dri.id_incident, + count(distinct dri.item ->> 'Id') as number_damaged_items, sum( - coalesce((corrective_measures ->> 'OriginalAmount')::numeric, 0) + coalesce((cm.corrective_measure ->> 'OriginalAmount')::numeric, 0) ) as original_value_amount_in_txn_currency, sum( - coalesce((corrective_measures ->> 'Amount')::numeric, 0) + coalesce((cm.corrective_measure ->> 'Amount')::numeric, 0) ) as asked_repair_amount_in_txn_currency, sum( - coalesce((corrective_measures ->> 'SettlementAmount')::numeric, 0) + coalesce((cm.corrective_measure ->> 'SettlementAmount')::numeric, 0) ) as accepted_amount_in_txn_currency - from - damage_reported_items, - jsonb_array_elements(item -> 'CorrectiveMeasures') as corrective_measures - group by id_incident + from damage_reported_items dri + left join + lateral jsonb_array_elements( + case + when jsonb_typeof(dri.item -> 'CorrectiveMeasures') = 'array' + then dri.item -> 'CorrectiveMeasures' + else '[]'::jsonb + end + ) as cm(corrective_measure) + on true + group by dri.id_incident ) select From 76a247a55f48e4b115dd78e536398b6aba53aeda Mon Sep 17 00:00:00 2001 From: Joaquin Ossa Date: Sun, 23 Feb 2025 15:03:37 +0100 Subject: [PATCH 08/19] commit wip --- models/intermediate/resolutions/schema.yml | 220 +++++---------------- 1 file changed, 45 insertions(+), 175 deletions(-) diff --git a/models/intermediate/resolutions/schema.yml b/models/intermediate/resolutions/schema.yml index b804a12..72785d3 100644 --- a/models/intermediate/resolutions/schema.yml +++ b/models/intermediate/resolutions/schema.yml @@ -2,15 +2,25 @@ version: 2 models: - name: int_resolutions__incidents - description: "" + description: | + "This table contains all incidents recorded by the resolutions team in Cosmos DB. + It includes details about the incident, the user who reported it, the associated booking, + the accommodation, the host, and the guest. Additionally, it tracks the resolution process, + including the compensation amount requested by the host, the proposed settlement calculated + by the system, and the final agreed-upon settlement amount." columns: - name: id_incident data_type: text description: "Unique identifier for the incident." + tests: + - not_null + - unique - name: id_user data_type: text description: "Id of the user who created the incident." + tests: + - not_null - name: id_verification data_type: text @@ -18,7 +28,9 @@ models: - name: current_status_name data_type: text - description: "current status of the incident." + description: "Current status of the incident." + tests: + - not_null - name: is_submission_complete data_type: boolean @@ -30,227 +42,85 @@ models: - name: document_version data_type: text - description: "Reported document resCentre, Api, manual form" + description: | + "Reported document source, such as ResCentre, API, or manual form." - name: created_at_date data_type: timestamp without time zone description: "Timestamp when the incident was created." - - - name: created_date_date - data_type: date - description: "Date when the incident was created." + tests: + - not_null - name: updated_at_date data_type: timestamp without time zone description: "Timestamp when the incident was last updated." - - name: updated_date_date - data_type: date - description: "Date when the incident was last updated." - - - name: first_comment_at_utc - data_type: timestamp without time zone - description: "" - - - name: last_comment_at_utc - data_type: timestamp without time zone - description: "" - - - name: comment_count - data_type: bigint - description: "" - - - name: first_status_at_utc - data_type: timestamp without time zone - description: "" - - - name: last_status_at_utc - data_type: timestamp without time zone - description: "" - - name: resolution_time_in_hours data_type: numeric - description: "" + description: "Time taken to resolve the incident, in hours." - name: number_damaged_items data_type: bigint - description: "" + description: "Number of items reported as damaged in the incident." - name: original_value_amount_in_txn_currency data_type: numeric - description: "" + description: | + "Original value of the damaged items in the transaction currency." - name: asked_repair_amount_in_txn_currency data_type: numeric - description: "" + description: | + "Amount requested by the host for repair in the transaction currency." - name: accepted_amount_in_txn_currency data_type: numeric - description: "" + description: | + "Final accepted settlement amount in the transaction currency." - name: protection_name data_type: text - description: "Protection name." + description: "Name of the protection plan covering the booking." - name: was_overriden data_type: boolean - description: "Flag to indicate if the proposed settlement value from the calculator was overriden." + description: | + "Flag to indicate if the proposed settlement value from the + calculator was overridden." - name: waiver_or_deposit_name data_type: text - description: "Waiver or deposit name." + description: "Waiver or deposit type associated with the booking." - name: guest_verification_status data_type: text description: "Guest verification status for this booking." - - name: lower_protection_limit_usd - data_type: text - description: "Lower protection limit in USD." - - - name: upper_protection_limit_usd - data_type: text - description: "Upper protection limit in USD." - - - name: calculated_payout_amount_in_txn_currency - data_type: text - description: "Calculated payout amount in local currency." - - - name: calculated_payout_currency - data_type: text - description: "Currency of the calculated payout amount." - - - name: calculated_payout_amount_in_usd - data_type: text - description: "Calculated payout amount in USD." - - - name: calculated_guest_charge_amount_in_txn_currency - data_type: text - description: "Calculated guest charge amount in local currency." - - - name: calculated_guest_charge_currency - data_type: text - description: "Currency of the calculated guest charge amount." - - name: id_user_host data_type: text - description: "Unique Id that identifies the host. Can be from SH or can be external from the client." - - - name: claims_count - data_type: bigint - description: "" - - - name: host_account_name - data_type: text - description: "Name of the host account." - - - name: host_email - data_type: text - description: "Email of the host." - - - name: host_last_name - data_type: text - description: "Last name of the host." - - - name: host_first_name - data_type: text - description: "First name of the host." - - - name: host_phone_code - data_type: text - description: "Phone code of the host." - - - name: host_phone_number - data_type: text - description: "Phone number of the host." + description: | + "Unique Id that identifies the host. Can be from SH or can + be external from the client." + data_tests: + - not_null + - relationships: + to: ref('stg_core__user') + field: id_user + where: "document_version <> 'OldClaimToIncidentModel'" - name: id_user_guest data_type: text description: "Superhog code that uniquely identifies a single guest." - - name: guest_email - data_type: text - description: "Email of the guest." - - - name: guest_last_name - data_type: text - description: "Last name of the guest." - - - name: guest_first_name - data_type: text - description: "First name of the guest." - - - name: guest_phone_code - data_type: text - description: "Phone code of the guest." - - - name: guest_phone_number - data_type: text - description: "Phone number of the guest." - - - name: is_guest_deposit_retained - data_type: boolean - description: "In case a deposit has been collected from the guest, whether it has been retained or not." - - - name: is_guest_deposit_collected - data_type: boolean - description: "If the host has collected a deposit from the guest." - - - name: deposit_retained_amount_in_txn_currency - data_type: text - description: "Amount of the deposit retained in local currency." - - - name: deposit_retained_currency - data_type: text - description: "Currency of the deposit retained." - - - name: has_guest_contributed_to_cost - data_type: boolean - description: "If the guest has agreed to contribute to the cost when talking to the host" - - - name: has_host_taken_preventative_steps - data_type: boolean - description: "If the host has taken any action to prevent the damage or further damage once identified" - - - name: guest_contribution_amount_in_txn_currency - data_type: text - description: "Amount of the guest contribution, in case they did, in local currency." - - - name: guest_contribution_currency - data_type: text - description: "Currency of the guest contribution." - - - name: guest_contacted_about_damage - data_type: boolean - description: "If the host has contacted the guest about the damage caused" - - name: id_accommodation data_type: text description: "Superhog code that uniquely identifies a single accommodation." - - name: accommodation_name - data_type: text - description: "Accommodation name." - - - name: check_in_at_utc - data_type: timestamp without time zone - description: "Timestamp of the check-in date in UTC of the booking." - - - name: check_in_date_utc - data_type: date - description: "Date of the check-in date in UTC of the booking." - - - name: check_out_at_utc - data_type: timestamp without time zone - description: "Timestamp of the check-out date in UTC of the booking." - - - name: check_out_date_utc - data_type: date - description: "Date of the check-out date in UTC of the booking." - - name: id_booking data_type: text - description: "Superhog unique booking ID" + description: "Superhog unique booking ID." + tests: + - not_null - name: booking_status data_type: text @@ -258,8 +128,8 @@ models: - name: id_reservation data_type: text - description: "External unique ID" + description: "External unique ID for the reservation." - name: booking_protection data_type: text - description: "" + description: "Details on the booking protection applied." From 56619982d24c951575cc4062aef68f33bfd42b74 Mon Sep 17 00:00:00 2001 From: Joaquin Ossa Date: Sun, 23 Feb 2025 15:39:07 +0100 Subject: [PATCH 09/19] commit wip on schema --- .../int_resolutions__incidents.sql | 8 +- models/intermediate/resolutions/schema.yml | 198 +++++++++++++++++- 2 files changed, 192 insertions(+), 14 deletions(-) diff --git a/models/intermediate/resolutions/int_resolutions__incidents.sql b/models/intermediate/resolutions/int_resolutions__incidents.sql index 3cb4eb2..c3a5a0e 100644 --- a/models/intermediate/resolutions/int_resolutions__incidents.sql +++ b/models/intermediate/resolutions/int_resolutions__incidents.sql @@ -78,10 +78,10 @@ select is_submission_complete, current_agent_name, document_version, - created_at_date, - created_date_date, - updated_at_date, - updated_date_date, + created_at_utc, + created_date_utc, + updated_at_utc, + updated_date_utc, -- Resolution Details lt.first_comment_at_utc, diff --git a/models/intermediate/resolutions/schema.yml b/models/intermediate/resolutions/schema.yml index 72785d3..7ff833b 100644 --- a/models/intermediate/resolutions/schema.yml +++ b/models/intermediate/resolutions/schema.yml @@ -12,14 +12,14 @@ models: - name: id_incident data_type: text description: "Unique identifier for the incident." - tests: + data_tests: - not_null - unique - name: id_user data_type: text description: "Id of the user who created the incident." - tests: + data_tests: - not_null - name: id_verification @@ -29,7 +29,7 @@ models: - name: current_status_name data_type: text description: "Current status of the incident." - tests: + data_tests: - not_null - name: is_submission_complete @@ -45,19 +45,49 @@ models: description: | "Reported document source, such as ResCentre, API, or manual form." - - name: created_at_date - data_type: timestamp without time zone + - name: created_at_utc + data_type: timestamp description: "Timestamp when the incident was created." - tests: + data_tests: - not_null - - name: updated_at_date - data_type: timestamp without time zone + - name: created_date_utc + data_type: date + description: "Date when the incident was created." + data_tests: + - not_null + + - name: updated_at_utc + data_type: timestamp description: "Timestamp when the incident was last updated." + - name: updated_date_utc + data_type: date + description: "Date when the incident was last updated." + + - name: first_comment_at_utc + data_type: timestamp + description: "Timestamp of the first comment on the incident." + + - name: last_comment_at_utc + data_type: timestamp + description: "Timestamp of the last comment on the incident." + + - name: comment_count + data_type: bigint + description: "Number of comments on the incident." + + - name: first_status_at_utc + data_type: timestamp + description: "Timestamp of the first status change on the incident." + + - name: last_status_at_utc + data_type: timestamp + description: "Timestamp of the last status change on the incident." + - name: resolution_time_in_hours data_type: numeric - description: "Time taken to resolve the incident, in hours." + description: "Time taken to resolve the incident, in hours, if completed." - name: number_damaged_items data_type: bigint @@ -67,16 +97,28 @@ models: data_type: numeric description: | "Original value of the damaged items in the transaction currency." + data_tests: + - dbt_expectations.expect_column_values_to_be_between: + min_value: 0 + strictly: false - name: asked_repair_amount_in_txn_currency data_type: numeric description: | "Amount requested by the host for repair in the transaction currency." + data_tests: + - dbt_expectations.expect_column_values_to_be_between: + min_value: 0 + strictly: false - name: accepted_amount_in_txn_currency data_type: numeric description: | "Final accepted settlement amount in the transaction currency." + data_tests: + - dbt_expectations.expect_column_values_to_be_between: + min_value: 0 + strictly: false - name: protection_name data_type: text @@ -96,6 +138,50 @@ models: data_type: text description: "Guest verification status for this booking." + - name: lower_protection_limit_usd + data_type: text + description: "Lower protection limit in USD." + + - name: upper_protection_limit_usd + data_type: text + description: "Upper protection limit in USD." + + - name: calculated_payout_amount_in_txn_currency + data_type: text + description: "Calculated payout amount in local currency." + data_tests: + - dbt_expectations.expect_column_values_to_be_between: + min_value: 0 + strictly: false + + - name: calculated_payout_currency + data_type: text + description: "Currency of the calculated payout amount." + data_tests: + - not_null + + - name: calculated_payout_amount_in_usd + data_type: text + description: "Calculated payout amount in USD." + data_tests: + - dbt_expectations.expect_column_values_to_be_between: + min_value: 0 + strictly: false + + - name: calculated_guest_charge_amount_in_txn_currency + data_type: text + description: "Calculated guest charge amount in local currency." + data_tests: + - dbt_expectations.expect_column_values_to_be_between: + min_value: 0 + strictly: false + + - name: calculated_guest_charge_currency + data_type: text + description: "Currency of the calculated guest charge amount." + data_tests: + - not_null + - name: id_user_host data_type: text description: | @@ -108,9 +194,101 @@ models: field: id_user where: "document_version <> 'OldClaimToIncidentModel'" + - name: claims_count + data_type: bigint + description: "Number of claims made by the host." + + - name: host_account_name + data_type: text + description: "Name of the host account." + + - name: host_email + data_type: text + description: "Email of the host." + + - name: host_last_name + data_type: text + description: "Last name of the host." + + - name: host_first_name + data_type: text + description: "First name of the host." + + - name: host_phone_code + data_type: text + description: "Phone code of the host." + + - name: host_phone_number + data_type: text + description: "Phone number of the host." + - name: id_user_guest data_type: text description: "Superhog code that uniquely identifies a single guest." + data_tests: + - not_null: + where: "document_version <> 'OldClaimToIncidentModel'" + - relationships: + to: ref('stg_core__user') + field: id_user + where: "document_version <> 'OldClaimToIncidentModel'" + + - name: guest_email + data_type: text + description: "Email of the guest." + + - name: guest_last_name + data_type: text + description: "Last name of the guest." + + - name: guest_first_name + data_type: text + description: "First name of the guest." + + - name: guest_phone_code + data_type: text + description: "Phone code of the guest." + + - name: guest_phone_number + data_type: text + description: "Phone number of the guest." + + - name: is_guest_deposit_retained + data_type: boolean + description: "In case a deposit has been collected from the guest, + whether it has been retained or not." + + - name: is_guest_deposit_collected + data_type: boolean + description: "If the host has collected a deposit from the guest." + + - name: deposit_retained_amount_in_txn_currency + data_type: text + description: "Amount of the deposit retained in local currency." + + - name: deposit_retained_currency + data_type: text + description: "Currency of the deposit retained." + + - name: has_guest_contributed_to_cost + data_type: boolean + description: "If the guest has agreed to contribute to the cost when talking to the host" + + - name: has_host_taken_preventative_steps + data_type: boolean + description: "If the host has taken any action to prevent the damage or further damage once identified" + + - name: guest_contribution_amount_in_txn_currency + data_type: text + description: "Amount of the guest contribution, in case they did, in local currency." + + - name: guest_contribution_currency + data_type: text + description: "Currency of the guest contribution." + + - name: guest_contacted_about_damage + data_type: boolean + description: "If the host has contacted the guest about the damage caused" - name: id_accommodation data_type: text @@ -119,7 +297,7 @@ models: - name: id_booking data_type: text description: "Superhog unique booking ID." - tests: + data_tests: - not_null - name: booking_status From f1a0b04388d5df492e0f9dd7b73031ef40ce89df Mon Sep 17 00:00:00 2001 From: Joaquin Ossa Date: Sun, 23 Feb 2025 16:07:13 +0100 Subject: [PATCH 10/19] commit wip --- models/intermediate/resolutions/schema.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/models/intermediate/resolutions/schema.yml b/models/intermediate/resolutions/schema.yml index 7ff833b..b525438 100644 --- a/models/intermediate/resolutions/schema.yml +++ b/models/intermediate/resolutions/schema.yml @@ -263,12 +263,18 @@ models: description: "If the host has collected a deposit from the guest." - name: deposit_retained_amount_in_txn_currency - data_type: text + data_type: numeric description: "Amount of the deposit retained in local currency." + data_tests: + - dbt_expectations.expect_column_values_to_be_between: + min_value: 0 + strictly: false - name: deposit_retained_currency data_type: text description: "Currency of the deposit retained." + data_tests: + - not_null - name: has_guest_contributed_to_cost data_type: boolean From 8b7dfa083921f87149b19bbf030078fb0a44a7cf Mon Sep 17 00:00:00 2001 From: Joaquin Ossa Date: Sun, 23 Feb 2025 16:30:16 +0100 Subject: [PATCH 11/19] commit wip --- models/intermediate/resolutions/schema.yml | 60 ++++++++++++++++++---- 1 file changed, 49 insertions(+), 11 deletions(-) diff --git a/models/intermediate/resolutions/schema.yml b/models/intermediate/resolutions/schema.yml index b525438..a8e339d 100644 --- a/models/intermediate/resolutions/schema.yml +++ b/models/intermediate/resolutions/schema.yml @@ -88,6 +88,11 @@ models: - name: resolution_time_in_hours data_type: numeric description: "Time taken to resolve the incident, in hours, if completed." + data_tests: + - dbt_expectations.expect_column_values_to_be_between: + min_value: 0 + strictly: true + where: "document_version <> 'OldClaimToIncidentModel'" - name: number_damaged_items data_type: bigint @@ -139,12 +144,20 @@ models: description: "Guest verification status for this booking." - name: lower_protection_limit_usd - data_type: text + data_type: numeric description: "Lower protection limit in USD." + data_tests: + - dbt_expectations.expect_column_values_to_be_between: + min_value: 0 + strictly: false - name: upper_protection_limit_usd - data_type: text + data_type: numeric description: "Upper protection limit in USD." + data_tests: + - dbt_expectations.expect_column_values_to_be_between: + min_value: 0 + strictly: false - name: calculated_payout_amount_in_txn_currency data_type: text @@ -158,7 +171,8 @@ models: data_type: text description: "Currency of the calculated payout amount." data_tests: - - not_null + - not_null: + where: "calculated_payout_amount_in_txn_currency > 0" - name: calculated_payout_amount_in_usd data_type: text @@ -274,7 +288,8 @@ models: data_type: text description: "Currency of the deposit retained." data_tests: - - not_null + - not_null: + where: "deposit_retained_amount_in_txn_currency > 0 and document_version <> 'OldClaimToIncidentModel'" - name: has_guest_contributed_to_cost data_type: boolean @@ -285,26 +300,49 @@ models: description: "If the host has taken any action to prevent the damage or further damage once identified" - name: guest_contribution_amount_in_txn_currency - data_type: text - description: "Amount of the guest contribution, in case they did, in local currency." + data_type: numeric + description: "Amount of the guest contribution, in case they did, + in local currency." + data_tests: + - dbt_expectations.expect_column_values_to_be_between: + min_value: 0 + strictly: false - name: guest_contribution_currency data_type: text description: "Currency of the guest contribution." + data_tests: + - not_null: + where: "guest_contribution_amount_in_txn_currency > 0 and document_version <> 'OldClaimToIncidentModel'" - name: guest_contacted_about_damage data_type: boolean description: "If the host has contacted the guest about the damage caused" - name: id_accommodation - data_type: text + data_type: numeric description: "Superhog code that uniquely identifies a single accommodation." - - - name: id_booking - data_type: text - description: "Superhog unique booking ID." data_tests: - not_null + - relationships: + to: ref('stg_core__accommodation') + field: id_accommodation + where: "document_version <> 'OldClaimToIncidentModel'" + + - name: accommodation_name + data_type: text + description: "Accommodation name." + + - name: id_booking + data_type: numeric + description: "Superhog unique booking ID" + data_tests: + - not_null: + where: "document_version <> 'OldClaimToIncidentModel'" + - relationships: + to: ref('stg_core__booking') + field: id_booking + where: "document_version <> 'OldClaimToIncidentModel'" - name: booking_status data_type: text From 63a2eff6e274eee40702d20e6760fd3848f37825 Mon Sep 17 00:00:00 2001 From: Joaquin Ossa Date: Sun, 23 Feb 2025 16:59:43 +0100 Subject: [PATCH 12/19] finished schema --- .../int_resolutions__incidents.sql | 110 +++++++++--------- models/intermediate/resolutions/schema.yml | 73 ++++++------ 2 files changed, 93 insertions(+), 90 deletions(-) diff --git a/models/intermediate/resolutions/int_resolutions__incidents.sql b/models/intermediate/resolutions/int_resolutions__incidents.sql index c3a5a0e..d93b3f0 100644 --- a/models/intermediate/resolutions/int_resolutions__incidents.sql +++ b/models/intermediate/resolutions/int_resolutions__incidents.sql @@ -72,16 +72,16 @@ with select -- Basic Incident Details i.id_incident, - id_user, - id_verification, - current_status_name, - is_submission_complete, - current_agent_name, - document_version, - created_at_utc, - created_date_utc, - updated_at_utc, - updated_date_utc, + i.id_user, + i.id_verification, + i.current_status_name, + i.is_submission_complete, + i.current_agent_name, + i.document_version, + i.created_at_utc, + i.created_date_utc, + i.updated_at_utc, + i.updated_date_utc, -- Resolution Details lt.first_comment_at_utc, @@ -90,7 +90,7 @@ select lt.first_status_at_utc, lt.last_status_at_utc, case - when is_submission_complete + when i.is_submission_complete then ( extract(epoch from (lt.last_status_at_utc - lt.first_status_at_utc)) @@ -104,65 +104,65 @@ select dr.accepted_amount_in_txn_currency, -- Resolutions calculator - protection_name, - was_overriden, - waiver_or_deposit_name, - guest_verification_status, - lower_protection_limit_usd, - upper_protection_limit_usd, - calculated_payout_amount_in_txn_currency, - calculated_payout_currency, - calculated_payout_amount_in_usd, - calculated_guest_charge_amount_in_txn_currency, - calculated_guest_charge_currency, - calculated_guest_charge_amount_in_usd + i.protection_name, + i.was_overriden, + i.waiver_or_deposit_name, + i.guest_verification_status, + i.lower_protection_limit_usd, + i.upper_protection_limit_usd, + i.calculated_payout_amount_in_txn_currency, + i.calculated_payout_currency, + i.calculated_payout_amount_in_usd, + i.calculated_guest_charge_amount_in_txn_currency, + i.calculated_guest_charge_currency, + i.calculated_guest_charge_amount_in_usd, -- Host Details - id_user_host, + i.id_user_host, lt.claims_count, - host_account_name, + i.host_account_name, -- Host Contact Details - host_email, - host_last_name, - host_first_name, - host_phone_code, - host_phone_number, + i.host_email, + i.host_last_name, + i.host_first_name, + i.host_phone_code, + i.host_phone_number, -- Guest Details - id_user_guest, - guest_email, - guest_last_name, - guest_first_name, - guest_phone_code, - guest_phone_number, + i.id_user_guest, + i.guest_email, + i.guest_last_name, + i.guest_first_name, + i.guest_phone_code, + i.guest_phone_number, -- Guest Deposit Details - is_guest_deposit_retained, - is_guest_deposit_collected, - deposit_retained_amount_in_txn_currency, - deposit_retained_currency, + i.is_guest_deposit_retained, + i.is_guest_deposit_collected, + i.deposit_retained_amount_in_txn_currency, + i.deposit_retained_currency, -- Guest Involvements - has_guest_contributed_to_cost, - has_host_taken_preventative_steps, - guest_contribution_amount_in_txn_currency, - guest_contribution_currency, - guest_contacted_about_damage, + i.has_guest_contributed_to_cost, + i.has_host_taken_preventative_steps, + i.guest_contribution_amount_in_txn_currency, + i.guest_contribution_currency, + i.guest_contacted_about_damage, -- Accommodation Details - id_accommodation, - accommodation_name, + i.id_accommodation, + i.accommodation_name, -- Booking Details - check_in_at_utc, - check_in_date_utc, - check_out_at_utc, - check_out_date_utc, - id_booking, - booking_status, - id_reservation, - booking_protection + i.check_in_at_utc, + i.check_in_date_utc, + i.check_out_at_utc, + i.check_out_date_utc, + i.id_booking, + i.booking_status, + i.id_reservation, + i.booking_protection from stg_resolutions__incidents i left join logs_timeline lt on i.id_incident = lt.id_incident diff --git a/models/intermediate/resolutions/schema.yml b/models/intermediate/resolutions/schema.yml index a8e339d..3be73c9 100644 --- a/models/intermediate/resolutions/schema.yml +++ b/models/intermediate/resolutions/schema.yml @@ -14,7 +14,7 @@ models: description: "Unique identifier for the incident." data_tests: - not_null - - unique + # - unique - name: id_user data_type: text @@ -49,13 +49,15 @@ models: data_type: timestamp description: "Timestamp when the incident was created." data_tests: - - not_null + - not_null: + where: "document_version <> 'OldClaimToIncidentModel'" - name: created_date_utc data_type: date description: "Date when the incident was created." data_tests: - - not_null + - not_null: + where: "document_version <> 'OldClaimToIncidentModel'" - name: updated_at_utc data_type: timestamp @@ -88,11 +90,11 @@ models: - name: resolution_time_in_hours data_type: numeric description: "Time taken to resolve the incident, in hours, if completed." - data_tests: - - dbt_expectations.expect_column_values_to_be_between: - min_value: 0 - strictly: true - where: "document_version <> 'OldClaimToIncidentModel'" + # data_tests: + # - dbt_expectations.expect_column_values_to_be_between: + # min_value: 0 + # strictly: true + # where: "document_version <> 'OldClaimToIncidentModel'" - name: number_damaged_items data_type: bigint @@ -194,19 +196,20 @@ models: data_type: text description: "Currency of the calculated guest charge amount." data_tests: - - not_null + - not_null: + where: "calculated_guest_charge_amount_in_txn_currency > 0" - name: id_user_host data_type: text description: | "Unique Id that identifies the host. Can be from SH or can be external from the client." - data_tests: - - not_null - - relationships: - to: ref('stg_core__user') - field: id_user - where: "document_version <> 'OldClaimToIncidentModel'" + # data_tests: + # - not_null + # - relationships: + # to: ref('stg_core__user') + # field: id_user + # where: "document_version <> 'OldClaimToIncidentModel'" - name: claims_count data_type: bigint @@ -239,13 +242,13 @@ models: - name: id_user_guest data_type: text description: "Superhog code that uniquely identifies a single guest." - data_tests: - - not_null: - where: "document_version <> 'OldClaimToIncidentModel'" - - relationships: - to: ref('stg_core__user') - field: id_user - where: "document_version <> 'OldClaimToIncidentModel'" + # data_tests: + # - not_null: + # where: "document_version <> 'OldClaimToIncidentModel'" + # - relationships: + # to: ref('stg_core__user') + # field: id_user + # where: "document_version <> 'OldClaimToIncidentModel'" - name: guest_email data_type: text @@ -322,12 +325,12 @@ models: - name: id_accommodation data_type: numeric description: "Superhog code that uniquely identifies a single accommodation." - data_tests: - - not_null - - relationships: - to: ref('stg_core__accommodation') - field: id_accommodation - where: "document_version <> 'OldClaimToIncidentModel'" + # data_tests: + # - not_null + # - relationships: + # to: ref('stg_core__accommodation') + # field: id_accommodation + # where: "document_version <> 'OldClaimToIncidentModel'" - name: accommodation_name data_type: text @@ -336,13 +339,13 @@ models: - name: id_booking data_type: numeric description: "Superhog unique booking ID" - data_tests: - - not_null: - where: "document_version <> 'OldClaimToIncidentModel'" - - relationships: - to: ref('stg_core__booking') - field: id_booking - where: "document_version <> 'OldClaimToIncidentModel'" + # data_tests: + # - not_null: + # where: "document_version <> 'OldClaimToIncidentModel'" + # - relationships: + # to: ref('stg_core__booking') + # field: id_booking + # where: "document_version <> 'OldClaimToIncidentModel'" - name: booking_status data_type: text From 7f8e7489e42207495d5d550d9f97dbd4bea81cf2 Mon Sep 17 00:00:00 2001 From: Joaquin Ossa Date: Mon, 24 Feb 2025 10:48:34 +0100 Subject: [PATCH 13/19] finished schema --- models/intermediate/resolutions/schema.yml | 24 ++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/models/intermediate/resolutions/schema.yml b/models/intermediate/resolutions/schema.yml index 3be73c9..d150edd 100644 --- a/models/intermediate/resolutions/schema.yml +++ b/models/intermediate/resolutions/schema.yml @@ -336,6 +336,30 @@ models: data_type: text description: "Accommodation name." + - name: check_in_at_utc + data_type: timestamp without time zone + description: "Timestamp of the check-in date in UTC of the booking." + data_tests: + - not_null + + - name: check_in_date_utc + data_type: date + description: "Date of the check-in date in UTC of the booking." + data_tests: + - not_null + + - name: check_out_at_utc + data_type: timestamp without time zone + description: "Timestamp of the check-out date in UTC of the booking." + data_tests: + - not_null + + - name: check_out_date_utc + data_type: date + description: "Date of the check-out date in UTC of the booking." + data_tests: + - not_null + - name: id_booking data_type: numeric description: "Superhog unique booking ID" From 68bddf2b7d1c8614650f113bc0701715bccd2bdb Mon Sep 17 00:00:00 2001 From: Joaquin Ossa Date: Mon, 24 Feb 2025 14:53:24 +0100 Subject: [PATCH 14/19] Commit update --- .../int_resolutions__incidents.sql | 13 +-- models/intermediate/resolutions/schema.yml | 88 +++++++------------ 2 files changed, 36 insertions(+), 65 deletions(-) diff --git a/models/intermediate/resolutions/int_resolutions__incidents.sql b/models/intermediate/resolutions/int_resolutions__incidents.sql index d93b3f0..71212f2 100644 --- a/models/intermediate/resolutions/int_resolutions__incidents.sql +++ b/models/intermediate/resolutions/int_resolutions__incidents.sql @@ -31,11 +31,7 @@ with from jsonb_array_elements(status_history_logs::jsonb) status_step order by (status_step ->> 'CreatedDate')::timestamp desc limit 1 - ) as last_status_at_utc, - ( - select count(*) - from jsonb_array_elements(host_user_claim_logs::jsonb) comment - ) as claims_count + ) as last_status_at_utc from stg_resolutions__incidents ), damage_reported_items as ( @@ -72,7 +68,6 @@ with select -- Basic Incident Details i.id_incident, - i.id_user, i.id_verification, i.current_status_name, i.is_submission_complete, @@ -119,7 +114,6 @@ select -- Host Details i.id_user_host, - lt.claims_count, i.host_account_name, -- Host Contact Details @@ -148,7 +142,7 @@ select i.has_host_taken_preventative_steps, i.guest_contribution_amount_in_txn_currency, i.guest_contribution_currency, - i.guest_contacted_about_damage, + i.is_guest_contacted_about_damage, -- Accommodation Details i.id_accommodation, @@ -161,8 +155,7 @@ select i.check_out_date_utc, i.id_booking, i.booking_status, - i.id_reservation, - i.booking_protection + i.id_reservation from stg_resolutions__incidents i left join logs_timeline lt on i.id_incident = lt.id_incident diff --git a/models/intermediate/resolutions/schema.yml b/models/intermediate/resolutions/schema.yml index d150edd..b37d808 100644 --- a/models/intermediate/resolutions/schema.yml +++ b/models/intermediate/resolutions/schema.yml @@ -14,13 +14,7 @@ models: description: "Unique identifier for the incident." data_tests: - not_null - # - unique - - - name: id_user - data_type: text - description: "Id of the user who created the incident." - data_tests: - - not_null + - unique - name: id_verification data_type: text @@ -90,11 +84,11 @@ models: - name: resolution_time_in_hours data_type: numeric description: "Time taken to resolve the incident, in hours, if completed." - # data_tests: - # - dbt_expectations.expect_column_values_to_be_between: - # min_value: 0 - # strictly: true - # where: "document_version <> 'OldClaimToIncidentModel'" + data_tests: + - dbt_expectations.expect_column_values_to_be_between: + min_value: 0 + strictly: true + where: "document_version <> 'OldClaimToIncidentModel'" - name: number_damaged_items data_type: bigint @@ -127,20 +121,12 @@ models: min_value: 0 strictly: false - - name: protection_name - data_type: text - description: "Name of the protection plan covering the booking." - - name: was_overriden data_type: boolean description: | "Flag to indicate if the proposed settlement value from the calculator was overridden." - - name: waiver_or_deposit_name - data_type: text - description: "Waiver or deposit type associated with the booking." - - name: guest_verification_status data_type: text description: "Guest verification status for this booking." @@ -204,16 +190,12 @@ models: description: | "Unique Id that identifies the host. Can be from SH or can be external from the client." - # data_tests: - # - not_null - # - relationships: - # to: ref('stg_core__user') - # field: id_user - # where: "document_version <> 'OldClaimToIncidentModel'" - - - name: claims_count - data_type: bigint - description: "Number of claims made by the host." + data_tests: + - not_null + - relationships: + to: ref('stg_core__user') + field: id_user + where: "document_version <> 'OldClaimToIncidentModel'" - name: host_account_name data_type: text @@ -242,13 +224,13 @@ models: - name: id_user_guest data_type: text description: "Superhog code that uniquely identifies a single guest." - # data_tests: - # - not_null: - # where: "document_version <> 'OldClaimToIncidentModel'" - # - relationships: - # to: ref('stg_core__user') - # field: id_user - # where: "document_version <> 'OldClaimToIncidentModel'" + data_tests: + - not_null: + where: "document_version <> 'OldClaimToIncidentModel'" + - relationships: + to: ref('stg_core__user') + field: id_user + where: "document_version <> 'OldClaimToIncidentModel'" - name: guest_email data_type: text @@ -318,19 +300,19 @@ models: - not_null: where: "guest_contribution_amount_in_txn_currency > 0 and document_version <> 'OldClaimToIncidentModel'" - - name: guest_contacted_about_damage + - name: is_guest_contacted_about_damage data_type: boolean description: "If the host has contacted the guest about the damage caused" - name: id_accommodation data_type: numeric description: "Superhog code that uniquely identifies a single accommodation." - # data_tests: - # - not_null - # - relationships: - # to: ref('stg_core__accommodation') - # field: id_accommodation - # where: "document_version <> 'OldClaimToIncidentModel'" + data_tests: + - not_null + - relationships: + to: ref('stg_core__accommodation') + field: id_accommodation + where: "document_version <> 'OldClaimToIncidentModel'" - name: accommodation_name data_type: text @@ -363,13 +345,13 @@ models: - name: id_booking data_type: numeric description: "Superhog unique booking ID" - # data_tests: - # - not_null: - # where: "document_version <> 'OldClaimToIncidentModel'" - # - relationships: - # to: ref('stg_core__booking') - # field: id_booking - # where: "document_version <> 'OldClaimToIncidentModel'" + data_tests: + - not_null: + where: "document_version <> 'OldClaimToIncidentModel'" + - relationships: + to: ref('stg_core__booking') + field: id_booking + where: "document_version <> 'OldClaimToIncidentModel'" - name: booking_status data_type: text @@ -378,7 +360,3 @@ models: - name: id_reservation data_type: text description: "External unique ID for the reservation." - - - name: booking_protection - data_type: text - description: "Details on the booking protection applied." From 335fccd57e5244c388268fb3fed6bade47f2ce6d Mon Sep 17 00:00:00 2001 From: Joaquin Ossa Date: Mon, 24 Feb 2025 16:39:37 +0100 Subject: [PATCH 15/19] updated schema --- models/intermediate/resolutions/schema.yml | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/models/intermediate/resolutions/schema.yml b/models/intermediate/resolutions/schema.yml index b37d808..b86a506 100644 --- a/models/intermediate/resolutions/schema.yml +++ b/models/intermediate/resolutions/schema.yml @@ -43,15 +43,13 @@ models: data_type: timestamp description: "Timestamp when the incident was created." data_tests: - - not_null: - where: "document_version <> 'OldClaimToIncidentModel'" + - not_null - name: created_date_utc data_type: date description: "Date when the incident was created." data_tests: - - not_null: - where: "document_version <> 'OldClaimToIncidentModel'" + - not_null - name: updated_at_utc data_type: timestamp @@ -88,7 +86,6 @@ models: - dbt_expectations.expect_column_values_to_be_between: min_value: 0 strictly: true - where: "document_version <> 'OldClaimToIncidentModel'" - name: number_damaged_items data_type: bigint @@ -195,7 +192,6 @@ models: - relationships: to: ref('stg_core__user') field: id_user - where: "document_version <> 'OldClaimToIncidentModel'" - name: host_account_name data_type: text @@ -225,12 +221,9 @@ models: data_type: text description: "Superhog code that uniquely identifies a single guest." data_tests: - - not_null: - where: "document_version <> 'OldClaimToIncidentModel'" - relationships: to: ref('stg_core__user') field: id_user - where: "document_version <> 'OldClaimToIncidentModel'" - name: guest_email data_type: text @@ -274,7 +267,7 @@ models: description: "Currency of the deposit retained." data_tests: - not_null: - where: "deposit_retained_amount_in_txn_currency > 0 and document_version <> 'OldClaimToIncidentModel'" + where: "deposit_retained_amount_in_txn_currency > 0" - name: has_guest_contributed_to_cost data_type: boolean @@ -298,7 +291,7 @@ models: description: "Currency of the guest contribution." data_tests: - not_null: - where: "guest_contribution_amount_in_txn_currency > 0 and document_version <> 'OldClaimToIncidentModel'" + where: "guest_contribution_amount_in_txn_currency > 0" - name: is_guest_contacted_about_damage data_type: boolean @@ -312,7 +305,6 @@ models: - relationships: to: ref('stg_core__accommodation') field: id_accommodation - where: "document_version <> 'OldClaimToIncidentModel'" - name: accommodation_name data_type: text @@ -346,12 +338,10 @@ models: data_type: numeric description: "Superhog unique booking ID" data_tests: - - not_null: - where: "document_version <> 'OldClaimToIncidentModel'" + - not_null - relationships: to: ref('stg_core__booking') field: id_booking - where: "document_version <> 'OldClaimToIncidentModel'" - name: booking_status data_type: text From daa93c056c3731b7fc0b45d8c21affe341a540d7 Mon Sep 17 00:00:00 2001 From: Joaquin Ossa Date: Mon, 24 Feb 2025 16:50:18 +0100 Subject: [PATCH 16/19] commit wip --- .../intermediate/resolutions/int_resolutions__incidents.sql | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/models/intermediate/resolutions/int_resolutions__incidents.sql b/models/intermediate/resolutions/int_resolutions__incidents.sql index 71212f2..94b0ec8 100644 --- a/models/intermediate/resolutions/int_resolutions__incidents.sql +++ b/models/intermediate/resolutions/int_resolutions__incidents.sql @@ -2,6 +2,9 @@ with stg_resolutions__incidents as ( select * from {{ ref("stg_resolutions__incidents") }} ), + int_daily_currency_exchange_rates as ( + select * from {{ ref("int_daily_currency_exchange_rates") }} + ), logs_timeline as ( select id_incident, @@ -35,7 +38,7 @@ with from stg_resolutions__incidents ), damage_reported_items as ( - select id_incident, jsonb_array_elements(damage_report_items::jsonb) as item -- No need for 'Items' if already an array + select id_incident, jsonb_array_elements(damage_report_items::jsonb) as item, from stg_resolutions__incidents where damage_report_items is not null ), @@ -53,6 +56,7 @@ with coalesce((cm.corrective_measure ->> 'SettlementAmount')::numeric, 0) ) as accepted_amount_in_txn_currency from damage_reported_items dri + left join int_daily_currency_exchange_rates cer left join lateral jsonb_array_elements( case From c49239a7cf7e69108c84fd72a9dd78f65fad976a Mon Sep 17 00:00:00 2001 From: Joaquin Ossa Date: Tue, 25 Feb 2025 09:13:41 +0100 Subject: [PATCH 17/19] Final changes --- .../int_resolutions__incidents.sql | 47 +++++++++++++++---- models/intermediate/resolutions/schema.yml | 20 ++++++-- 2 files changed, 54 insertions(+), 13 deletions(-) diff --git a/models/intermediate/resolutions/int_resolutions__incidents.sql b/models/intermediate/resolutions/int_resolutions__incidents.sql index 94b0ec8..c72441c 100644 --- a/models/intermediate/resolutions/int_resolutions__incidents.sql +++ b/models/intermediate/resolutions/int_resolutions__incidents.sql @@ -2,6 +2,7 @@ with stg_resolutions__incidents as ( select * from {{ ref("stg_resolutions__incidents") }} ), + int_core__user_host as (select * from {{ ref("int_core__user_host") }}), int_daily_currency_exchange_rates as ( select * from {{ ref("int_daily_currency_exchange_rates") }} ), @@ -37,26 +38,50 @@ with ) as last_status_at_utc from stg_resolutions__incidents ), + -- The host can submit any number of damaged items in the damage report. + -- Each damaged item can have different currencies, so we need to convert + -- the amount to the host's currency. damage_reported_items as ( - select id_incident, jsonb_array_elements(damage_report_items::jsonb) as item, - from stg_resolutions__incidents - where damage_report_items is not null + select + i.id_incident, + jsonb_array_elements(i.damage_report_items::jsonb) as item, + uh.account_currency_iso4217 as host_currency, + i.created_date_utc + from stg_resolutions__incidents i + left join int_core__user_host uh on i.id_user_host = uh.id_user_host + where i.damage_report_items is not null ), damage_report as ( select dri.id_incident, + dri.host_currency, count(distinct dri.item ->> 'Id') as number_damaged_items, sum( - coalesce((cm.corrective_measure ->> 'OriginalAmount')::numeric, 0) + coalesce( + ((dri.item ->> 'OriginalAmount')::numeric * cer.rate)::decimal( + 19, 4 + ), + 0 + ) ) as original_value_amount_in_txn_currency, sum( - coalesce((cm.corrective_measure ->> 'Amount')::numeric, 0) + coalesce( + ((cm.corrective_measure ->> 'Amount')::numeric * cer.rate)::decimal( + 19, 4 + ), + 0 + ) ) as asked_repair_amount_in_txn_currency, sum( - coalesce((cm.corrective_measure ->> 'SettlementAmount')::numeric, 0) + coalesce( + ( + (cm.corrective_measure ->> 'SettlementAmount')::numeric + * cer.rate + )::decimal(19, 4), + 0 + ) ) as accepted_amount_in_txn_currency from damage_reported_items dri - left join int_daily_currency_exchange_rates cer left join lateral jsonb_array_elements( case @@ -66,7 +91,12 @@ with end ) as cm(corrective_measure) on true - group by dri.id_incident + left join + int_daily_currency_exchange_rates cer + on dri.created_date_utc = cer.rate_date_utc + and (cm.corrective_measure ->> 'Currency') = cer.from_currency + and dri.host_currency = cer.to_currency + group by dri.id_incident, dri.host_currency ) select @@ -101,6 +131,7 @@ select dr.original_value_amount_in_txn_currency, dr.asked_repair_amount_in_txn_currency, dr.accepted_amount_in_txn_currency, + dr.host_currency, -- Resolutions calculator i.protection_name, diff --git a/models/intermediate/resolutions/schema.yml b/models/intermediate/resolutions/schema.yml index b86a506..3615617 100644 --- a/models/intermediate/resolutions/schema.yml +++ b/models/intermediate/resolutions/schema.yml @@ -69,7 +69,7 @@ models: - name: comment_count data_type: bigint - description: "Number of comments on the incident." + description: "Number of comments submitted on the incident." - name: first_status_at_utc data_type: timestamp @@ -118,6 +118,15 @@ models: min_value: 0 strictly: false + - name: host_currency + data_type: text + description: "Host or local currency" + data_tests: + - not_null: + where: "original_value_amount_in_txn_currency > 0 + or asked_repair_amount_in_txn_currency > 0 + or accepted_amount_in_txn_currency > 0" + - name: was_overriden data_type: boolean description: | @@ -185,8 +194,7 @@ models: - name: id_user_host data_type: text description: | - "Unique Id that identifies the host. Can be from SH or can - be external from the client." + "Unique Id that identifies the host." data_tests: - not_null - relationships: @@ -271,11 +279,13 @@ models: - name: has_guest_contributed_to_cost data_type: boolean - description: "If the guest has agreed to contribute to the cost when talking to the host" + description: "If the guest has agreed to contribute to the cost + when talking to the host" - name: has_host_taken_preventative_steps data_type: boolean - description: "If the host has taken any action to prevent the damage or further damage once identified" + description: "If the host has taken any action to prevent the damage + or further damage once identified" - name: guest_contribution_amount_in_txn_currency data_type: numeric From cfc6971cb4265766005d49073b94575690352c05 Mon Sep 17 00:00:00 2001 From: Joaquin Ossa Date: Tue, 25 Feb 2025 16:58:14 +0100 Subject: [PATCH 18/19] Addressed comments --- .../int_resolutions__incidents.sql | 81 +++++++++++++++-- models/intermediate/resolutions/schema.yml | 86 +++++++++++++++++-- 2 files changed, 152 insertions(+), 15 deletions(-) diff --git a/models/intermediate/resolutions/int_resolutions__incidents.sql b/models/intermediate/resolutions/int_resolutions__incidents.sql index c72441c..4e00548 100644 --- a/models/intermediate/resolutions/int_resolutions__incidents.sql +++ b/models/intermediate/resolutions/int_resolutions__incidents.sql @@ -63,7 +63,15 @@ with ), 0 ) - ) as original_value_amount_in_txn_currency, + ) as original_value_amount_in_host_currency, + sum( + coalesce( + ((dri.item ->> 'OriginalAmount')::numeric * cer_gbp.rate)::decimal( + 19, 4 + ), + 0 + ) + ) as original_value_amount_in_gbp, sum( coalesce( ((cm.corrective_measure ->> 'Amount')::numeric * cer.rate)::decimal( @@ -71,7 +79,15 @@ with ), 0 ) - ) as asked_repair_amount_in_txn_currency, + ) as asked_repair_amount_in_host_currency, + sum( + coalesce( + ( + (cm.corrective_measure ->> 'Amount')::numeric * cer_gbp.rate + )::decimal(19, 4), + 0 + ) + ) as asked_repair_amount_in_gbp, sum( coalesce( ( @@ -80,7 +96,16 @@ with )::decimal(19, 4), 0 ) - ) as accepted_amount_in_txn_currency + ) as accepted_amount_in_host_currency, + sum( + coalesce( + ( + (cm.corrective_measure ->> 'SettlementAmount')::numeric + * cer_gbp.rate + )::decimal(19, 4), + 0 + ) + ) as accepted_amount_in_gbp from damage_reported_items dri left join lateral jsonb_array_elements( @@ -96,7 +121,34 @@ with on dri.created_date_utc = cer.rate_date_utc and (cm.corrective_measure ->> 'Currency') = cer.from_currency and dri.host_currency = cer.to_currency + left join + int_daily_currency_exchange_rates cer_gbp + on dri.created_date_utc = cer_gbp.rate_date_utc + and (cm.corrective_measure ->> 'Currency') = cer_gbp.from_currency + and 'GBP' = cer.to_currency group by dri.id_incident, dri.host_currency + ), + guest_amounts_in_gbp as ( + select + i.id_incident, + (i.deposit_retained_amount_in_txn_currency * cer_deposit.rate)::decimal( + 19, 4 + ) as deposit_retained_amount_in_gbp, + ( + i.guest_contribution_amount_in_txn_currency * cer_contribution.rate + )::decimal(19, 4) as guest_contribution_amount_in_gbp + from stg_resolutions__incidents i + left join logs_timeline lt on i.id_incident = lt.id_incident + left join + int_daily_currency_exchange_rates cer_deposit + on (lt.last_status_at_utc)::date = cer_deposit.rate_date_utc + and cer_deposit.from_currency = i.deposit_retained_currency + and cer_deposit.to_currency = 'GBP' + left join + int_daily_currency_exchange_rates cer_contribution + on (lt.last_status_at_utc)::date = cer_contribution.rate_date_utc + and cer_contribution.from_currency = i.guest_contribution_currency + and cer_contribution.to_currency = 'GBP' ) select @@ -128,9 +180,12 @@ select else null end as resolution_time_in_hours, dr.number_damaged_items, - dr.original_value_amount_in_txn_currency, - dr.asked_repair_amount_in_txn_currency, - dr.accepted_amount_in_txn_currency, + dr.original_value_amount_in_host_currency, + dr.original_value_amount_in_gbp, + dr.asked_repair_amount_in_host_currency, + dr.asked_repair_amount_in_gbp, + dr.accepted_amount_in_host_currency, + dr.accepted_amount_in_gbp, dr.host_currency, -- Resolutions calculator @@ -143,9 +198,15 @@ select i.calculated_payout_amount_in_txn_currency, i.calculated_payout_currency, i.calculated_payout_amount_in_usd, + (i.calculated_payout_amount_in_usd * cer.rate)::decimal( + 19, 4 + ) as calculated_payout_amount_in_gbp, i.calculated_guest_charge_amount_in_txn_currency, i.calculated_guest_charge_currency, i.calculated_guest_charge_amount_in_usd, + (i.calculated_guest_charge_amount_in_usd * cer.rate)::decimal( + 19, 4 + ) as calculated_guest_charge_amount_in_gbp, -- Host Details i.id_user_host, @@ -171,12 +232,14 @@ select i.is_guest_deposit_collected, i.deposit_retained_amount_in_txn_currency, i.deposit_retained_currency, + ga.deposit_retained_amount_in_gbp, -- Guest Involvements i.has_guest_contributed_to_cost, i.has_host_taken_preventative_steps, i.guest_contribution_amount_in_txn_currency, i.guest_contribution_currency, + ga.guest_contribution_amount_in_gbp, i.is_guest_contacted_about_damage, -- Accommodation Details @@ -195,3 +258,9 @@ select from stg_resolutions__incidents i left join logs_timeline lt on i.id_incident = lt.id_incident left join damage_report dr on i.id_incident = dr.id_incident +left join guest_amounts_in_gbp ga on i.id_incident = ga.id_incident +left join + int_daily_currency_exchange_rates cer + on (lt.last_status_at_utc)::date = cer.rate_date_utc + and cer.from_currency = 'USD' + and cer.to_currency = 'GBP' diff --git a/models/intermediate/resolutions/schema.yml b/models/intermediate/resolutions/schema.yml index 3615617..996b745 100644 --- a/models/intermediate/resolutions/schema.yml +++ b/models/intermediate/resolutions/schema.yml @@ -91,28 +91,55 @@ models: data_type: bigint description: "Number of items reported as damaged in the incident." - - name: original_value_amount_in_txn_currency + - name: original_value_amount_in_host_currency data_type: numeric description: | - "Original value of the damaged items in the transaction currency." + "Original value of the damaged items in the host currency." data_tests: - dbt_expectations.expect_column_values_to_be_between: min_value: 0 strictly: false - - name: asked_repair_amount_in_txn_currency + - name: original_value_amount_in_gbp data_type: numeric description: | - "Amount requested by the host for repair in the transaction currency." + "Original value of the damaged items in GBP." data_tests: - dbt_expectations.expect_column_values_to_be_between: min_value: 0 strictly: false - - name: accepted_amount_in_txn_currency + - name: asked_repair_amount_in_host_currency data_type: numeric description: | - "Final accepted settlement amount in the transaction currency." + "Amount requested by the host for repair in the host currency." + data_tests: + - dbt_expectations.expect_column_values_to_be_between: + min_value: 0 + strictly: false + + - name: asked_repair_amount_in_gbp + data_type: numeric + description: | + "Amount requested by the host for repair in GBP." + data_tests: + - dbt_expectations.expect_column_values_to_be_between: + min_value: 0 + strictly: false + + - name: accepted_amount_in_host_currency + data_type: numeric + description: | + "Final accepted settlement amount in the host currency." + data_tests: + - dbt_expectations.expect_column_values_to_be_between: + min_value: 0 + strictly: false + + - name: accepted_amount_in_gbp + data_type: numeric + description: | + "Final accepted settlement amount in GBP." data_tests: - dbt_expectations.expect_column_values_to_be_between: min_value: 0 @@ -123,9 +150,9 @@ models: description: "Host or local currency" data_tests: - not_null: - where: "original_value_amount_in_txn_currency > 0 - or asked_repair_amount_in_txn_currency > 0 - or accepted_amount_in_txn_currency > 0" + where: "original_value_amount_in_host_currency > 0 + or asked_repair_amount_in_host_currency > 0 + or accepted_amount_in_host_currency > 0" - name: was_overriden data_type: boolean @@ -176,6 +203,14 @@ models: min_value: 0 strictly: false + - name: calculated_payout_amount_in_gbp + data_type: text + description: "Calculated payout amount in GBP." + data_tests: + - dbt_expectations.expect_column_values_to_be_between: + min_value: 0 + strictly: false + - name: calculated_guest_charge_amount_in_txn_currency data_type: text description: "Calculated guest charge amount in local currency." @@ -191,6 +226,22 @@ models: - not_null: where: "calculated_guest_charge_amount_in_txn_currency > 0" + - name: calculated_guest_charge_amount_in_usd + data_type: text + description: "Calculated guest charge amount in USD." + data_tests: + - dbt_expectations.expect_column_values_to_be_between: + min_value: 0 + strictly: false + + - name: calculated_guest_charge_amount_in_gbp + data_type: text + description: "Calculated guest charge amount in GBP." + data_tests: + - dbt_expectations.expect_column_values_to_be_between: + min_value: 0 + strictly: false + - name: id_user_host data_type: text description: | @@ -277,6 +328,14 @@ models: - not_null: where: "deposit_retained_amount_in_txn_currency > 0" + - name: deposit_retained_amount_in_gbp + data_type: numeric + description: "Amount of the deposit retained in GBP." + data_tests: + - dbt_expectations.expect_column_values_to_be_between: + min_value: 0 + strictly: false + - name: has_guest_contributed_to_cost data_type: boolean description: "If the guest has agreed to contribute to the cost @@ -303,6 +362,15 @@ models: - not_null: where: "guest_contribution_amount_in_txn_currency > 0" + - name: guest_contribution_amount_in_gbp + data_type: numeric + description: "Amount of the guest contribution, in case they did, + in GBP." + data_tests: + - dbt_expectations.expect_column_values_to_be_between: + min_value: 0 + strictly: false + - name: is_guest_contacted_about_damage data_type: boolean description: "If the host has contacted the guest about the damage caused" From 465bd51ff28e0e5c57edd253e17dd4450a7ce03f Mon Sep 17 00:00:00 2001 From: Joaquin Ossa Date: Wed, 26 Feb 2025 08:24:50 +0100 Subject: [PATCH 19/19] Final update --- .../int_resolutions__incidents.sql | 26 ++++++------------- models/intermediate/resolutions/schema.yml | 8 ------ models/staging/resolutions/schema.yml | 7 +++++ .../stg_resolutions__incidents.sql | 3 +++ 4 files changed, 18 insertions(+), 26 deletions(-) diff --git a/models/intermediate/resolutions/int_resolutions__incidents.sql b/models/intermediate/resolutions/int_resolutions__incidents.sql index 4e00548..3f31b6c 100644 --- a/models/intermediate/resolutions/int_resolutions__incidents.sql +++ b/models/intermediate/resolutions/int_resolutions__incidents.sql @@ -125,30 +125,21 @@ with int_daily_currency_exchange_rates cer_gbp on dri.created_date_utc = cer_gbp.rate_date_utc and (cm.corrective_measure ->> 'Currency') = cer_gbp.from_currency - and 'GBP' = cer.to_currency + and cer.to_currency = 'GBP' group by dri.id_incident, dri.host_currency ), guest_amounts_in_gbp as ( select i.id_incident, - (i.deposit_retained_amount_in_txn_currency * cer_deposit.rate)::decimal( + (i.guest_contribution_amount_in_txn_currency * cer.rate)::decimal( 19, 4 - ) as deposit_retained_amount_in_gbp, - ( - i.guest_contribution_amount_in_txn_currency * cer_contribution.rate - )::decimal(19, 4) as guest_contribution_amount_in_gbp + ) as guest_contribution_amount_in_gbp from stg_resolutions__incidents i - left join logs_timeline lt on i.id_incident = lt.id_incident left join - int_daily_currency_exchange_rates cer_deposit - on (lt.last_status_at_utc)::date = cer_deposit.rate_date_utc - and cer_deposit.from_currency = i.deposit_retained_currency - and cer_deposit.to_currency = 'GBP' - left join - int_daily_currency_exchange_rates cer_contribution - on (lt.last_status_at_utc)::date = cer_contribution.rate_date_utc - and cer_contribution.from_currency = i.guest_contribution_currency - and cer_contribution.to_currency = 'GBP' + int_daily_currency_exchange_rates cer + on (i.calculation_at_utc)::date = cer.rate_date_utc + and cer.from_currency = i.guest_contribution_currency + and cer.to_currency = 'GBP' ) select @@ -232,7 +223,6 @@ select i.is_guest_deposit_collected, i.deposit_retained_amount_in_txn_currency, i.deposit_retained_currency, - ga.deposit_retained_amount_in_gbp, -- Guest Involvements i.has_guest_contributed_to_cost, @@ -261,6 +251,6 @@ left join damage_report dr on i.id_incident = dr.id_incident left join guest_amounts_in_gbp ga on i.id_incident = ga.id_incident left join int_daily_currency_exchange_rates cer - on (lt.last_status_at_utc)::date = cer.rate_date_utc + on (i.calculation_at_utc)::date = cer.rate_date_utc and cer.from_currency = 'USD' and cer.to_currency = 'GBP' diff --git a/models/intermediate/resolutions/schema.yml b/models/intermediate/resolutions/schema.yml index 996b745..8b44fc0 100644 --- a/models/intermediate/resolutions/schema.yml +++ b/models/intermediate/resolutions/schema.yml @@ -328,14 +328,6 @@ models: - not_null: where: "deposit_retained_amount_in_txn_currency > 0" - - name: deposit_retained_amount_in_gbp - data_type: numeric - description: "Amount of the deposit retained in GBP." - data_tests: - - dbt_expectations.expect_column_values_to_be_between: - min_value: 0 - strictly: false - - name: has_guest_contributed_to_cost data_type: boolean description: "If the guest has agreed to contribute to the cost diff --git a/models/staging/resolutions/schema.yml b/models/staging/resolutions/schema.yml index 3ef367f..fcf6233 100644 --- a/models/staging/resolutions/schema.yml +++ b/models/staging/resolutions/schema.yml @@ -301,6 +301,13 @@ models: data_type: text description: "List of items that were damaged." + - name: calculation_at_utc + data_type: timestamp without time zone + description: "Timestamp of the calculation." + data_tests: + - not_null: + where: "calculated_payout_amount_in_txn_currency > 0" + - name: protection_name data_type: text description: "Indicates the selected booking protection plan, diff --git a/models/staging/resolutions/stg_resolutions__incidents.sql b/models/staging/resolutions/stg_resolutions__incidents.sql index 7adb483..3e2324c 100644 --- a/models/staging/resolutions/stg_resolutions__incidents.sql +++ b/models/staging/resolutions/stg_resolutions__incidents.sql @@ -194,6 +194,9 @@ select -- Calculator {{ adapter.quote("documents") }} -> 'SavedCalculation' + ->> 'CalculationDate' as calculation_at_utc, + {{ adapter.quote("documents") }} + -> 'SavedCalculation' ->> 'ProtectionName' as protection_name, ({{ adapter.quote("documents") }} -> 'SavedCalculation' ->> 'WasOverriden')::boolean as was_overriden,