From 725f2a54c3a5b724ce58550cb394ed197e9b890c Mon Sep 17 00:00:00 2001 From: Joaquin Ossa Date: Fri, 21 Feb 2025 15:25:24 +0100 Subject: [PATCH 01/15] Staging model for resolution incidents --- .../resolutions/_resolutions_sources.yml | 8 + models/staging/resolutions/schema.yml | 363 ++++++++++++++++++ .../stg_resolutions__incidents.sql | 229 +++++++++++ 3 files changed, 600 insertions(+) create mode 100644 models/staging/resolutions/_resolutions_sources.yml create mode 100644 models/staging/resolutions/schema.yml create mode 100644 models/staging/resolutions/stg_resolutions__incidents.sql diff --git a/models/staging/resolutions/_resolutions_sources.yml b/models/staging/resolutions/_resolutions_sources.yml new file mode 100644 index 0000000..ccb919a --- /dev/null +++ b/models/staging/resolutions/_resolutions_sources.yml @@ -0,0 +1,8 @@ +version: 2 + +sources: + - name: resolutions + schema: sync_cdb_resolutions + tables: + - name: incident + identifier: incident diff --git a/models/staging/resolutions/schema.yml b/models/staging/resolutions/schema.yml new file mode 100644 index 0000000..95cd508 --- /dev/null +++ b/models/staging/resolutions/schema.yml @@ -0,0 +1,363 @@ +version: 2 + +models: + - name: stg_resolutions__incidents + description: | + "Contains all incidents stored by the resolutions team in Cosmos DB. + This includes data about the incident, the user claiming the incident, the booking, + the accommodation, the host and the guest." + columns: + - name: id_incident + data_type: text + description: "Unique identifier for the incident." + data_tests: + - unique + - not_null + + - name: id_user + data_type: text + description: "Id of the user who created the incident." + data_tests: + - not_null + + - name: id_verification + data_type: text + description: "Superhog unique validation ID, coming from other APIs." + + - name: current_status_name + data_type: text + description: "current status of the incident." + data_tests: + - not_null + + - name: is_submission_complete + data_type: boolean + description: "Flag to indicate if the submission is complete." + data_tests: + - not_null + + - name: current_agent_name + data_type: text + description: "Name of the agent who is currently handling the incident." + + - name: read_only_for_customers + data_type: boolean + description: "Flag to indicate if the incident is read only for customers." + + - name: status_history_logs + data_type: text + description: "Log of the status history." + + - name: document_version + data_type: text + description: "Reported document resCentre, Api, manual form" + + - name: task_execution_logs + data_type: text + description: "" + + - name: created_at_date + data_type: timestamp without time zone + description: "Timestamp when the incident was created." + data_tests: + - not_null + + - name: created_date_date + data_type: date + description: "Date when the incident was created." + data_tests: + - not_null + + - name: updated_at_date + data_type: timestamp without time zone + description: "Timestamp when the incident was last updated." + data_tests: + - not_null + + - name: updated_date_date + data_type: date + description: "Date when the incident was last updated." + data_tests: + - not_null + + - name: id_user_host + data_type: text + description: "Unique Id that identifies the host. + Can be from SH or can be external from the client." + data_tests: + - not_null + + - name: host_user_claim_logs + data_type: text + description: "" + + - name: host_account_name + data_type: text + description: "Name of the host account." + data_tests: + - not_null + + - name: host_email + data_type: text + description: "Email of the host." + data_tests: + - not_null + + - name: host_last_name + data_type: text + description: "Last name of the host." + data_tests: + - not_null + + - name: host_first_name + data_type: text + description: "First name of the host." + data_tests: + - not_null + + - name: host_phone_code + data_type: text + description: "Phone code of the host." + + - name: host_phone_number + data_type: text + description: "Phone number of the host." + + - name: host_phone_number_with_code + data_type: text + description: "Phone number of the host with the code." + + - name: id_user_guest + data_type: text + description: "Superhog code that uniquely identifies a single guest." + data_tests: + - not_null + + - name: guest_email + data_type: text + description: "Email of the guest." + data_tests: + - not_null + + - name: guest_last_name + data_type: text + description: "Last name of the guest." + data_tests: + - not_null + + - name: guest_first_name + data_type: text + description: "First name of the guest." + data_tests: + - not_null + + - name: guest_phone_code + data_type: text + description: "Phone code of the guest." + + - name: guest_phone_number + data_type: text + description: "Phone number of the guest." + + - name: guest_phone_number_with_code + data_type: text + description: "Phone number of the guest with the code." + + - name: is_guest_deposit_retained + data_type: boolean + description: "In case a deposit has been collected from the guest, + whether it has been retained or not." + + - name: is_guest_deposit_collected + data_type: boolean + description: "If the host has collected a deposit from the guest." + + - name: deposit_retained_amount_in_txn_currency + data_type: text + description: "Amount of the deposit retained in local currency." + + - name: deposit_retained_currency + data_type: text + description: "Currency of the deposit retained." + + - name: has_guest_contributed_to_cost + data_type: boolean + description: "If the guest has agreed to contribute to the cost when + talking to the host" + + - name: has_host_taken_preventative_steps + data_type: boolean + description: "If the host has taken any action to prevent the damage + or further damage once identified" + + - name: guest_contribution_amount_in_txn_currency + data_type: text + description: "Amount of the guest contribution, in case they did, + in local currency." + + - name: guest_contribution_currency + data_type: text + description: "Currency of the guest contribution." + + - name: guest_contacted_about_damage + data_type: boolean + description: "If the host has contacted the guest about the damage caused" + + - name: guest_contacted_evidence_files + data_type: text + description: "" + + - name: guest_preventative_steps_details + data_type: text + description: "Description of any action to prevent the damage or further + damage once identified" + + - name: id_accommodation + data_type: text + description: "Superhog code that uniquely identifies a single accommodation." + data_tests: + - not_null + + - name: accommodation_url + data_type: text + description: "Accommodation URL from any distribution channel, + preferable direct website." + + - name: accommodation_name + data_type: text + description: "Accommodation name." + + - name: accommodation_address + data_type: text + description: "Address of the accommodation." + + - name: are_pets_allowed + data_type: boolean + description: "Flag to indicate if pets are allowed in the accommodation." + + - name: comment_logs + data_type: text + description: "" + + - name: has_underlying_insurance + data_type: boolean + description: "If the property has underlying insurance" + + - name: insurance_claim_submitted + data_type: boolean + description: "If an insurance claim has been submitted to an insurance company" + + - name: insurance_evidence_files + data_type: text + description: "" + + - name: check_in_at_utc + data_type: timestamp without time zone + description: "Timestamp of the check-in date in UTC of the booking." + data_tests: + - not_null + + - name: check_in_date_utc + data_type: date + description: "Date of the check-in date in UTC of the booking." + data_tests: + - not_null + + - name: check_out_at_utc + data_type: timestamp without time zone + description: "Timestamp of the check-out date in UTC of the booking." + data_tests: + - not_null + + - name: check_out_date_utc + data_type: date + description: "Date of the check-out date in UTC of the booking." + data_tests: + - not_null + + - name: id_booking + data_type: text + description: "Superhog unique booking ID" + data_tests: + - not_null + + - name: booking_status + data_type: text + description: "Status of the booking." + + - name: id_reservation + data_type: text + description: "External unique ID" + + - name: booking_details + data_type: text + description: "" + + - name: number_of_guests + data_type: text + description: "Number of guests in the booking." + + - name: booking_services + data_type: text + description: "List of services included in the booking." + + - name: booking_protection + data_type: text + description: "" + + - name: booking_platform_used + data_type: text + description: "The booking platform used" + + - name: booking_platform_reference + data_type: text + description: "" + + - name: agent_history + data_type: text + description: "" + + - name: before_damage_evidence + data_type: text + description: "" + + - name: original_invoice_evidence + data_type: text + description: "" + + - name: damage_incident_details + data_type: text + description: "Description of what happened in general terms" + + - name: has_confirmed_loss + data_type: boolean + description: "Confirmation that the information they have provided in + the report is true to the best of their knowledge." + + - name: damage_report_items + data_type: text + description: "List of items that were damaged." + + - name: third_party_claim_submitted + data_type: boolean + description: "If the host is seeking compensation from another platform" + + - name: third_party_claim_from + data_type: text + description: "For example, Aircover from Airbnb." + + - name: third_party_claim_amount_in_txn_currency + data_type: text + description: "Claim amount in local currency if the host is seeking + compensation from another platform." + + - name: third_party_claim_currency + data_type: text + description: "Currency of the claim amount if the host is seeking + compensation from another platform." + + - name: cosmos_db_timestamp_utc + data_type: timestamp + description: Internal Cosmos DB timestamp of the last record update. + data_tests: + - not_null diff --git a/models/staging/resolutions/stg_resolutions__incidents.sql b/models/staging/resolutions/stg_resolutions__incidents.sql new file mode 100644 index 0000000..7a38fa4 --- /dev/null +++ b/models/staging/resolutions/stg_resolutions__incidents.sql @@ -0,0 +1,229 @@ +with raw_incident as (select * from {{ source("resolutions", "incident") }}) +select + -- Basic Incident Details + {{ adapter.quote("documents") }} ->> 'id' as id_incident, + {{ adapter.quote("documents") }} ->> 'partitionKey' as id_user, + {{ adapter.quote("documents") }} ->> 'VerificationId' as id_verification, + {{ adapter.quote("documents") }} ->> 'CurrentStatusName' as current_status_name, + ({{ adapter.quote("documents") }} ->> 'IsSubmissionComplete')::boolean + as is_submission_complete, + {{ adapter.quote("documents") }} ->> 'CurrentAgentName' as current_agent_name, + ({{ adapter.quote("documents") }} ->> 'ReadOnlyForCustomers')::boolean + as read_only_for_customers, + {{ adapter.quote("documents") }} ->> 'StatusHistory' as status_history_logs, + {{ adapter.quote("documents") }} ->> 'documentVersion' as document_version, + {{ adapter.quote("documents") }} ->> 'TaskExecutionLogs' as task_execution_logs, + ({{ adapter.quote("documents") }} ->> 'CreatedDate')::timestamp as created_at_date, + ({{ adapter.quote("documents") }} ->> 'CreatedDate')::date as created_date_date, + ({{ adapter.quote("documents") }} ->> 'UpdatedDate')::timestamp as updated_at_date, + ({{ adapter.quote("documents") }} ->> 'UpdatedDate')::date as updated_date_date, + + -- Host Details + {{ adapter.quote("documents") }} -> 'Host' ->> 'Id' as id_user_host, + {{ adapter.quote("documents") }} -> 'Host' ->> 'UserClaims' as host_user_claim_logs, + {{ adapter.quote("documents") }} -> 'Host' ->> 'AccountName' as host_account_name, + + -- Host Contact Details + {{ adapter.quote("documents") }} + -> 'Host' + -> 'ContactDetails' + ->> 'Email' as host_email, + {{ adapter.quote("documents") }} + -> 'Host' + -> 'ContactDetails' + ->> 'LastName' as host_last_name, + {{ adapter.quote("documents") }} + -> 'Host' + -> 'ContactDetails' + ->> 'FirstName' as host_first_name, + {{ adapter.quote("documents") }} + -> 'Host' + -> 'ContactDetails' + ->> 'PhoneCode' as host_phone_code, + {{ adapter.quote("documents") }} + -> 'Host' + -> 'ContactDetails' + ->> 'PhoneNumber' as host_phone_number, + {{ adapter.quote("documents") }} + -> 'Host' + -> 'ContactDetails' + ->> 'PhoneNumberWithCode' as host_phone_number_with_code, + + -- Guest Details + {{ adapter.quote("documents") }} -> 'Guest' ->> 'Id' as id_user_guest, + {{ adapter.quote("documents") }} + -> 'Guest' + -> 'ContactDetails' + ->> 'Email' as guest_email, + {{ adapter.quote("documents") }} + -> 'Guest' + -> 'ContactDetails' + ->> 'LastName' as guest_last_name, + {{ adapter.quote("documents") }} + -> 'Guest' + -> 'ContactDetails' + ->> 'FirstName' as guest_first_name, + {{ adapter.quote("documents") }} + -> 'Guest' + -> 'ContactDetails' + ->> 'PhoneCode' as guest_phone_code, + {{ adapter.quote("documents") }} + -> 'Guest' + -> 'ContactDetails' + ->> 'PhoneNumber' as guest_phone_number, + {{ adapter.quote("documents") }} + -> 'Guest' + -> 'ContactDetails' + ->> 'PhoneNumberWithCode' as guest_phone_number_with_code, + + -- Guest Deposit Details + ({{ adapter.quote("documents") }} -> 'Guest' -> 'Deposit' ->> 'IsRetained')::boolean + as is_guest_deposit_retained, + ( + {{ adapter.quote("documents") }} -> 'Guest' -> 'Deposit' ->> 'IsCollected' + )::boolean as is_guest_deposit_collected, + {{ adapter.quote("documents") }} + -> 'Guest' + -> 'Deposit' + ->> 'RetainedAmount' as deposit_retained_amount_in_txn_currency, + {{ adapter.quote("documents") }} + -> 'Guest' + -> 'Deposit' + ->> 'RetainedCurrency' as deposit_retained_currency, + + -- Guest Involvements + ( + {{ adapter.quote("documents") }} + -> 'Guest' + -> 'Involvements' + ->> 'ContributedToCost' + )::boolean as has_guest_contributed_to_cost, + ( + {{ adapter.quote("documents") }} + -> 'Guest' + -> 'Involvements' + ->> 'PreventativeSteps' + )::boolean as has_host_taken_preventative_steps, + {{ adapter.quote("documents") }} + -> 'Guest' + -> 'Involvements' + ->> 'ContributionAmount' as guest_contribution_amount_in_txn_currency, + {{ adapter.quote("documents") }} + -> 'Guest' + -> 'Involvements' + ->> 'ContributionCurrency' as guest_contribution_currency, + ( + {{ adapter.quote("documents") }} + -> 'Guest' + -> 'Involvements' + ->> 'ContactedAboutDamage' + )::boolean as guest_contacted_about_damage, + {{ adapter.quote("documents") }} + -> 'Guest' + -> 'Involvements' + ->> 'ContactedEvidenceFiles' as guest_contacted_evidence_files, + {{ adapter.quote("documents") }} + -> 'Guest' + -> 'Involvements' + ->> 'PreventativeStepsDetails' as guest_preventative_steps_details, + + -- Accommodation Details + {{ adapter.quote("documents") }} -> 'Listing' ->> 'Id' as id_accommodation, + {{ adapter.quote("documents") }} -> 'Listing' ->> 'Url' as accommodation_url, + {{ adapter.quote("documents") }} -> 'Listing' ->> 'Name' as accommodation_name, + {{ adapter.quote("documents") }} + -> 'Listing' + ->> 'Address' as accommodation_address, + ({{ adapter.quote("documents") }} -> 'Listing' ->> 'PetsAllowed')::boolean + as are_pets_allowed, + + -- Comments + {{ adapter.quote("documents") }} ->> 'Comments' as comment_logs, + + -- Insurance Details + ({{ adapter.quote("documents") }} -> 'Insurance' ->> 'UnderlyingInsurance')::boolean + as has_underlying_insurance, + ({{ adapter.quote("documents") }} -> 'Insurance' ->> 'ClaimSubmitted')::boolean + as insurance_claim_submitted, + {{ adapter.quote("documents") }} + -> 'Insurance' + ->> 'UnderlyingInsuranceEvidenceFiles' as insurance_evidence_files, + + -- Booking Details + ({{ adapter.quote("documents") }} -> 'Reservation' ->> 'CheckIn')::timestamp + as check_in_at_utc, + ({{ adapter.quote("documents") }} -> 'Reservation' ->> 'CheckIn')::date + as check_in_date_utc, + ({{ adapter.quote("documents") }} -> 'Reservation' ->> 'CheckOut')::timestamp + as check_out_at_utc, + ({{ adapter.quote("documents") }} -> 'Reservation' ->> 'CheckOut')::date + as check_out_date_utc, + {{ adapter.quote("documents") }} -> 'Reservation' ->> 'BookingId' as id_booking, + {{ adapter.quote("documents") }} + -> 'Reservation' + ->> 'BookingStatus' as booking_status, + {{ adapter.quote("documents") }} + -> 'Reservation' + ->> 'ReservationId' as id_reservation, + {{ adapter.quote("documents") }} + -> 'Reservation' + ->> 'BookingDetails' as booking_details, + {{ adapter.quote("documents") }} + -> 'Reservation' + ->> 'NumberOfGuests' as number_of_guests, + {{ adapter.quote("documents") }} + -> 'Reservation' + ->> 'BookingServices' as booking_services, + {{ adapter.quote("documents") }} + -> 'Reservation' + ->> 'BookingProtection' as booking_protection, + {{ adapter.quote("documents") }} + -> 'Reservation' + ->> 'BookingPlatformUsed' as booking_platform_used, + {{ adapter.quote("documents") }} + -> 'Reservation' + ->> 'BookingPlatformReference' as booking_platform_reference, + + -- Agent History + {{ adapter.quote("documents") }} + -> 'AgentHistory' + ->> 'AgentHistory' as agent_history, + + -- Damage Report + {{ adapter.quote("documents") }} + -> 'BeforeDamageEvidence' + ->> 'BeforeDamageEvidence' as before_damage_evidence, + {{ adapter.quote("documents") }} + -> 'OriginalInvoiceEvidence' + ->> 'OriginalInvoiceEvidence' as original_invoice_evidence, + {{ adapter.quote("documents") }} + -> 'DamageReport' + ->> 'IncidentDetails' as damage_incident_details, + ( + {{ adapter.quote("documents") }} -> 'DamageReport' ->> 'DeclarationOfLoss' + )::boolean as has_confirmed_loss, + {{ adapter.quote("documents") }} + -> 'DamageReport' + ->> 'Items' as damage_report_items, + + -- Third Party Compensation Claim + ( + {{ adapter.quote("documents") }} + -> 'ThirdPartyCompensationClaim' + ->> 'ClaimSubmitted' + )::boolean as third_party_claim_submitted, + {{ adapter.quote("documents") }} + -> 'ThirdPartyCompensationClaim' + ->> 'From' as third_party_claim_from, + {{ adapter.quote("documents") }} + -> 'ThirdPartyCompensationClaim' + ->> 'Amount' as third_party_claim_amount_in_txn_currency, + {{ adapter.quote("documents") }} + -> 'ThirdPartyCompensationClaim' + ->> 'Currency' as third_party_claim_currency, + + to_timestamp( + (({{ adapter.quote("documents") }} ->> '_ts'))::integer + ) as cosmos_db_timestamp_utc + +from raw_incident From 1b853a95034642830a0a2ce9385493d0a18daece Mon Sep 17 00:00:00 2001 From: Joaquin Ossa Date: Fri, 21 Feb 2025 16:40:53 +0100 Subject: [PATCH 02/15] commit wip (id_incident duplicates have to check) --- models/staging/resolutions/schema.yml | 36 ++++++++++++++++++--------- 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/models/staging/resolutions/schema.yml b/models/staging/resolutions/schema.yml index 95cd508..4ee8728 100644 --- a/models/staging/resolutions/schema.yml +++ b/models/staging/resolutions/schema.yml @@ -60,25 +60,29 @@ models: data_type: timestamp without time zone description: "Timestamp when the incident was created." data_tests: - - not_null + - not_null: + where: "document_version <> 'OldClaimToIncidentModel'" - name: created_date_date data_type: date description: "Date when the incident was created." data_tests: - - not_null + - not_null: + where: "document_version <> 'OldClaimToIncidentModel'" - name: updated_at_date data_type: timestamp without time zone description: "Timestamp when the incident was last updated." data_tests: - - not_null + - not_null: + where: "document_version <> 'OldClaimToIncidentModel'" - name: updated_date_date data_type: date description: "Date when the incident was last updated." data_tests: - - not_null + - not_null: + where: "document_version <> 'OldClaimToIncidentModel'" - name: id_user_host data_type: text @@ -95,7 +99,8 @@ models: data_type: text description: "Name of the host account." data_tests: - - not_null + - not_null: + where: "document_version <> 'OldClaimToIncidentModel'" - name: host_email data_type: text @@ -107,13 +112,15 @@ models: data_type: text description: "Last name of the host." data_tests: - - not_null + - not_null: + where: "document_version <> 'OldClaimToIncidentModel'" - name: host_first_name data_type: text description: "First name of the host." data_tests: - - not_null + - not_null: + where: "document_version <> 'OldClaimToIncidentModel'" - name: host_phone_code data_type: text @@ -131,25 +138,29 @@ models: data_type: text description: "Superhog code that uniquely identifies a single guest." data_tests: - - not_null + - not_null: + where: "document_version <> 'OldClaimToIncidentModel'" - name: guest_email data_type: text description: "Email of the guest." data_tests: - - not_null + - not_null: + where: "document_version <> 'OldClaimToIncidentModel'" - name: guest_last_name data_type: text description: "Last name of the guest." data_tests: - - not_null + - not_null: + where: "document_version <> 'OldClaimToIncidentModel'" - name: guest_first_name data_type: text description: "First name of the guest." data_tests: - - not_null + - not_null: + where: "document_version <> 'OldClaimToIncidentModel'" - name: guest_phone_code data_type: text @@ -279,7 +290,8 @@ models: data_type: text description: "Superhog unique booking ID" data_tests: - - not_null + - not_null: + where: "document_version <> 'OldClaimToIncidentModel'" - name: booking_status data_type: text From 05adc07674b7dbd0b704a0ce2ec133205df27e45 Mon Sep 17 00:00:00 2001 From: Joaquin Ossa Date: Fri, 21 Feb 2025 17:44:00 +0100 Subject: [PATCH 03/15] commit wip --- models/staging/resolutions/schema.yml | 49 +++++++++++++++++++ .../stg_resolutions__incidents.sql | 37 ++++++++++++++ 2 files changed, 86 insertions(+) diff --git a/models/staging/resolutions/schema.yml b/models/staging/resolutions/schema.yml index 4ee8728..c6b9f3d 100644 --- a/models/staging/resolutions/schema.yml +++ b/models/staging/resolutions/schema.yml @@ -350,6 +350,55 @@ models: data_type: text description: "List of items that were damaged." + - name: protection_name + data_type: text + description: "Protection name." + + - name: was_overriden + data_type: boolean + description: "Flag to indicate if the proposed settlement value from + the calculator was overriden." + + - name: waiver_or_deposit_name + data_type: text + description: "Waiver or deposit name." + + - name: guest_verification_status + data_type: text + description: "Guest verification status for this booking." + + - name: lower_protection_limit_usd + data_type: text + description: "Lower protection limit in USD." + + - name: upper_protection_limit_usd + data_type: numeric + description: "Upper protection limit in USD." + + - name: calculated_payout_amount_in_txn_currency + data_type: numeric + description: "Calculated payout amount in local currency." + + - name: calculated_payout_currency + data_type: text + description: "Currency of the calculated payout amount." + + - name: calculated_payout_amount_in_usd + data_type: numeric + description: "Calculated payout amount in USD." + + - name: calculated_guest_charge_amount_in_txn_currency + data_type: numeric + description: "Calculated guest charge amount in local currency." + + - name: calculated_guest_charge_currency + data_type: text + description: "Currency of the calculated guest charge amount." + + - name: calculated_guest_charge_amount_in_usd + data_type: numeric + description: "Calculated guest charge amount in USD." + - name: third_party_claim_submitted data_type: boolean description: "If the host is seeking compensation from another platform" diff --git a/models/staging/resolutions/stg_resolutions__incidents.sql b/models/staging/resolutions/stg_resolutions__incidents.sql index 7a38fa4..57a9035 100644 --- a/models/staging/resolutions/stg_resolutions__incidents.sql +++ b/models/staging/resolutions/stg_resolutions__incidents.sql @@ -206,6 +206,43 @@ select -> 'DamageReport' ->> 'Items' as damage_report_items, + -- Calculator + {{ adapter.quote("documents") }} + -> 'SavedCalculation' + ->> 'ProtectionName' as protection_name, + ({{ adapter.quote("documents") }} -> 'SavedCalculation' ->> 'WasOverriden')::boolean + as was_overriden, + {{ adapter.quote("documents") }} + -> 'SavedCalculation' + ->> 'WaiverOrDepositName' as waiver_or_deposit_name, + {{ adapter.quote("documents") }} + -> 'SavedCalculation' + ->> 'GuestVerificationStatus' as guest_verification_status, + {{ adapter.quote("documents") }} + -> 'SavedCalculation' + ->> 'LowerProtectionLimitUsd' as lower_protection_limit_usd, + {{ adapter.quote("documents") }} + -> 'SavedCalculation' + ->> 'UpperProtectionLimitUsd' as upper_protection_limit_usd, + {{ adapter.quote("documents") }} + -> 'SavedCalculation' + ->> 'SubmittedHostPayoutAmount' as calculated_payout_amount_in_txn_currency, + {{ adapter.quote("documents") }} + -> 'SavedCalculation' + ->> 'SubmittedHostPayoutCurrency' as calculated_payout_currency, + {{ adapter.quote("documents") }} + -> 'SavedCalculation' + ->> 'CalculatedHostPayoutAmountInUsd' as calculated_payout_amount_in_usd, + {{ adapter.quote("documents") }} + -> 'SavedCalculation' + ->> 'SubmittedGuestChargeAmount' as calculated_guest_charge_amount_in_txn_currency, + {{ adapter.quote("documents") }} + -> 'SavedCalculation' + ->> 'SubmittedGuestChargeCurrency' as calculated_guest_charge_currency, + {{ adapter.quote("documents") }} + -> 'SavedCalculation' + ->> 'CalculatedGuestChargeAmountInUsd' as calculated_guest_charge_amount_in_usd, + -- Third Party Compensation Claim ( {{ adapter.quote("documents") }} From 9704426dd64258eb33af926d5534358f3ff453da Mon Sep 17 00:00:00 2001 From: Joaquin Ossa Date: Sat, 22 Feb 2025 14:16:41 +0100 Subject: [PATCH 04/15] commit wip --- models/staging/resolutions/schema.yml | 23 +++++++++++++++---- .../stg_resolutions__incidents.sql | 9 +++++--- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/models/staging/resolutions/schema.yml b/models/staging/resolutions/schema.yml index c6b9f3d..36edff6 100644 --- a/models/staging/resolutions/schema.yml +++ b/models/staging/resolutions/schema.yml @@ -4,8 +4,11 @@ models: - name: stg_resolutions__incidents description: | "Contains all incidents stored by the resolutions team in Cosmos DB. - This includes data about the incident, the user claiming the incident, the booking, - the accommodation, the host and the guest." + This includes data about the incident, the user claiming the incident, the booking, + the accommodation, the host and the guest. + Some records (document_version <> 'OldClaimToIncidentModel') are older versions + of the incident model and don't have the same data structure as the current one. + Many of them don't have the same necessary fields as the current model." columns: - name: id_incident data_type: text @@ -90,6 +93,9 @@ models: Can be from SH or can be external from the client." data_tests: - not_null + - relationships: + to: ref('stg_core__user') + field: id_user - name: host_user_claim_logs data_type: text @@ -140,6 +146,9 @@ models: data_tests: - not_null: where: "document_version <> 'OldClaimToIncidentModel'" + - relationships: + to: ref('stg_core__user') + field: id_user - name: guest_email data_type: text @@ -224,10 +233,13 @@ models: damage once identified" - name: id_accommodation - data_type: text + data_type: numeric description: "Superhog code that uniquely identifies a single accommodation." data_tests: - not_null + - relationships: + to: ref('stg_core__accommodation') + field: id_accommodation - name: accommodation_url data_type: text @@ -287,11 +299,14 @@ models: - not_null - name: id_booking - data_type: text + data_type: numeric description: "Superhog unique booking ID" data_tests: - not_null: where: "document_version <> 'OldClaimToIncidentModel'" + - relationships: + to: ref('stg_core__booking') + field: id_booking - name: booking_status data_type: text diff --git a/models/staging/resolutions/stg_resolutions__incidents.sql b/models/staging/resolutions/stg_resolutions__incidents.sql index 57a9035..936335d 100644 --- a/models/staging/resolutions/stg_resolutions__incidents.sql +++ b/models/staging/resolutions/stg_resolutions__incidents.sql @@ -50,7 +50,7 @@ select ->> 'PhoneNumberWithCode' as host_phone_number_with_code, -- Guest Details - {{ adapter.quote("documents") }} -> 'Guest' ->> 'Id' as id_user_guest, + nullif({{ adapter.quote("documents") }} -> 'Guest' ->> 'Id', '') as id_user_guest, {{ adapter.quote("documents") }} -> 'Guest' -> 'ContactDetails' @@ -128,7 +128,8 @@ select ->> 'PreventativeStepsDetails' as guest_preventative_steps_details, -- Accommodation Details - {{ adapter.quote("documents") }} -> 'Listing' ->> 'Id' as id_accommodation, + (nullif({{ adapter.quote("documents") }} -> 'Listing' ->> 'Id', ''))::numeric + as id_accommodation, {{ adapter.quote("documents") }} -> 'Listing' ->> 'Url' as accommodation_url, {{ adapter.quote("documents") }} -> 'Listing' ->> 'Name' as accommodation_name, {{ adapter.quote("documents") }} @@ -158,7 +159,9 @@ select as check_out_at_utc, ({{ adapter.quote("documents") }} -> 'Reservation' ->> 'CheckOut')::date as check_out_date_utc, - {{ adapter.quote("documents") }} -> 'Reservation' ->> 'BookingId' as id_booking, + nullif( + ({{ adapter.quote("documents") }} -> 'Reservation' ->> 'BookingId'), '' + )::numeric as id_booking, {{ adapter.quote("documents") }} -> 'Reservation' ->> 'BookingStatus' as booking_status, From f3842c950b7a971b93c10b404262c23b2b6b1763 Mon Sep 17 00:00:00 2001 From: Joaquin Ossa Date: Sat, 22 Feb 2025 19:55:30 +0100 Subject: [PATCH 05/15] commit wip --- models/staging/resolutions/schema.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/models/staging/resolutions/schema.yml b/models/staging/resolutions/schema.yml index 36edff6..d9eb962 100644 --- a/models/staging/resolutions/schema.yml +++ b/models/staging/resolutions/schema.yml @@ -96,6 +96,7 @@ models: - relationships: to: ref('stg_core__user') field: id_user + where: "document_version <> 'OldClaimToIncidentModel'" - name: host_user_claim_logs data_type: text @@ -149,6 +150,7 @@ models: - relationships: to: ref('stg_core__user') field: id_user + where: "document_version <> 'OldClaimToIncidentModel'" - name: guest_email data_type: text @@ -240,6 +242,7 @@ models: - relationships: to: ref('stg_core__accommodation') field: id_accommodation + where: "document_version <> 'OldClaimToIncidentModel'" - name: accommodation_url data_type: text @@ -307,6 +310,7 @@ models: - relationships: to: ref('stg_core__booking') field: id_booking + where: "document_version <> 'OldClaimToIncidentModel'" - name: booking_status data_type: text From d6dcbcc67eba38db6a6420d97c3211f7714b7819 Mon Sep 17 00:00:00 2001 From: Joaquin Ossa Date: Sun, 23 Feb 2025 15:04:59 +0100 Subject: [PATCH 06/15] commit wip --- models/staging/resolutions/schema.yml | 8 ++++---- models/staging/resolutions/stg_resolutions__incidents.sql | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/models/staging/resolutions/schema.yml b/models/staging/resolutions/schema.yml index d9eb962..442287f 100644 --- a/models/staging/resolutions/schema.yml +++ b/models/staging/resolutions/schema.yml @@ -59,28 +59,28 @@ models: data_type: text description: "" - - name: created_at_date + - name: created_at_utc data_type: timestamp without time zone description: "Timestamp when the incident was created." data_tests: - not_null: where: "document_version <> 'OldClaimToIncidentModel'" - - name: created_date_date + - name: created_date_utc data_type: date description: "Date when the incident was created." data_tests: - not_null: where: "document_version <> 'OldClaimToIncidentModel'" - - name: updated_at_date + - name: updated_at_utc data_type: timestamp without time zone description: "Timestamp when the incident was last updated." data_tests: - not_null: where: "document_version <> 'OldClaimToIncidentModel'" - - name: updated_date_date + - name: updated_date_utc data_type: date description: "Date when the incident was last updated." data_tests: diff --git a/models/staging/resolutions/stg_resolutions__incidents.sql b/models/staging/resolutions/stg_resolutions__incidents.sql index 936335d..a5202cd 100644 --- a/models/staging/resolutions/stg_resolutions__incidents.sql +++ b/models/staging/resolutions/stg_resolutions__incidents.sql @@ -13,10 +13,10 @@ select {{ adapter.quote("documents") }} ->> 'StatusHistory' as status_history_logs, {{ adapter.quote("documents") }} ->> 'documentVersion' as document_version, {{ adapter.quote("documents") }} ->> 'TaskExecutionLogs' as task_execution_logs, - ({{ adapter.quote("documents") }} ->> 'CreatedDate')::timestamp as created_at_date, - ({{ adapter.quote("documents") }} ->> 'CreatedDate')::date as created_date_date, - ({{ adapter.quote("documents") }} ->> 'UpdatedDate')::timestamp as updated_at_date, - ({{ adapter.quote("documents") }} ->> 'UpdatedDate')::date as updated_date_date, + ({{ adapter.quote("documents") }} ->> 'CreatedDate')::timestamp as created_at_utc, + ({{ adapter.quote("documents") }} ->> 'CreatedDate')::date as created_date_utc, + ({{ adapter.quote("documents") }} ->> 'UpdatedDate')::timestamp as updated_at_utc, + ({{ adapter.quote("documents") }} ->> 'UpdatedDate')::date as updated_date_utc, -- Host Details {{ adapter.quote("documents") }} -> 'Host' ->> 'Id' as id_user_host, From fc85dd9e70b6c7899f6e57718c96f83b4fc6c62e Mon Sep 17 00:00:00 2001 From: Joaquin Ossa Date: Sun, 23 Feb 2025 16:05:11 +0100 Subject: [PATCH 07/15] Fixed data_types and added tests --- models/staging/resolutions/schema.yml | 59 ++++++++++++-- .../stg_resolutions__incidents.sql | 76 +++++++++++-------- 2 files changed, 98 insertions(+), 37 deletions(-) diff --git a/models/staging/resolutions/schema.yml b/models/staging/resolutions/schema.yml index 442287f..6e587f1 100644 --- a/models/staging/resolutions/schema.yml +++ b/models/staging/resolutions/schema.yml @@ -195,8 +195,12 @@ models: description: "If the host has collected a deposit from the guest." - name: deposit_retained_amount_in_txn_currency - data_type: text + data_type: numeric description: "Amount of the deposit retained in local currency." + data_tests: + - dbt_expectations.expect_column_values_to_be_between: + min_value: 0 + strictly: false - name: deposit_retained_currency data_type: text @@ -213,9 +217,13 @@ models: or further damage once identified" - name: guest_contribution_amount_in_txn_currency - data_type: text + data_type: numeric description: "Amount of the guest contribution, in case they did, in local currency." + data_tests: + - dbt_expectations.expect_column_values_to_be_between: + min_value: 0 + strictly: false - name: guest_contribution_currency data_type: text @@ -325,8 +333,12 @@ models: description: "" - name: number_of_guests - data_type: text + data_type: numeric description: "Number of guests in the booking." + data_tests: + - dbt_expectations.expect_column_values_to_be_between: + min_value: 0 + strictly: false - name: booking_services data_type: text @@ -387,36 +399,66 @@ models: description: "Guest verification status for this booking." - name: lower_protection_limit_usd - data_type: text + data_type: numeric description: "Lower protection limit in USD." + data_tests: + - dbt_expectations.expect_column_values_to_be_between: + min_value: 0 + strictly: false - name: upper_protection_limit_usd data_type: numeric description: "Upper protection limit in USD." + data_tests: + - dbt_expectations.expect_column_values_to_be_between: + min_value: 0 + strictly: false - name: calculated_payout_amount_in_txn_currency data_type: numeric description: "Calculated payout amount in local currency." + data_tests: + - dbt_expectations.expect_column_values_to_be_between: + min_value: 0 + strictly: false - name: calculated_payout_currency data_type: text description: "Currency of the calculated payout amount." + data_tests: + - not_null: + where: "calculated_payout_amount_in_txn_currency > 0" - name: calculated_payout_amount_in_usd data_type: numeric description: "Calculated payout amount in USD." + data_tests: + - dbt_expectations.expect_column_values_to_be_between: + min_value: 0 + strictly: false - name: calculated_guest_charge_amount_in_txn_currency data_type: numeric description: "Calculated guest charge amount in local currency." + data_tests: + - dbt_expectations.expect_column_values_to_be_between: + min_value: 0 + strictly: false - name: calculated_guest_charge_currency data_type: text description: "Currency of the calculated guest charge amount." + data_tests: + - not_null: + where: "calculated_guest_charge_amount_in_txn_currency > 0" - name: calculated_guest_charge_amount_in_usd data_type: numeric description: "Calculated guest charge amount in USD." + data_tests: + - dbt_expectations.expect_column_values_to_be_between: + min_value: 0 + strictly: false - name: third_party_claim_submitted data_type: boolean @@ -427,14 +469,21 @@ models: description: "For example, Aircover from Airbnb." - name: third_party_claim_amount_in_txn_currency - data_type: text + data_type: numeric description: "Claim amount in local currency if the host is seeking compensation from another platform." + data_tests: + - dbt_expectations.expect_column_values_to_be_between: + min_value: 0 + strictly: false - name: third_party_claim_currency data_type: text description: "Currency of the claim amount if the host is seeking compensation from another platform." + data_tests: + - not_null: + where: "third_party_claim_amount_in_txn_currency > 0" - name: cosmos_db_timestamp_utc data_type: timestamp diff --git a/models/staging/resolutions/stg_resolutions__incidents.sql b/models/staging/resolutions/stg_resolutions__incidents.sql index a5202cd..b5f9346 100644 --- a/models/staging/resolutions/stg_resolutions__incidents.sql +++ b/models/staging/resolutions/stg_resolutions__incidents.sql @@ -82,10 +82,9 @@ select ( {{ adapter.quote("documents") }} -> 'Guest' -> 'Deposit' ->> 'IsCollected' )::boolean as is_guest_deposit_collected, - {{ adapter.quote("documents") }} - -> 'Guest' - -> 'Deposit' - ->> 'RetainedAmount' as deposit_retained_amount_in_txn_currency, + ( + {{ adapter.quote("documents") }} -> 'Guest' -> 'Deposit' ->> 'RetainedAmount' + )::numeric as deposit_retained_amount_in_txn_currency, {{ adapter.quote("documents") }} -> 'Guest' -> 'Deposit' @@ -104,10 +103,12 @@ select -> 'Involvements' ->> 'PreventativeSteps' )::boolean as has_host_taken_preventative_steps, - {{ adapter.quote("documents") }} - -> 'Guest' - -> 'Involvements' - ->> 'ContributionAmount' as guest_contribution_amount_in_txn_currency, + ( + {{ adapter.quote("documents") }} + -> 'Guest' + -> 'Involvements' + ->> 'ContributionAmount' + )::numeric as guest_contribution_amount_in_txn_currency, {{ adapter.quote("documents") }} -> 'Guest' -> 'Involvements' @@ -171,9 +172,8 @@ select {{ adapter.quote("documents") }} -> 'Reservation' ->> 'BookingDetails' as booking_details, - {{ adapter.quote("documents") }} - -> 'Reservation' - ->> 'NumberOfGuests' as number_of_guests, + ({{ adapter.quote("documents") }} -> 'Reservation' ->> 'NumberOfGuests')::numeric + as number_of_guests, {{ adapter.quote("documents") }} -> 'Reservation' ->> 'BookingServices' as booking_services, @@ -221,30 +221,42 @@ select {{ adapter.quote("documents") }} -> 'SavedCalculation' ->> 'GuestVerificationStatus' as guest_verification_status, - {{ adapter.quote("documents") }} - -> 'SavedCalculation' - ->> 'LowerProtectionLimitUsd' as lower_protection_limit_usd, - {{ adapter.quote("documents") }} - -> 'SavedCalculation' - ->> 'UpperProtectionLimitUsd' as upper_protection_limit_usd, - {{ adapter.quote("documents") }} - -> 'SavedCalculation' - ->> 'SubmittedHostPayoutAmount' as calculated_payout_amount_in_txn_currency, + ( + {{ adapter.quote("documents") }} + -> 'SavedCalculation' + ->> 'LowerProtectionLimitUsd' + )::numeric as lower_protection_limit_usd, + ( + {{ adapter.quote("documents") }} + -> 'SavedCalculation' + ->> 'UpperProtectionLimitUsd' + )::numeric as upper_protection_limit_usd, + ( + {{ adapter.quote("documents") }} + -> 'SavedCalculation' + ->> 'SubmittedHostPayoutAmount' + )::numeric as calculated_payout_amount_in_txn_currency, {{ adapter.quote("documents") }} -> 'SavedCalculation' ->> 'SubmittedHostPayoutCurrency' as calculated_payout_currency, - {{ adapter.quote("documents") }} - -> 'SavedCalculation' - ->> 'CalculatedHostPayoutAmountInUsd' as calculated_payout_amount_in_usd, - {{ adapter.quote("documents") }} - -> 'SavedCalculation' - ->> 'SubmittedGuestChargeAmount' as calculated_guest_charge_amount_in_txn_currency, + ( + {{ adapter.quote("documents") }} + -> 'SavedCalculation' + ->> 'CalculatedHostPayoutAmountInUsd' + )::numeric as calculated_payout_amount_in_usd, + ( + {{ adapter.quote("documents") }} + -> 'SavedCalculation' + ->> 'SubmittedGuestChargeAmount' + )::numeric as calculated_guest_charge_amount_in_txn_currency, {{ adapter.quote("documents") }} -> 'SavedCalculation' ->> 'SubmittedGuestChargeCurrency' as calculated_guest_charge_currency, - {{ adapter.quote("documents") }} - -> 'SavedCalculation' - ->> 'CalculatedGuestChargeAmountInUsd' as calculated_guest_charge_amount_in_usd, + ( + {{ adapter.quote("documents") }} + -> 'SavedCalculation' + ->> 'CalculatedGuestChargeAmountInUsd' + )::numeric as calculated_guest_charge_amount_in_usd, -- Third Party Compensation Claim ( @@ -255,9 +267,9 @@ select {{ adapter.quote("documents") }} -> 'ThirdPartyCompensationClaim' ->> 'From' as third_party_claim_from, - {{ adapter.quote("documents") }} - -> 'ThirdPartyCompensationClaim' - ->> 'Amount' as third_party_claim_amount_in_txn_currency, + ( + {{ adapter.quote("documents") }} -> 'ThirdPartyCompensationClaim' ->> 'Amount' + )::numeric as third_party_claim_amount_in_txn_currency, {{ adapter.quote("documents") }} -> 'ThirdPartyCompensationClaim' ->> 'Currency' as third_party_claim_currency, From fb4775bd071c9e25c8aef24d0058d58d51107429 Mon Sep 17 00:00:00 2001 From: Joaquin Ossa Date: Sun, 23 Feb 2025 16:11:37 +0100 Subject: [PATCH 08/15] Updated tests --- models/staging/resolutions/schema.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/models/staging/resolutions/schema.yml b/models/staging/resolutions/schema.yml index 6e587f1..ed62f40 100644 --- a/models/staging/resolutions/schema.yml +++ b/models/staging/resolutions/schema.yml @@ -205,6 +205,9 @@ models: - name: deposit_retained_currency data_type: text description: "Currency of the deposit retained." + data_tests: + - not_null: + where: "deposit_retained_amount_in_txn_currency > 0 and document_version <> 'OldClaimToIncidentModel'" - name: has_guest_contributed_to_cost data_type: boolean @@ -228,6 +231,9 @@ models: - name: guest_contribution_currency data_type: text description: "Currency of the guest contribution." + data_tests: + - not_null: + where: "guest_contribution_amount_in_txn_currency > 0 and document_version <> 'OldClaimToIncidentModel'" - name: guest_contacted_about_damage data_type: boolean @@ -483,7 +489,7 @@ models: compensation from another platform." data_tests: - not_null: - where: "third_party_claim_amount_in_txn_currency > 0" + where: "third_party_claim_amount_in_txn_currency > 0 and document_version <> 'OldClaimToIncidentModel'" - name: cosmos_db_timestamp_utc data_type: timestamp From 2e19678182285cd4b0c958986a8de4eb91b96b67 Mon Sep 17 00:00:00 2001 From: Joaquin Ossa Date: Sun, 23 Feb 2025 16:39:43 +0100 Subject: [PATCH 09/15] Final tests update --- models/staging/resolutions/schema.yml | 21 ------------------- .../stg_resolutions__incidents.sql | 2 +- 2 files changed, 1 insertion(+), 22 deletions(-) diff --git a/models/staging/resolutions/schema.yml b/models/staging/resolutions/schema.yml index ed62f40..ef74b6e 100644 --- a/models/staging/resolutions/schema.yml +++ b/models/staging/resolutions/schema.yml @@ -76,16 +76,10 @@ models: - name: updated_at_utc data_type: timestamp without time zone description: "Timestamp when the incident was last updated." - data_tests: - - not_null: - where: "document_version <> 'OldClaimToIncidentModel'" - name: updated_date_utc data_type: date description: "Date when the incident was last updated." - data_tests: - - not_null: - where: "document_version <> 'OldClaimToIncidentModel'" - name: id_user_host data_type: text @@ -105,9 +99,6 @@ models: - name: host_account_name data_type: text description: "Name of the host account." - data_tests: - - not_null: - where: "document_version <> 'OldClaimToIncidentModel'" - name: host_email data_type: text @@ -118,16 +109,10 @@ models: - name: host_last_name data_type: text description: "Last name of the host." - data_tests: - - not_null: - where: "document_version <> 'OldClaimToIncidentModel'" - name: host_first_name data_type: text description: "First name of the host." - data_tests: - - not_null: - where: "document_version <> 'OldClaimToIncidentModel'" - name: host_phone_code data_type: text @@ -162,16 +147,10 @@ models: - name: guest_last_name data_type: text description: "Last name of the guest." - data_tests: - - not_null: - where: "document_version <> 'OldClaimToIncidentModel'" - name: guest_first_name data_type: text description: "First name of the guest." - data_tests: - - not_null: - where: "document_version <> 'OldClaimToIncidentModel'" - name: guest_phone_code data_type: text diff --git a/models/staging/resolutions/stg_resolutions__incidents.sql b/models/staging/resolutions/stg_resolutions__incidents.sql index b5f9346..173468b 100644 --- a/models/staging/resolutions/stg_resolutions__incidents.sql +++ b/models/staging/resolutions/stg_resolutions__incidents.sql @@ -11,7 +11,7 @@ select ({{ adapter.quote("documents") }} ->> 'ReadOnlyForCustomers')::boolean as read_only_for_customers, {{ adapter.quote("documents") }} ->> 'StatusHistory' as status_history_logs, - {{ adapter.quote("documents") }} ->> 'documentVersion' as document_version, + {{ adapter.quote("documents") }} ->> 'DocumentVersion' as document_version, {{ adapter.quote("documents") }} ->> 'TaskExecutionLogs' as task_execution_logs, ({{ adapter.quote("documents") }} ->> 'CreatedDate')::timestamp as created_at_utc, ({{ adapter.quote("documents") }} ->> 'CreatedDate')::date as created_date_utc, From 62c2e44b86ff9ce1fa0a98eeac9f5b4762186ff4 Mon Sep 17 00:00:00 2001 From: Joaquin Ossa Date: Mon, 24 Feb 2025 10:57:46 +0100 Subject: [PATCH 10/15] Deduplicated records --- models/staging/resolutions/stg_resolutions__incidents.sql | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/models/staging/resolutions/stg_resolutions__incidents.sql b/models/staging/resolutions/stg_resolutions__incidents.sql index 173468b..a1efe8d 100644 --- a/models/staging/resolutions/stg_resolutions__incidents.sql +++ b/models/staging/resolutions/stg_resolutions__incidents.sql @@ -1,4 +1,8 @@ -with raw_incident as (select * from {{ source("resolutions", "incident") }}) +with + raw_incident as (select * from {{ source("resolutions", "incident") }}), + deduped_verifications as ( + {{ cosmos_db_record_deduplication("raw_incident", "id") }} + ) select -- Basic Incident Details {{ adapter.quote("documents") }} ->> 'id' as id_incident, @@ -278,4 +282,4 @@ select (({{ adapter.quote("documents") }} ->> '_ts'))::integer ) as cosmos_db_timestamp_utc -from raw_incident +from deduped_verifications From 34f8fbe499169084a90ab9243615adf28987e0e1 Mon Sep 17 00:00:00 2001 From: Joaquin Ossa Date: Mon, 24 Feb 2025 15:06:55 +0100 Subject: [PATCH 11/15] commit wip --- models/staging/resolutions/schema.yml | 29 +++++-------------- .../stg_resolutions__incidents.sql | 9 ++++++ 2 files changed, 17 insertions(+), 21 deletions(-) diff --git a/models/staging/resolutions/schema.yml b/models/staging/resolutions/schema.yml index ef74b6e..d21bfa0 100644 --- a/models/staging/resolutions/schema.yml +++ b/models/staging/resolutions/schema.yml @@ -5,10 +5,7 @@ models: description: | "Contains all incidents stored by the resolutions team in Cosmos DB. This includes data about the incident, the user claiming the incident, the booking, - the accommodation, the host and the guest. - Some records (document_version <> 'OldClaimToIncidentModel') are older versions - of the incident model and don't have the same data structure as the current one. - Many of them don't have the same necessary fields as the current model." + the accommodation, the host and the guest." columns: - name: id_incident data_type: text @@ -63,15 +60,13 @@ models: data_type: timestamp without time zone description: "Timestamp when the incident was created." data_tests: - - not_null: - where: "document_version <> 'OldClaimToIncidentModel'" + - not_null - name: created_date_utc data_type: date description: "Date when the incident was created." data_tests: - - not_null: - where: "document_version <> 'OldClaimToIncidentModel'" + - not_null - name: updated_at_utc data_type: timestamp without time zone @@ -90,7 +85,6 @@ models: - relationships: to: ref('stg_core__user') field: id_user - where: "document_version <> 'OldClaimToIncidentModel'" - name: host_user_claim_logs data_type: text @@ -130,19 +124,15 @@ models: data_type: text description: "Superhog code that uniquely identifies a single guest." data_tests: - - not_null: - where: "document_version <> 'OldClaimToIncidentModel'" - relationships: to: ref('stg_core__user') field: id_user - where: "document_version <> 'OldClaimToIncidentModel'" - name: guest_email data_type: text description: "Email of the guest." data_tests: - - not_null: - where: "document_version <> 'OldClaimToIncidentModel'" + - not_null - name: guest_last_name data_type: text @@ -186,7 +176,7 @@ models: description: "Currency of the deposit retained." data_tests: - not_null: - where: "deposit_retained_amount_in_txn_currency > 0 and document_version <> 'OldClaimToIncidentModel'" + where: "deposit_retained_amount_in_txn_currency > 0" - name: has_guest_contributed_to_cost data_type: boolean @@ -212,7 +202,7 @@ models: description: "Currency of the guest contribution." data_tests: - not_null: - where: "guest_contribution_amount_in_txn_currency > 0 and document_version <> 'OldClaimToIncidentModel'" + where: "guest_contribution_amount_in_txn_currency > 0" - name: guest_contacted_about_damage data_type: boolean @@ -235,7 +225,6 @@ models: - relationships: to: ref('stg_core__accommodation') field: id_accommodation - where: "document_version <> 'OldClaimToIncidentModel'" - name: accommodation_url data_type: text @@ -298,12 +287,10 @@ models: data_type: numeric description: "Superhog unique booking ID" data_tests: - - not_null: - where: "document_version <> 'OldClaimToIncidentModel'" + - not_null - relationships: to: ref('stg_core__booking') field: id_booking - where: "document_version <> 'OldClaimToIncidentModel'" - name: booking_status data_type: text @@ -468,7 +455,7 @@ models: compensation from another platform." data_tests: - not_null: - where: "third_party_claim_amount_in_txn_currency > 0 and document_version <> 'OldClaimToIncidentModel'" + where: "third_party_claim_amount_in_txn_currency > 0" - name: cosmos_db_timestamp_utc data_type: timestamp diff --git a/models/staging/resolutions/stg_resolutions__incidents.sql b/models/staging/resolutions/stg_resolutions__incidents.sql index a1efe8d..e03072d 100644 --- a/models/staging/resolutions/stg_resolutions__incidents.sql +++ b/models/staging/resolutions/stg_resolutions__incidents.sql @@ -1,3 +1,9 @@ +-- Some records (document_version <> 'OldClaimToIncidentModel') are older versions +-- of the incident model and don't have the same data structure as the current one. +-- Many of them don't have the same necessary fields as the current model. +-- The same happens with the 'ManualFormWeb' document_version. +{% set old_or_manual_documents = ("OLDCLAIMTOINCIDENTMODEL", "MANUALFORMWEB") %} + with raw_incident as (select * from {{ source("resolutions", "incident") }}), deduped_verifications as ( @@ -283,3 +289,6 @@ select ) as cosmos_db_timestamp_utc from deduped_verifications +where + upper({{ adapter.quote("documents") }} ->> 'DocumentVersion') + not in {{ old_or_manual_documents }} From 1d514cec3ac865584c355a085d92aeb8c0747db9 Mon Sep 17 00:00:00 2001 From: Joaquin Ossa Date: Mon, 24 Feb 2025 16:27:16 +0100 Subject: [PATCH 12/15] Addressed comments --- models/staging/resolutions/schema.yml | 14 -------------- .../resolutions/stg_resolutions__incidents.sql | 16 ++-------------- 2 files changed, 2 insertions(+), 28 deletions(-) diff --git a/models/staging/resolutions/schema.yml b/models/staging/resolutions/schema.yml index d21bfa0..9bc2d52 100644 --- a/models/staging/resolutions/schema.yml +++ b/models/staging/resolutions/schema.yml @@ -14,12 +14,6 @@ models: - unique - not_null - - name: id_user - data_type: text - description: "Id of the user who created the incident." - data_tests: - - not_null - - name: id_verification data_type: text description: "Superhog unique validation ID, coming from other APIs." @@ -86,10 +80,6 @@ models: to: ref('stg_core__user') field: id_user - - name: host_user_claim_logs - data_type: text - description: "" - - name: host_account_name data_type: text description: "Name of the host account." @@ -316,10 +306,6 @@ models: data_type: text description: "List of services included in the booking." - - name: booking_protection - data_type: text - description: "" - - name: booking_platform_used data_type: text description: "The booking platform used" diff --git a/models/staging/resolutions/stg_resolutions__incidents.sql b/models/staging/resolutions/stg_resolutions__incidents.sql index e03072d..7f7066e 100644 --- a/models/staging/resolutions/stg_resolutions__incidents.sql +++ b/models/staging/resolutions/stg_resolutions__incidents.sql @@ -12,7 +12,6 @@ with select -- Basic Incident Details {{ adapter.quote("documents") }} ->> 'id' as id_incident, - {{ adapter.quote("documents") }} ->> 'partitionKey' as id_user, {{ adapter.quote("documents") }} ->> 'VerificationId' as id_verification, {{ adapter.quote("documents") }} ->> 'CurrentStatusName' as current_status_name, ({{ adapter.quote("documents") }} ->> 'IsSubmissionComplete')::boolean @@ -30,7 +29,6 @@ select -- Host Details {{ adapter.quote("documents") }} -> 'Host' ->> 'Id' as id_user_host, - {{ adapter.quote("documents") }} -> 'Host' ->> 'UserClaims' as host_user_claim_logs, {{ adapter.quote("documents") }} -> 'Host' ->> 'AccountName' as host_account_name, -- Host Contact Details @@ -128,11 +126,7 @@ select -> 'Guest' -> 'Involvements' ->> 'ContactedAboutDamage' - )::boolean as guest_contacted_about_damage, - {{ adapter.quote("documents") }} - -> 'Guest' - -> 'Involvements' - ->> 'ContactedEvidenceFiles' as guest_contacted_evidence_files, + )::boolean as is_guest_contacted_about_damage, {{ adapter.quote("documents") }} -> 'Guest' -> 'Involvements' @@ -156,10 +150,7 @@ select ({{ adapter.quote("documents") }} -> 'Insurance' ->> 'UnderlyingInsurance')::boolean as has_underlying_insurance, ({{ adapter.quote("documents") }} -> 'Insurance' ->> 'ClaimSubmitted')::boolean - as insurance_claim_submitted, - {{ adapter.quote("documents") }} - -> 'Insurance' - ->> 'UnderlyingInsuranceEvidenceFiles' as insurance_evidence_files, + as is_insurance_claim_submitted, -- Booking Details ({{ adapter.quote("documents") }} -> 'Reservation' ->> 'CheckIn')::timestamp @@ -189,9 +180,6 @@ select ->> 'BookingServices' as booking_services, {{ adapter.quote("documents") }} -> 'Reservation' - ->> 'BookingProtection' as booking_protection, - {{ adapter.quote("documents") }} - -> 'Reservation' ->> 'BookingPlatformUsed' as booking_platform_used, {{ adapter.quote("documents") }} -> 'Reservation' From 62316778fbaf199ebb17f62667e21b377ca70998 Mon Sep 17 00:00:00 2001 From: Joaquin Ossa Date: Mon, 24 Feb 2025 16:35:10 +0100 Subject: [PATCH 13/15] updated schema --- models/staging/resolutions/schema.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/staging/resolutions/schema.yml b/models/staging/resolutions/schema.yml index 9bc2d52..44d8d9c 100644 --- a/models/staging/resolutions/schema.yml +++ b/models/staging/resolutions/schema.yml @@ -194,7 +194,7 @@ models: - not_null: where: "guest_contribution_amount_in_txn_currency > 0" - - name: guest_contacted_about_damage + - name: is_guest_contacted_about_damage data_type: boolean description: "If the host has contacted the guest about the damage caused" @@ -241,7 +241,7 @@ models: data_type: boolean description: "If the property has underlying insurance" - - name: insurance_claim_submitted + - name: is_insurance_claim_submitted data_type: boolean description: "If an insurance claim has been submitted to an insurance company" From f036e626d70bcd4714d3a5b42414f21d96863097 Mon Sep 17 00:00:00 2001 From: Joaquin Ossa Date: Tue, 25 Feb 2025 12:19:45 +0100 Subject: [PATCH 14/15] Commit removed unnecessary fields --- models/staging/resolutions/schema.yml | 44 ++----------------- .../stg_resolutions__incidents.sql | 16 ------- 2 files changed, 3 insertions(+), 57 deletions(-) diff --git a/models/staging/resolutions/schema.yml b/models/staging/resolutions/schema.yml index 44d8d9c..c6ce496 100644 --- a/models/staging/resolutions/schema.yml +++ b/models/staging/resolutions/schema.yml @@ -34,10 +34,6 @@ models: data_type: text description: "Name of the agent who is currently handling the incident." - - name: read_only_for_customers - data_type: boolean - description: "Flag to indicate if the incident is read only for customers." - - name: status_history_logs data_type: text description: "Log of the status history." @@ -46,10 +42,6 @@ models: data_type: text description: "Reported document resCentre, Api, manual form" - - name: task_execution_logs - data_type: text - description: "" - - name: created_at_utc data_type: timestamp without time zone description: "Timestamp when the incident was created." @@ -72,8 +64,7 @@ models: - name: id_user_host data_type: text - description: "Unique Id that identifies the host. - Can be from SH or can be external from the client." + description: "Unique Id that identifies the host." data_tests: - not_null - relationships: @@ -198,10 +189,6 @@ models: data_type: boolean description: "If the host has contacted the guest about the damage caused" - - name: guest_contacted_evidence_files - data_type: text - description: "" - - name: guest_preventative_steps_details data_type: text description: "Description of any action to prevent the damage or further @@ -216,11 +203,6 @@ models: to: ref('stg_core__accommodation') field: id_accommodation - - name: accommodation_url - data_type: text - description: "Accommodation URL from any distribution channel, - preferable direct website." - - name: accommodation_name data_type: text description: "Accommodation name." @@ -235,7 +217,7 @@ models: - name: comment_logs data_type: text - description: "" + description: "Log of all comments submitted internally by the agents." - name: has_underlying_insurance data_type: boolean @@ -245,10 +227,6 @@ models: data_type: boolean description: "If an insurance claim has been submitted to an insurance company" - - name: insurance_evidence_files - data_type: text - description: "" - - name: check_in_at_utc data_type: timestamp without time zone description: "Timestamp of the check-in date in UTC of the booking." @@ -290,10 +268,6 @@ models: data_type: text description: "External unique ID" - - name: booking_details - data_type: text - description: "" - - name: number_of_guests data_type: numeric description: "Number of guests in the booking." @@ -310,21 +284,9 @@ models: data_type: text description: "The booking platform used" - - name: booking_platform_reference - data_type: text - description: "" - - name: agent_history data_type: text - description: "" - - - name: before_damage_evidence - data_type: text - description: "" - - - name: original_invoice_evidence - data_type: text - description: "" + description: "List of all agents who have handled the incident." - name: damage_incident_details data_type: text diff --git a/models/staging/resolutions/stg_resolutions__incidents.sql b/models/staging/resolutions/stg_resolutions__incidents.sql index 7f7066e..7adb483 100644 --- a/models/staging/resolutions/stg_resolutions__incidents.sql +++ b/models/staging/resolutions/stg_resolutions__incidents.sql @@ -17,11 +17,8 @@ select ({{ adapter.quote("documents") }} ->> 'IsSubmissionComplete')::boolean as is_submission_complete, {{ adapter.quote("documents") }} ->> 'CurrentAgentName' as current_agent_name, - ({{ adapter.quote("documents") }} ->> 'ReadOnlyForCustomers')::boolean - as read_only_for_customers, {{ adapter.quote("documents") }} ->> 'StatusHistory' as status_history_logs, {{ adapter.quote("documents") }} ->> 'DocumentVersion' as document_version, - {{ adapter.quote("documents") }} ->> 'TaskExecutionLogs' as task_execution_logs, ({{ adapter.quote("documents") }} ->> 'CreatedDate')::timestamp as created_at_utc, ({{ adapter.quote("documents") }} ->> 'CreatedDate')::date as created_date_utc, ({{ adapter.quote("documents") }} ->> 'UpdatedDate')::timestamp as updated_at_utc, @@ -135,7 +132,6 @@ select -- Accommodation Details (nullif({{ adapter.quote("documents") }} -> 'Listing' ->> 'Id', ''))::numeric as id_accommodation, - {{ adapter.quote("documents") }} -> 'Listing' ->> 'Url' as accommodation_url, {{ adapter.quote("documents") }} -> 'Listing' ->> 'Name' as accommodation_name, {{ adapter.quote("documents") }} -> 'Listing' @@ -170,9 +166,6 @@ select {{ adapter.quote("documents") }} -> 'Reservation' ->> 'ReservationId' as id_reservation, - {{ adapter.quote("documents") }} - -> 'Reservation' - ->> 'BookingDetails' as booking_details, ({{ adapter.quote("documents") }} -> 'Reservation' ->> 'NumberOfGuests')::numeric as number_of_guests, {{ adapter.quote("documents") }} @@ -181,9 +174,6 @@ select {{ adapter.quote("documents") }} -> 'Reservation' ->> 'BookingPlatformUsed' as booking_platform_used, - {{ adapter.quote("documents") }} - -> 'Reservation' - ->> 'BookingPlatformReference' as booking_platform_reference, -- Agent History {{ adapter.quote("documents") }} @@ -192,12 +182,6 @@ select -- Damage Report {{ adapter.quote("documents") }} - -> 'BeforeDamageEvidence' - ->> 'BeforeDamageEvidence' as before_damage_evidence, - {{ adapter.quote("documents") }} - -> 'OriginalInvoiceEvidence' - ->> 'OriginalInvoiceEvidence' as original_invoice_evidence, - {{ adapter.quote("documents") }} -> 'DamageReport' ->> 'IncidentDetails' as damage_incident_details, ( From 03955aadd501a9638938dd558c2936bea04a9e74 Mon Sep 17 00:00:00 2001 From: Joaquin Ossa Date: Tue, 25 Feb 2025 14:54:59 +0100 Subject: [PATCH 15/15] Finished schema --- models/staging/resolutions/schema.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/models/staging/resolutions/schema.yml b/models/staging/resolutions/schema.yml index c6ce496..3ef367f 100644 --- a/models/staging/resolutions/schema.yml +++ b/models/staging/resolutions/schema.yml @@ -303,7 +303,8 @@ models: - name: protection_name data_type: text - description: "Protection name." + description: "Indicates the selected booking protection plan, + e.g., Damage Protection Plan." - name: was_overriden data_type: boolean @@ -312,7 +313,7 @@ models: - name: waiver_or_deposit_name data_type: text - description: "Waiver or deposit name." + description: "Indicates whether the guest selected Deposit or Waiver as their cover." - name: guest_verification_status data_type: text