counterweight/data-dwh-dbt-project
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merged PR 4482: Staging model of resolutions incidents

Browse source 
# Description

New staging model for Resolutions Incidents
It extracts all of the data from the incidents after the ingestion from CosmosDB.
This includes data on the host, the guest, the booking, the listing and all the resolution process until completed if completed.
Though it currently has a lot of problems like missing values, ids not existing in the back end or payments without currencies. Everything is more clarified here https://www.notion.so/truvi/Data-Quality-Report-Resolutions-Incidents-data-1a40446ff9c9807180eaccfeebb53045?pvs=4.

# Checklist

- [x] The edited models and dependants run properly with production data.
- [x] The edited models are sufficiently documented.
- [x] The edited models contain PK tests, and I've ran and passed them.
- [x] I have checked for DRY opportunities with other models and docs.
- [x] I've picked the right materialization for the affected models.

# Other

- [ ] Check if a full-refresh is required after this PR is merged.

Related work items: #27811
This commit is contained in:
Joaquin Ossa 2025-02-25 14:15:27 +00:00
commit 271f017cf5
3 changed files with 687 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

8
models/staging/resolutions/_resolutions_sources.yml Normal file
View file

@ -0,0 +1,8 @@
version: 2
sources:
- name: resolutions
schema: sync_cdb_resolutions
tables:
- name: incident
identifier: incident

413
models/staging/resolutions/schema.yml Normal file
View file

@ -0,0 +1,413 @@
version: 2
models:
- name: stg_resolutions__incidents
description: |
"Contains all incidents stored by the resolutions team in Cosmos DB.
This includes data about the incident, the user claiming the incident, the booking,
the accommodation, the host and the guest."
columns:
- name: id_incident
data_type: text
description: "Unique identifier for the incident."
data_tests:
- unique
- not_null
- name: id_verification
data_type: text
description: "Superhog unique validation ID, coming from other APIs."
- name: current_status_name
data_type: text
description: "current status of the incident."
data_tests:
- not_null
- name: is_submission_complete
data_type: boolean
description: "Flag to indicate if the submission is complete."
data_tests:
- not_null
- name: current_agent_name
data_type: text
description: "Name of the agent who is currently handling the incident."
- name: status_history_logs
data_type: text
description: "Log of the status history."
- name: document_version
data_type: text
description: "Reported document resCentre, Api, manual form"
- name: created_at_utc
data_type: timestamp without time zone
description: "Timestamp when the incident was created."
data_tests:
- not_null
- name: created_date_utc
data_type: date
description: "Date when the incident was created."
data_tests:
- not_null
- name: updated_at_utc
data_type: timestamp without time zone
description: "Timestamp when the incident was last updated."
- name: updated_date_utc
data_type: date
description: "Date when the incident was last updated."
- name: id_user_host
data_type: text
description: "Unique Id that identifies the host."
data_tests:
- not_null
- relationships:
to: ref('stg_core__user')
field: id_user
- name: host_account_name
data_type: text
description: "Name of the host account."
- name: host_email
data_type: text
description: "Email of the host."
data_tests:
- not_null
- name: host_last_name
data_type: text
description: "Last name of the host."
- name: host_first_name
data_type: text
description: "First name of the host."
- name: host_phone_code
data_type: text
description: "Phone code of the host."
- name: host_phone_number
data_type: text
description: "Phone number of the host."
- name: host_phone_number_with_code
data_type: text
description: "Phone number of the host with the code."
- name: id_user_guest
data_type: text
description: "Superhog code that uniquely identifies a single guest."
data_tests:
- relationships:
to: ref('stg_core__user')
field: id_user
- name: guest_email
data_type: text
description: "Email of the guest."
data_tests:
- not_null
- name: guest_last_name
data_type: text
description: "Last name of the guest."
- name: guest_first_name
data_type: text
description: "First name of the guest."
- name: guest_phone_code
data_type: text
description: "Phone code of the guest."
- name: guest_phone_number
data_type: text
description: "Phone number of the guest."
- name: guest_phone_number_with_code
data_type: text
description: "Phone number of the guest with the code."
- name: is_guest_deposit_retained
data_type: boolean
description: "In case a deposit has been collected from the guest,
whether it has been retained or not."
- name: is_guest_deposit_collected
data_type: boolean
description: "If the host has collected a deposit from the guest."
- name: deposit_retained_amount_in_txn_currency
data_type: numeric
description: "Amount of the deposit retained in local currency."
data_tests:
- dbt_expectations.expect_column_values_to_be_between:
min_value: 0
strictly: false
- name: deposit_retained_currency
data_type: text
description: "Currency of the deposit retained."
data_tests:
- not_null:
where: "deposit_retained_amount_in_txn_currency > 0"
- name: has_guest_contributed_to_cost
data_type: boolean
description: "If the guest has agreed to contribute to the cost when
talking to the host"
- name: has_host_taken_preventative_steps
data_type: boolean
description: "If the host has taken any action to prevent the damage
or further damage once identified"
- name: guest_contribution_amount_in_txn_currency
data_type: numeric
description: "Amount of the guest contribution, in case they did,
in local currency."
data_tests:
- dbt_expectations.expect_column_values_to_be_between:
min_value: 0
strictly: false
- name: guest_contribution_currency
data_type: text
description: "Currency of the guest contribution."
data_tests:
- not_null:
where: "guest_contribution_amount_in_txn_currency > 0"
- name: is_guest_contacted_about_damage
data_type: boolean
description: "If the host has contacted the guest about the damage caused"
- name: guest_preventative_steps_details
data_type: text
description: "Description of any action to prevent the damage or further
damage once identified"
- name: id_accommodation
data_type: numeric
description: "Superhog code that uniquely identifies a single accommodation."
data_tests:
- not_null
- relationships:
to: ref('stg_core__accommodation')
field: id_accommodation
- name: accommodation_name
data_type: text
description: "Accommodation name."
- name: accommodation_address
data_type: text
description: "Address of the accommodation."
- name: are_pets_allowed
data_type: boolean
description: "Flag to indicate if pets are allowed in the accommodation."
- name: comment_logs
data_type: text
description: "Log of all comments submitted internally by the agents."
- name: has_underlying_insurance
data_type: boolean
description: "If the property has underlying insurance"
- name: is_insurance_claim_submitted
data_type: boolean
description: "If an insurance claim has been submitted to an insurance company"
- name: check_in_at_utc
data_type: timestamp without time zone
description: "Timestamp of the check-in date in UTC of the booking."
data_tests:
- not_null
- name: check_in_date_utc
data_type: date
description: "Date of the check-in date in UTC of the booking."
data_tests:
- not_null
- name: check_out_at_utc
data_type: timestamp without time zone
description: "Timestamp of the check-out date in UTC of the booking."
data_tests:
- not_null
- name: check_out_date_utc
data_type: date
description: "Date of the check-out date in UTC of the booking."
data_tests:
- not_null
- name: id_booking
data_type: numeric
description: "Superhog unique booking ID"
data_tests:
- not_null
- relationships:
to: ref('stg_core__booking')
field: id_booking
- name: booking_status
data_type: text
description: "Status of the booking."
- name: id_reservation
data_type: text
description: "External unique ID"
- name: number_of_guests
data_type: numeric
description: "Number of guests in the booking."
data_tests:
- dbt_expectations.expect_column_values_to_be_between:
min_value: 0
strictly: false
- name: booking_services
data_type: text
description: "List of services included in the booking."
- name: booking_platform_used
data_type: text
description: "The booking platform used"
- name: agent_history
data_type: text
description: "List of all agents who have handled the incident."
- name: damage_incident_details
data_type: text
description: "Description of what happened in general terms"
- name: has_confirmed_loss
data_type: boolean
description: "Confirmation that the information they have provided in
the report is true to the best of their knowledge."
- name: damage_report_items
data_type: text
description: "List of items that were damaged."
- name: protection_name
data_type: text
description: "Indicates the selected booking protection plan,
e.g., Damage Protection Plan."
- name: was_overriden
data_type: boolean
description: "Flag to indicate if the proposed settlement value from
the calculator was overriden."
- name: waiver_or_deposit_name
data_type: text
description: "Indicates whether the guest selected Deposit or Waiver as their cover."
- name: guest_verification_status
data_type: text
description: "Guest verification status for this booking."
- name: lower_protection_limit_usd
data_type: numeric
description: "Lower protection limit in USD."
data_tests:
- dbt_expectations.expect_column_values_to_be_between:
min_value: 0
strictly: false
- name: upper_protection_limit_usd
data_type: numeric
description: "Upper protection limit in USD."
data_tests:
- dbt_expectations.expect_column_values_to_be_between:
min_value: 0
strictly: false
- name: calculated_payout_amount_in_txn_currency
data_type: numeric
description: "Calculated payout amount in local currency."
data_tests:
- dbt_expectations.expect_column_values_to_be_between:
min_value: 0
strictly: false
- name: calculated_payout_currency
data_type: text
description: "Currency of the calculated payout amount."
data_tests:
- not_null:
where: "calculated_payout_amount_in_txn_currency > 0"
- name: calculated_payout_amount_in_usd
data_type: numeric
description: "Calculated payout amount in USD."
data_tests:
- dbt_expectations.expect_column_values_to_be_between:
min_value: 0
strictly: false
- name: calculated_guest_charge_amount_in_txn_currency
data_type: numeric
description: "Calculated guest charge amount in local currency."
data_tests:
- dbt_expectations.expect_column_values_to_be_between:
min_value: 0
strictly: false
- name: calculated_guest_charge_currency
data_type: text
description: "Currency of the calculated guest charge amount."
data_tests:
- not_null:
where: "calculated_guest_charge_amount_in_txn_currency > 0"
- name: calculated_guest_charge_amount_in_usd
data_type: numeric
description: "Calculated guest charge amount in USD."
data_tests:
- dbt_expectations.expect_column_values_to_be_between:
min_value: 0
strictly: false
- name: third_party_claim_submitted
data_type: boolean
description: "If the host is seeking compensation from another platform"
- name: third_party_claim_from
data_type: text
description: "For example, Aircover from Airbnb."
- name: third_party_claim_amount_in_txn_currency
data_type: numeric
description: "Claim amount in local currency if the host is seeking
compensation from another platform."
data_tests:
- dbt_expectations.expect_column_values_to_be_between:
min_value: 0
strictly: false
- name: third_party_claim_currency
data_type: text
description: "Currency of the claim amount if the host is seeking
compensation from another platform."
data_tests:
- not_null:
where: "third_party_claim_amount_in_txn_currency > 0"
- name: cosmos_db_timestamp_utc
data_type: timestamp
description: Internal Cosmos DB timestamp of the last record update.
data_tests:
- not_null

266
models/staging/resolutions/stg_resolutions__incidents.sql Normal file
View file

@ -0,0 +1,266 @@
-- Some records (document_version <> 'OldClaimToIncidentModel') are older versions
-- of the incident model and don't have the same data structure as the current one.
-- Many of them don't have the same necessary fields as the current model.
-- The same happens with the 'ManualFormWeb' document_version.
{% set old_or_manual_documents = ("OLDCLAIMTOINCIDENTMODEL", "MANUALFORMWEB") %}
with
raw_incident as (select * from {{ source("resolutions", "incident") }}),
deduped_verifications as (
{{ cosmos_db_record_deduplication("raw_incident", "id") }}
)
select
-- Basic Incident Details
{{ adapter.quote("documents") }} ->> 'id' as id_incident,
{{ adapter.quote("documents") }} ->> 'VerificationId' as id_verification,
{{ adapter.quote("documents") }} ->> 'CurrentStatusName' as current_status_name,
({{ adapter.quote("documents") }} ->> 'IsSubmissionComplete')::boolean
as is_submission_complete,
{{ adapter.quote("documents") }} ->> 'CurrentAgentName' as current_agent_name,
{{ adapter.quote("documents") }} ->> 'StatusHistory' as status_history_logs,
{{ adapter.quote("documents") }} ->> 'DocumentVersion' as document_version,
({{ adapter.quote("documents") }} ->> 'CreatedDate')::timestamp as created_at_utc,
({{ adapter.quote("documents") }} ->> 'CreatedDate')::date as created_date_utc,
({{ adapter.quote("documents") }} ->> 'UpdatedDate')::timestamp as updated_at_utc,
({{ adapter.quote("documents") }} ->> 'UpdatedDate')::date as updated_date_utc,
-- Host Details
{{ adapter.quote("documents") }} -> 'Host' ->> 'Id' as id_user_host,
{{ adapter.quote("documents") }} -> 'Host' ->> 'AccountName' as host_account_name,
-- Host Contact Details
{{ adapter.quote("documents") }}
-> 'Host'
-> 'ContactDetails'
->> 'Email' as host_email,
{{ adapter.quote("documents") }}
-> 'Host'
-> 'ContactDetails'
->> 'LastName' as host_last_name,
{{ adapter.quote("documents") }}
-> 'Host'
-> 'ContactDetails'
->> 'FirstName' as host_first_name,
{{ adapter.quote("documents") }}
-> 'Host'
-> 'ContactDetails'
->> 'PhoneCode' as host_phone_code,
{{ adapter.quote("documents") }}
-> 'Host'
-> 'ContactDetails'
->> 'PhoneNumber' as host_phone_number,
{{ adapter.quote("documents") }}
-> 'Host'
-> 'ContactDetails'
->> 'PhoneNumberWithCode' as host_phone_number_with_code,
-- Guest Details
nullif({{ adapter.quote("documents") }} -> 'Guest' ->> 'Id', '') as id_user_guest,
{{ adapter.quote("documents") }}
-> 'Guest'
-> 'ContactDetails'
->> 'Email' as guest_email,
{{ adapter.quote("documents") }}
-> 'Guest'
-> 'ContactDetails'
->> 'LastName' as guest_last_name,
{{ adapter.quote("documents") }}
-> 'Guest'
-> 'ContactDetails'
->> 'FirstName' as guest_first_name,
{{ adapter.quote("documents") }}
-> 'Guest'
-> 'ContactDetails'
->> 'PhoneCode' as guest_phone_code,
{{ adapter.quote("documents") }}
-> 'Guest'
-> 'ContactDetails'
->> 'PhoneNumber' as guest_phone_number,
{{ adapter.quote("documents") }}
-> 'Guest'
-> 'ContactDetails'
->> 'PhoneNumberWithCode' as guest_phone_number_with_code,
-- Guest Deposit Details
({{ adapter.quote("documents") }} -> 'Guest' -> 'Deposit' ->> 'IsRetained')::boolean
as is_guest_deposit_retained,
(
{{ adapter.quote("documents") }} -> 'Guest' -> 'Deposit' ->> 'IsCollected'
)::boolean as is_guest_deposit_collected,
(
{{ adapter.quote("documents") }} -> 'Guest' -> 'Deposit' ->> 'RetainedAmount'
)::numeric as deposit_retained_amount_in_txn_currency,
{{ adapter.quote("documents") }}
-> 'Guest'
-> 'Deposit'
->> 'RetainedCurrency' as deposit_retained_currency,
-- Guest Involvements
(
{{ adapter.quote("documents") }}
-> 'Guest'
-> 'Involvements'
->> 'ContributedToCost'
)::boolean as has_guest_contributed_to_cost,
(
{{ adapter.quote("documents") }}
-> 'Guest'
-> 'Involvements'
->> 'PreventativeSteps'
)::boolean as has_host_taken_preventative_steps,
(
{{ adapter.quote("documents") }}
-> 'Guest'
-> 'Involvements'
->> 'ContributionAmount'
)::numeric as guest_contribution_amount_in_txn_currency,
{{ adapter.quote("documents") }}
-> 'Guest'
-> 'Involvements'
->> 'ContributionCurrency' as guest_contribution_currency,
(
{{ adapter.quote("documents") }}
-> 'Guest'
-> 'Involvements'
->> 'ContactedAboutDamage'
)::boolean as is_guest_contacted_about_damage,
{{ adapter.quote("documents") }}
-> 'Guest'
-> 'Involvements'
->> 'PreventativeStepsDetails' as guest_preventative_steps_details,
-- Accommodation Details
(nullif({{ adapter.quote("documents") }} -> 'Listing' ->> 'Id', ''))::numeric
as id_accommodation,
{{ adapter.quote("documents") }} -> 'Listing' ->> 'Name' as accommodation_name,
{{ adapter.quote("documents") }}
-> 'Listing'
->> 'Address' as accommodation_address,
({{ adapter.quote("documents") }} -> 'Listing' ->> 'PetsAllowed')::boolean
as are_pets_allowed,
-- Comments
{{ adapter.quote("documents") }} ->> 'Comments' as comment_logs,
-- Insurance Details
({{ adapter.quote("documents") }} -> 'Insurance' ->> 'UnderlyingInsurance')::boolean
as has_underlying_insurance,
({{ adapter.quote("documents") }} -> 'Insurance' ->> 'ClaimSubmitted')::boolean
as is_insurance_claim_submitted,
-- Booking Details
({{ adapter.quote("documents") }} -> 'Reservation' ->> 'CheckIn')::timestamp
as check_in_at_utc,
({{ adapter.quote("documents") }} -> 'Reservation' ->> 'CheckIn')::date
as check_in_date_utc,
({{ adapter.quote("documents") }} -> 'Reservation' ->> 'CheckOut')::timestamp
as check_out_at_utc,
({{ adapter.quote("documents") }} -> 'Reservation' ->> 'CheckOut')::date
as check_out_date_utc,
nullif(
({{ adapter.quote("documents") }} -> 'Reservation' ->> 'BookingId'), ''
)::numeric as id_booking,
{{ adapter.quote("documents") }}
-> 'Reservation'
->> 'BookingStatus' as booking_status,
{{ adapter.quote("documents") }}
-> 'Reservation'
->> 'ReservationId' as id_reservation,
({{ adapter.quote("documents") }} -> 'Reservation' ->> 'NumberOfGuests')::numeric
as number_of_guests,
{{ adapter.quote("documents") }}
-> 'Reservation'
->> 'BookingServices' as booking_services,
{{ adapter.quote("documents") }}
-> 'Reservation'
->> 'BookingPlatformUsed' as booking_platform_used,
-- Agent History
{{ adapter.quote("documents") }}
-> 'AgentHistory'
->> 'AgentHistory' as agent_history,
-- Damage Report
{{ adapter.quote("documents") }}
-> 'DamageReport'
->> 'IncidentDetails' as damage_incident_details,
(
{{ adapter.quote("documents") }} -> 'DamageReport' ->> 'DeclarationOfLoss'
)::boolean as has_confirmed_loss,
{{ adapter.quote("documents") }}
-> 'DamageReport'
->> 'Items' as damage_report_items,
-- Calculator
{{ adapter.quote("documents") }}
-> 'SavedCalculation'
->> 'ProtectionName' as protection_name,
({{ adapter.quote("documents") }} -> 'SavedCalculation' ->> 'WasOverriden')::boolean
as was_overriden,
{{ adapter.quote("documents") }}
-> 'SavedCalculation'
->> 'WaiverOrDepositName' as waiver_or_deposit_name,
{{ adapter.quote("documents") }}
-> 'SavedCalculation'
->> 'GuestVerificationStatus' as guest_verification_status,
(
{{ adapter.quote("documents") }}
-> 'SavedCalculation'
->> 'LowerProtectionLimitUsd'
)::numeric as lower_protection_limit_usd,
(
{{ adapter.quote("documents") }}
-> 'SavedCalculation'
->> 'UpperProtectionLimitUsd'
)::numeric as upper_protection_limit_usd,
(
{{ adapter.quote("documents") }}
-> 'SavedCalculation'
->> 'SubmittedHostPayoutAmount'
)::numeric as calculated_payout_amount_in_txn_currency,
{{ adapter.quote("documents") }}
-> 'SavedCalculation'
->> 'SubmittedHostPayoutCurrency' as calculated_payout_currency,
(
{{ adapter.quote("documents") }}
-> 'SavedCalculation'
->> 'CalculatedHostPayoutAmountInUsd'
)::numeric as calculated_payout_amount_in_usd,
(
{{ adapter.quote("documents") }}
-> 'SavedCalculation'
->> 'SubmittedGuestChargeAmount'
)::numeric as calculated_guest_charge_amount_in_txn_currency,
{{ adapter.quote("documents") }}
-> 'SavedCalculation'
->> 'SubmittedGuestChargeCurrency' as calculated_guest_charge_currency,
(
{{ adapter.quote("documents") }}
-> 'SavedCalculation'
->> 'CalculatedGuestChargeAmountInUsd'
)::numeric as calculated_guest_charge_amount_in_usd,
-- Third Party Compensation Claim
(
{{ adapter.quote("documents") }}
-> 'ThirdPartyCompensationClaim'
->> 'ClaimSubmitted'
)::boolean as third_party_claim_submitted,
{{ adapter.quote("documents") }}
-> 'ThirdPartyCompensationClaim'
->> 'From' as third_party_claim_from,
(
{{ adapter.quote("documents") }} -> 'ThirdPartyCompensationClaim' ->> 'Amount'
)::numeric as third_party_claim_amount_in_txn_currency,
{{ adapter.quote("documents") }}
-> 'ThirdPartyCompensationClaim'
->> 'Currency' as third_party_claim_currency,
to_timestamp(
(({{ adapter.quote("documents") }} ->> '_ts'))::integer
) as cosmos_db_timestamp_utc
from deduped_verifications
where
upper({{ adapter.quote("documents") }} ->> 'DocumentVersion')
not in {{ old_or_manual_documents }}