counterweight/data-dwh-dbt-project
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

commit wip

Browse source
This commit is contained in:
Joaquin Ossa 2025-02-24 15:06:55 +01:00
parent 62c2e44b86
commit 34f8fbe499
2 changed files with 17 additions and 21 deletions
Download patch file Download diff file Expand all files Collapse all files

29
models/staging/resolutions/schema.yml
View file

@ -5,10 +5,7 @@ models:
description: | description: |
"Contains all incidents stored by the resolutions team in Cosmos DB. "Contains all incidents stored by the resolutions team in Cosmos DB.
This includes data about the incident, the user claiming the incident, the booking, This includes data about the incident, the user claiming the incident, the booking,
the accommodation, the host and the guest. the accommodation, the host and the guest."
Some records (document_version <> 'OldClaimToIncidentModel') are older versions
of the incident model and don't have the same data structure as the current one.
Many of them don't have the same necessary fields as the current model."
columns: columns:
- name: id_incident - name: id_incident
data_type: text data_type: text
@ -63,15 +60,13 @@ models:
data_type: timestamp without time zone data_type: timestamp without time zone
description: "Timestamp when the incident was created." description: "Timestamp when the incident was created."
data_tests: data_tests:
- not_null: - not_null
where: "document_version <> 'OldClaimToIncidentModel'"
- name: created_date_utc - name: created_date_utc
data_type: date data_type: date
description: "Date when the incident was created." description: "Date when the incident was created."
data_tests: data_tests:
- not_null: - not_null
where: "document_version <> 'OldClaimToIncidentModel'"
- name: updated_at_utc - name: updated_at_utc
data_type: timestamp without time zone data_type: timestamp without time zone
@ -90,7 +85,6 @@ models:
- relationships: - relationships:
to: ref('stg_core__user') to: ref('stg_core__user')
field: id_user field: id_user
where: "document_version <> 'OldClaimToIncidentModel'"
- name: host_user_claim_logs - name: host_user_claim_logs
data_type: text data_type: text
@ -130,19 +124,15 @@ models:
data_type: text data_type: text
description: "Superhog code that uniquely identifies a single guest." description: "Superhog code that uniquely identifies a single guest."
data_tests: data_tests:
- not_null:
where: "document_version <> 'OldClaimToIncidentModel'"
- relationships: - relationships:
to: ref('stg_core__user') to: ref('stg_core__user')
field: id_user field: id_user
where: "document_version <> 'OldClaimToIncidentModel'"
- name: guest_email - name: guest_email
data_type: text data_type: text
description: "Email of the guest." description: "Email of the guest."
data_tests: data_tests:
- not_null: - not_null
where: "document_version <> 'OldClaimToIncidentModel'"
- name: guest_last_name - name: guest_last_name
data_type: text data_type: text
@ -186,7 +176,7 @@ models:
description: "Currency of the deposit retained." description: "Currency of the deposit retained."
data_tests: data_tests:
- not_null: - not_null:
where: "deposit_retained_amount_in_txn_currency > 0 and document_version <> 'OldClaimToIncidentModel'" where: "deposit_retained_amount_in_txn_currency > 0"
- name: has_guest_contributed_to_cost - name: has_guest_contributed_to_cost
data_type: boolean data_type: boolean
@ -212,7 +202,7 @@ models:
description: "Currency of the guest contribution." description: "Currency of the guest contribution."
data_tests: data_tests:
- not_null: - not_null:
where: "guest_contribution_amount_in_txn_currency > 0 and document_version <> 'OldClaimToIncidentModel'" where: "guest_contribution_amount_in_txn_currency > 0"
- name: guest_contacted_about_damage - name: guest_contacted_about_damage
data_type: boolean data_type: boolean
@ -235,7 +225,6 @@ models:
- relationships: - relationships:
to: ref('stg_core__accommodation') to: ref('stg_core__accommodation')
field: id_accommodation field: id_accommodation
where: "document_version <> 'OldClaimToIncidentModel'"
- name: accommodation_url - name: accommodation_url
data_type: text data_type: text
@ -298,12 +287,10 @@ models:
data_type: numeric data_type: numeric
description: "Superhog unique booking ID" description: "Superhog unique booking ID"
data_tests: data_tests:
- not_null: - not_null
where: "document_version <> 'OldClaimToIncidentModel'"
- relationships: - relationships:
to: ref('stg_core__booking') to: ref('stg_core__booking')
field: id_booking field: id_booking
where: "document_version <> 'OldClaimToIncidentModel'"
- name: booking_status - name: booking_status
data_type: text data_type: text
@ -468,7 +455,7 @@ models:
compensation from another platform." compensation from another platform."
data_tests: data_tests:
- not_null: - not_null:
where: "third_party_claim_amount_in_txn_currency > 0 and document_version <> 'OldClaimToIncidentModel'" where: "third_party_claim_amount_in_txn_currency > 0"
- name: cosmos_db_timestamp_utc - name: cosmos_db_timestamp_utc
data_type: timestamp data_type: timestamp

9
models/staging/resolutions/stg_resolutions__incidents.sql
View file

@ -1,3 +1,9 @@
-- Some records (document_version <> 'OldClaimToIncidentModel') are older versions
-- of the incident model and don't have the same data structure as the current one.
-- Many of them don't have the same necessary fields as the current model.
-- The same happens with the 'ManualFormWeb' document_version.
{% set old_or_manual_documents = ("OLDCLAIMTOINCIDENTMODEL", "MANUALFORMWEB") %}
with with
raw_incident as (select * from {{ source("resolutions", "incident") }}), raw_incident as (select * from {{ source("resolutions", "incident") }}),
deduped_verifications as ( deduped_verifications as (
@ -283,3 +289,6 @@ select
) as cosmos_db_timestamp_utc ) as cosmos_db_timestamp_utc
from deduped_verifications from deduped_verifications
where
upper({{ adapter.quote("documents") }} ->> 'DocumentVersion')
not in {{ old_or_manual_documents }}