diff --git a/models/staging/resolutions/schema.yml b/models/staging/resolutions/schema.yml index ef74b6e..d21bfa0 100644 --- a/models/staging/resolutions/schema.yml +++ b/models/staging/resolutions/schema.yml @@ -5,10 +5,7 @@ models: description: | "Contains all incidents stored by the resolutions team in Cosmos DB. This includes data about the incident, the user claiming the incident, the booking, - the accommodation, the host and the guest. - Some records (document_version <> 'OldClaimToIncidentModel') are older versions - of the incident model and don't have the same data structure as the current one. - Many of them don't have the same necessary fields as the current model." + the accommodation, the host and the guest." columns: - name: id_incident data_type: text @@ -63,15 +60,13 @@ models: data_type: timestamp without time zone description: "Timestamp when the incident was created." data_tests: - - not_null: - where: "document_version <> 'OldClaimToIncidentModel'" + - not_null - name: created_date_utc data_type: date description: "Date when the incident was created." data_tests: - - not_null: - where: "document_version <> 'OldClaimToIncidentModel'" + - not_null - name: updated_at_utc data_type: timestamp without time zone @@ -90,7 +85,6 @@ models: - relationships: to: ref('stg_core__user') field: id_user - where: "document_version <> 'OldClaimToIncidentModel'" - name: host_user_claim_logs data_type: text @@ -130,19 +124,15 @@ models: data_type: text description: "Superhog code that uniquely identifies a single guest." data_tests: - - not_null: - where: "document_version <> 'OldClaimToIncidentModel'" - relationships: to: ref('stg_core__user') field: id_user - where: "document_version <> 'OldClaimToIncidentModel'" - name: guest_email data_type: text description: "Email of the guest." data_tests: - - not_null: - where: "document_version <> 'OldClaimToIncidentModel'" + - not_null - name: guest_last_name data_type: text @@ -186,7 +176,7 @@ models: description: "Currency of the deposit retained." data_tests: - not_null: - where: "deposit_retained_amount_in_txn_currency > 0 and document_version <> 'OldClaimToIncidentModel'" + where: "deposit_retained_amount_in_txn_currency > 0" - name: has_guest_contributed_to_cost data_type: boolean @@ -212,7 +202,7 @@ models: description: "Currency of the guest contribution." data_tests: - not_null: - where: "guest_contribution_amount_in_txn_currency > 0 and document_version <> 'OldClaimToIncidentModel'" + where: "guest_contribution_amount_in_txn_currency > 0" - name: guest_contacted_about_damage data_type: boolean @@ -235,7 +225,6 @@ models: - relationships: to: ref('stg_core__accommodation') field: id_accommodation - where: "document_version <> 'OldClaimToIncidentModel'" - name: accommodation_url data_type: text @@ -298,12 +287,10 @@ models: data_type: numeric description: "Superhog unique booking ID" data_tests: - - not_null: - where: "document_version <> 'OldClaimToIncidentModel'" + - not_null - relationships: to: ref('stg_core__booking') field: id_booking - where: "document_version <> 'OldClaimToIncidentModel'" - name: booking_status data_type: text @@ -468,7 +455,7 @@ models: compensation from another platform." data_tests: - not_null: - where: "third_party_claim_amount_in_txn_currency > 0 and document_version <> 'OldClaimToIncidentModel'" + where: "third_party_claim_amount_in_txn_currency > 0" - name: cosmos_db_timestamp_utc data_type: timestamp diff --git a/models/staging/resolutions/stg_resolutions__incidents.sql b/models/staging/resolutions/stg_resolutions__incidents.sql index a1efe8d..e03072d 100644 --- a/models/staging/resolutions/stg_resolutions__incidents.sql +++ b/models/staging/resolutions/stg_resolutions__incidents.sql @@ -1,3 +1,9 @@ +-- Some records (document_version <> 'OldClaimToIncidentModel') are older versions +-- of the incident model and don't have the same data structure as the current one. +-- Many of them don't have the same necessary fields as the current model. +-- The same happens with the 'ManualFormWeb' document_version. +{% set old_or_manual_documents = ("OLDCLAIMTOINCIDENTMODEL", "MANUALFORMWEB") %} + with raw_incident as (select * from {{ source("resolutions", "incident") }}), deduped_verifications as ( @@ -283,3 +289,6 @@ select ) as cosmos_db_timestamp_utc from deduped_verifications +where + upper({{ adapter.quote("documents") }} ->> 'DocumentVersion') + not in {{ old_or_manual_documents }}