commit wip

models/staging/resolutions/schema.yml

@@ -5,10 +5,7 @@ models:
     description: |
       "Contains all incidents stored by the resolutions team in Cosmos DB.
         This includes data about the incident, the user claiming the incident, the booking, 
-        the accommodation, the host and the guest.
-        Some records (document_version <> 'OldClaimToIncidentModel') are older versions
-        of the incident model and don't have the same data structure as the current one.
-        Many of them don't have the same necessary fields as the current model."
+        the accommodation, the host and the guest."
     columns:
       - name: id_incident
         data_type: text
@@ -63,15 +60,13 @@ models:
         data_type: timestamp without time zone
         description: "Timestamp when the incident was created."
         data_tests:
-          - not_null:
-              where: "document_version <> 'OldClaimToIncidentModel'"
+          - not_null
 
       - name: created_date_utc
         data_type: date
         description: "Date when the incident was created."
         data_tests:
-          - not_null:
-              where: "document_version <> 'OldClaimToIncidentModel'"
+          - not_null
 
       - name: updated_at_utc
         data_type: timestamp without time zone
@@ -90,7 +85,6 @@ models:
           - relationships:
               to: ref('stg_core__user')
               field: id_user
-              where: "document_version <> 'OldClaimToIncidentModel'"
 
       - name: host_user_claim_logs
         data_type: text
@@ -130,19 +124,15 @@ models:
         data_type: text
         description: "Superhog code that uniquely identifies a single guest."
         data_tests:
-          - not_null:
-              where: "document_version <> 'OldClaimToIncidentModel'"
           - relationships:
               to: ref('stg_core__user')
               field: id_user
-              where: "document_version <> 'OldClaimToIncidentModel'"
 
       - name: guest_email
         data_type: text
         description: "Email of the guest."
         data_tests:
-          - not_null:
-              where: "document_version <> 'OldClaimToIncidentModel'"
+          - not_null
 
       - name: guest_last_name
         data_type: text
@@ -186,7 +176,7 @@ models:
         description: "Currency of the deposit retained."
         data_tests:
           - not_null:
-              where: "deposit_retained_amount_in_txn_currency > 0 and document_version <> 'OldClaimToIncidentModel'"
+              where: "deposit_retained_amount_in_txn_currency > 0"
 
       - name: has_guest_contributed_to_cost
         data_type: boolean
@@ -212,7 +202,7 @@ models:
         description: "Currency of the guest contribution."
         data_tests:
           - not_null:
-              where: "guest_contribution_amount_in_txn_currency > 0 and document_version <> 'OldClaimToIncidentModel'"
+              where: "guest_contribution_amount_in_txn_currency > 0"
 
       - name: guest_contacted_about_damage
         data_type: boolean
@@ -235,7 +225,6 @@ models:
           - relationships:
               to: ref('stg_core__accommodation')
               field: id_accommodation
-              where: "document_version <> 'OldClaimToIncidentModel'"
 
       - name: accommodation_url
         data_type: text
@@ -298,12 +287,10 @@ models:
         data_type: numeric
         description: "Superhog unique booking ID"
         data_tests:
-          - not_null:
-              where: "document_version <> 'OldClaimToIncidentModel'"
+          - not_null
           - relationships:
               to: ref('stg_core__booking')
               field: id_booking
-              where: "document_version <> 'OldClaimToIncidentModel'"
 
       - name: booking_status
         data_type: text
@@ -468,7 +455,7 @@ models:
           compensation from another platform."
         data_tests:
           - not_null:
-              where: "third_party_claim_amount_in_txn_currency > 0 and document_version <> 'OldClaimToIncidentModel'"
+              where: "third_party_claim_amount_in_txn_currency > 0"
 
       - name: cosmos_db_timestamp_utc
         data_type: timestamp

models/staging/resolutions/stg_resolutions__incidents.sql

@@ -1,3 +1,9 @@
+-- Some records (document_version <> 'OldClaimToIncidentModel') are older versions
+-- of the incident model and don't have the same data structure as the current one.
+-- Many of them don't have the same necessary fields as the current model.
+-- The same happens with the 'ManualFormWeb' document_version.
+{% set old_or_manual_documents = ("OLDCLAIMTOINCIDENTMODEL", "MANUALFORMWEB") %}
+
 with
     raw_incident as (select * from {{ source("resolutions", "incident") }}),
     deduped_verifications as (
@@ -283,3 +289,6 @@ select
     ) as cosmos_db_timestamp_utc
 
 from deduped_verifications
+where
+    upper({{ adapter.quote("documents") }} ->> 'DocumentVersion')
+    not in {{ old_or_manual_documents }}