From 8bc525e4c21dab6f836aca2aa269bad58e595ef8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oriol=20Roqu=C3=A9=20Paniagua?= Date: Tue, 1 Jul 2025 09:44:54 +0000 Subject: [PATCH] Merged PR 5589: Limits data test coverage on resolutions data # Description This relates to the data alerts on resolutions models. We have one incident status that refers to the record having missing information. The record that is currently triggering this alerts is indeed tagged as such. The idea is to: * Create a new boolean that flags if the incident has missing information. This is propagated stg to reporting. * Apply a where clause to any existing data alert that might be susceptible to fail due to not having enough quality, in staging and reporting. * Remove over coverage in intermediate since it's already happening in reporting. However we keep a few not-null tests and the PK on critical information (i.e., id_booking). # Checklist - [X] The edited models and dependants run properly with production data. - [X] The edited models are sufficiently documented. - [X] The edited models contain PK tests, and I've ran and passed them. - [NA] I have checked for DRY opportunities with other models and docs. - [NA] I've picked the right materialization for the affected models. # Other - [ ] Check if a full-refresh is required after this PR is merged. Related work items: #31668 --- .../int_resolutions__incidents.sql | 1 + models/intermediate/resolutions/schema.yml | 125 +----------------- .../resolutions/resolutions__incidents.sql | 1 + models/reporting/resolutions/schema.yml | 51 +++++-- models/staging/resolutions/schema.yml | 56 ++++++-- .../stg_resolutions__incidents.sql | 6 + 6 files changed, 99 insertions(+), 141 deletions(-) diff --git a/models/intermediate/resolutions/int_resolutions__incidents.sql b/models/intermediate/resolutions/int_resolutions__incidents.sql index 7159ff5..4fe7b04 100644 --- a/models/intermediate/resolutions/int_resolutions__incidents.sql +++ b/models/intermediate/resolutions/int_resolutions__incidents.sql @@ -147,6 +147,7 @@ select i.id_incident, i.id_verification, i.current_status_name, + i.is_incident_missing_details, i.is_submission_complete, i.current_agent_name, i.document_version, diff --git a/models/intermediate/resolutions/schema.yml b/models/intermediate/resolutions/schema.yml index 5f40e89..2d3aa1b 100644 --- a/models/intermediate/resolutions/schema.yml +++ b/models/intermediate/resolutions/schema.yml @@ -26,6 +26,12 @@ models: data_tests: - not_null + - name: is_incident_missing_details + data_type: boolean + description: | + Flag to indicate if the incident is currently missing details. + This is used to limit the data tests coverage. + - name: is_submission_complete data_type: boolean description: "Flag to indicate if the submission is complete." @@ -74,10 +80,6 @@ models: - name: time_to_first_comment_in_hours data_type: numeric description: "Time taken to reach the client for the first time." - data_tests: - - dbt_expectations.expect_column_values_to_be_between: - min_value: 0 - strictly: true - name: first_status_at_utc data_type: timestamp @@ -90,10 +92,6 @@ models: - name: resolution_time_in_hours data_type: numeric description: "Time taken to resolve the incident, in hours, if completed." - data_tests: - - dbt_expectations.expect_column_values_to_be_between: - min_value: 0 - strictly: true - name: number_damaged_items data_type: bigint @@ -103,64 +101,35 @@ models: data_type: numeric description: | "Original value of the damaged items in the host currency." - data_tests: - - dbt_expectations.expect_column_values_to_be_between: - min_value: 0 - strictly: false - name: original_value_amount_in_gbp data_type: numeric description: | "Original value of the damaged items in GBP." - data_tests: - - dbt_expectations.expect_column_values_to_be_between: - min_value: 0 - strictly: false - name: asked_repair_amount_in_host_currency data_type: numeric description: | "Amount requested by the host for repair in the host currency." - data_tests: - - dbt_expectations.expect_column_values_to_be_between: - min_value: 0 - strictly: false - name: asked_repair_amount_in_gbp data_type: numeric description: | "Amount requested by the host for repair in GBP." - data_tests: - - dbt_expectations.expect_column_values_to_be_between: - min_value: 0 - strictly: false - name: accepted_amount_in_host_currency data_type: numeric description: | "Final accepted settlement amount in the host currency." - data_tests: - - dbt_expectations.expect_column_values_to_be_between: - min_value: 0 - strictly: false - name: accepted_amount_in_gbp data_type: numeric description: | "Final accepted settlement amount in GBP." - data_tests: - - dbt_expectations.expect_column_values_to_be_between: - min_value: 0 - strictly: false - name: host_currency data_type: text description: "Host or local currency" - data_tests: - - not_null: - where: "original_value_amount_in_host_currency > 0 - or asked_repair_amount_in_host_currency > 0 - or accepted_amount_in_host_currency > 0" - name: guest_verification_status data_type: text @@ -169,119 +138,63 @@ models: - name: lower_protection_limit_usd data_type: numeric description: "Lower protection limit in USD." - data_tests: - - dbt_expectations.expect_column_values_to_be_between: - min_value: 0 - strictly: false - name: upper_protection_limit_usd data_type: numeric description: "Upper protection limit in USD." - data_tests: - - dbt_expectations.expect_column_values_to_be_between: - min_value: 0 - strictly: false - name: calculated_payout_amount_in_txn_currency data_type: numeric description: "Calculated payout amount in local currency." - data_tests: - - dbt_expectations.expect_column_values_to_be_between: - min_value: 0 - strictly: false - name: calculated_payout_currency data_type: text description: "Currency of the calculated payout amount." - data_tests: - - not_null: - where: "calculated_payout_amount_in_txn_currency > 0" - name: calculated_payout_amount_in_gbp data_type: numeric description: "Calculated payout amount in GBP." - data_tests: - - dbt_expectations.expect_column_values_to_be_between: - min_value: 0 - strictly: false - name: submitted_payout_amount_in_txn_currency data_type: numeric description: "Submitted payout amount in local currency." - data_tests: - - dbt_expectations.expect_column_values_to_be_between: - min_value: 0 - strictly: false - name: submitted_payout_currency data_type: text description: "Currency of the submitted payout amount." - data_tests: - - not_null: - where: "submitted_payout_amount_in_txn_currency > 0" - name: submitted_payout_amount_in_gbp data_type: numeric description: "Submitted payout amount in GBP." - data_tests: - - dbt_expectations.expect_column_values_to_be_between: - min_value: 0 - strictly: false - name: calculated_guest_charge_amount_in_txn_currency data_type: numeric description: "Calculated guest charge amount in local currency." - data_tests: - - dbt_expectations.expect_column_values_to_be_between: - min_value: 0 - strictly: false - name: calculated_guest_charge_currency data_type: text description: "Currency of the calculated guest charge amount." - data_tests: - - not_null: - where: "calculated_guest_charge_amount_in_txn_currency > 0" - name: calculated_guest_charge_amount_in_gbp data_type: numeric description: "Calculated guest charge amount in GBP." - data_tests: - - dbt_expectations.expect_column_values_to_be_between: - min_value: 0 - strictly: false - name: submitted_guest_charge_amount_in_txn_currency data_type: numeric description: "Submitted guest charge amount in local currency." - data_tests: - - dbt_expectations.expect_column_values_to_be_between: - min_value: 0 - strictly: false - name: submitted_guest_charge_currency data_type: text description: "Currency of the submitted guest charge amount." - data_tests: - - not_null: - where: "submitted_guest_charge_amount_in_txn_currency > 0" - name: submitted_guest_charge_amount_in_gbp data_type: numeric description: "Submitted guest charge amount in GBP." - data_tests: - - dbt_expectations.expect_column_values_to_be_between: - min_value: 0 - strictly: false - name: id_user_host data_type: text description: | "Unique Id that identifies the host." - data_tests: - - relationships: - to: ref('stg_core__user') - field: id_user - name: host_account_name data_type: text @@ -310,10 +223,6 @@ models: - name: id_user_guest data_type: text description: "Superhog code that uniquely identifies a single guest." - data_tests: - - relationships: - to: ref('stg_core__user') - field: id_user - name: guest_email data_type: text @@ -347,17 +256,10 @@ models: - name: deposit_retained_amount_in_txn_currency data_type: numeric description: "Amount of the deposit retained in local currency." - data_tests: - - dbt_expectations.expect_column_values_to_be_between: - min_value: 0 - strictly: false - name: deposit_retained_currency data_type: text description: "Currency of the deposit retained." - data_tests: - - not_null: - where: "deposit_retained_amount_in_txn_currency > 0" - name: has_guest_contributed_to_cost data_type: boolean @@ -373,26 +275,15 @@ models: data_type: numeric description: "Amount of the guest contribution, in case they did, in local currency." - data_tests: - - dbt_expectations.expect_column_values_to_be_between: - min_value: 0 - strictly: false - name: guest_contribution_currency data_type: text description: "Currency of the guest contribution." - data_tests: - - not_null: - where: "guest_contribution_amount_in_txn_currency > 0" - name: guest_contribution_amount_in_gbp data_type: numeric description: "Amount of the guest contribution, in case they did, in GBP." - data_tests: - - dbt_expectations.expect_column_values_to_be_between: - min_value: 0 - strictly: false - name: is_guest_contacted_about_damage data_type: boolean @@ -401,10 +292,6 @@ models: - name: id_accommodation data_type: numeric description: "Superhog code that uniquely identifies a single accommodation." - data_tests: - - relationships: - to: ref('stg_core__accommodation') - field: id_accommodation - name: accommodation_name data_type: text diff --git a/models/reporting/resolutions/resolutions__incidents.sql b/models/reporting/resolutions/resolutions__incidents.sql index 5dd9376..135c925 100644 --- a/models/reporting/resolutions/resolutions__incidents.sql +++ b/models/reporting/resolutions/resolutions__incidents.sql @@ -7,6 +7,7 @@ select id_incident as id_incident, id_verification as id_verification, current_status_name as current_status_name, + is_incident_missing_details as is_incident_missing_details, is_submission_complete as is_submission_complete, current_agent_name as current_agent_name, document_version as document_version, diff --git a/models/reporting/resolutions/schema.yml b/models/reporting/resolutions/schema.yml index 643b4a7..dd310e1 100644 --- a/models/reporting/resolutions/schema.yml +++ b/models/reporting/resolutions/schema.yml @@ -26,6 +26,14 @@ models: data_tests: - not_null + - name: is_incident_missing_details + data_type: boolean + description: | + Flag to indicate if the incident is currently missing details. + This is used to limit the data tests coverage. + data_tests: + - not_null + - name: is_submission_complete data_type: boolean description: "Flag to indicate if the submission is complete." @@ -78,6 +86,7 @@ models: - dbt_expectations.expect_column_values_to_be_between: min_value: 0 strictly: true + where: not is_incident_missing_details - name: first_status_at_utc data_type: timestamp @@ -94,6 +103,7 @@ models: - dbt_expectations.expect_column_values_to_be_between: min_value: 0 strictly: true + where: not is_incident_missing_details - name: number_damaged_items data_type: bigint @@ -107,6 +117,7 @@ models: - dbt_expectations.expect_column_values_to_be_between: min_value: 0 strictly: false + where: not is_incident_missing_details - name: original_value_amount_in_gbp data_type: numeric @@ -116,6 +127,7 @@ models: - dbt_expectations.expect_column_values_to_be_between: min_value: 0 strictly: false + where: not is_incident_missing_details - name: asked_repair_amount_in_host_currency data_type: numeric @@ -125,6 +137,7 @@ models: - dbt_expectations.expect_column_values_to_be_between: min_value: 0 strictly: false + where: not is_incident_missing_details - name: asked_repair_amount_in_gbp data_type: numeric @@ -134,6 +147,7 @@ models: - dbt_expectations.expect_column_values_to_be_between: min_value: 0 strictly: false + where: not is_incident_missing_details - name: accepted_amount_in_host_currency data_type: numeric @@ -143,6 +157,7 @@ models: - dbt_expectations.expect_column_values_to_be_between: min_value: 0 strictly: false + where: not is_incident_missing_details - name: accepted_amount_in_gbp data_type: numeric @@ -152,15 +167,17 @@ models: - dbt_expectations.expect_column_values_to_be_between: min_value: 0 strictly: false + where: not is_incident_missing_details - name: host_currency data_type: text description: "Host or local currency" data_tests: - not_null: - where: "original_value_amount_in_host_currency > 0 + where: "(original_value_amount_in_host_currency > 0 or asked_repair_amount_in_host_currency > 0 - or accepted_amount_in_host_currency > 0" + or accepted_amount_in_host_currency > 0) + and not is_incident_missing_details" - name: guest_verification_status data_type: text @@ -173,6 +190,7 @@ models: - dbt_expectations.expect_column_values_to_be_between: min_value: 0 strictly: false + where: not is_incident_missing_details - name: upper_protection_limit_usd data_type: numeric @@ -181,6 +199,7 @@ models: - dbt_expectations.expect_column_values_to_be_between: min_value: 0 strictly: false + where: not is_incident_missing_details - name: calculated_payout_amount_in_txn_currency data_type: numeric @@ -189,13 +208,14 @@ models: - dbt_expectations.expect_column_values_to_be_between: min_value: 0 strictly: false + where: not is_incident_missing_details - name: calculated_payout_currency data_type: text description: "Currency of the calculated payout amount." data_tests: - not_null: - where: "calculated_payout_amount_in_txn_currency > 0" + where: "calculated_payout_amount_in_txn_currency > 0 and not is_incident_missing_details" - name: calculated_payout_amount_in_gbp data_type: numeric @@ -204,6 +224,7 @@ models: - dbt_expectations.expect_column_values_to_be_between: min_value: 0 strictly: false + where: not is_incident_missing_details - name: submitted_payout_amount_in_txn_currency data_type: numeric @@ -212,13 +233,14 @@ models: - dbt_expectations.expect_column_values_to_be_between: min_value: 0 strictly: false + where: not is_incident_missing_details - name: submitted_payout_currency data_type: text description: "Currency of the submitted payout amount." data_tests: - not_null: - where: "submitted_payout_amount_in_txn_currency > 0" + where: "submitted_payout_amount_in_txn_currency > 0 and not is_incident_missing_details" - name: submitted_payout_amount_in_gbp data_type: numeric @@ -227,6 +249,7 @@ models: - dbt_expectations.expect_column_values_to_be_between: min_value: 0 strictly: false + where: not is_incident_missing_details - name: calculated_guest_charge_amount_in_txn_currency data_type: numeric @@ -235,13 +258,14 @@ models: - dbt_expectations.expect_column_values_to_be_between: min_value: 0 strictly: false + where: not is_incident_missing_details - name: calculated_guest_charge_currency data_type: text description: "Currency of the calculated guest charge amount." data_tests: - not_null: - where: "calculated_guest_charge_amount_in_txn_currency > 0" + where: "calculated_guest_charge_amount_in_txn_currency > 0 and not is_incident_missing_details" - name: calculated_guest_charge_amount_in_gbp data_type: numeric @@ -250,6 +274,7 @@ models: - dbt_expectations.expect_column_values_to_be_between: min_value: 0 strictly: false + where: not is_incident_missing_details - name: submitted_guest_charge_amount_in_txn_currency data_type: numeric @@ -258,13 +283,14 @@ models: - dbt_expectations.expect_column_values_to_be_between: min_value: 0 strictly: false + where: not is_incident_missing_details - name: submitted_guest_charge_currency data_type: text description: "Currency of the submitted guest charge amount." data_tests: - not_null: - where: "submitted_guest_charge_amount_in_txn_currency > 0" + where: "submitted_guest_charge_amount_in_txn_currency > 0 and not is_incident_missing_details" - name: submitted_guest_charge_amount_in_gbp data_type: numeric @@ -273,6 +299,7 @@ models: - dbt_expectations.expect_column_values_to_be_between: min_value: 0 strictly: false + where: not is_incident_missing_details - name: id_user_host data_type: text @@ -282,6 +309,7 @@ models: - relationships: to: ref('stg_core__user') field: id_user + where: not is_incident_missing_details - name: host_account_name data_type: text @@ -314,6 +342,7 @@ models: - relationships: to: ref('stg_core__user') field: id_user + where: not is_incident_missing_details - name: guest_email data_type: text @@ -351,13 +380,14 @@ models: - dbt_expectations.expect_column_values_to_be_between: min_value: 0 strictly: false + where: not is_incident_missing_details - name: deposit_retained_currency data_type: text description: "Currency of the deposit retained." data_tests: - not_null: - where: "deposit_retained_amount_in_txn_currency > 0" + where: "deposit_retained_amount_in_txn_currency > 0 and not is_incident_missing_details" - name: has_guest_contributed_to_cost data_type: boolean @@ -377,13 +407,14 @@ models: - dbt_expectations.expect_column_values_to_be_between: min_value: 0 strictly: false + where: not is_incident_missing_details - name: guest_contribution_currency data_type: text description: "Currency of the guest contribution." data_tests: - not_null: - where: "guest_contribution_amount_in_txn_currency > 0" + where: "guest_contribution_amount_in_txn_currency > 0 and not is_incident_missing_details" - name: guest_contribution_amount_in_gbp data_type: numeric @@ -393,6 +424,7 @@ models: - dbt_expectations.expect_column_values_to_be_between: min_value: 0 strictly: false + where: not is_incident_missing_details - name: is_guest_contacted_about_damage data_type: boolean @@ -405,6 +437,7 @@ models: - relationships: to: ref('stg_core__accommodation') field: id_accommodation + where: not is_incident_missing_details - name: accommodation_name data_type: text @@ -441,4 +474,4 @@ models: - name: id_reservation data_type: text - description: "External unique ID for the reservation." \ No newline at end of file + description: "External unique ID for the reservation." diff --git a/models/staging/resolutions/schema.yml b/models/staging/resolutions/schema.yml index 275b224..a648bcf 100644 --- a/models/staging/resolutions/schema.yml +++ b/models/staging/resolutions/schema.yml @@ -24,6 +24,14 @@ models: data_tests: - not_null + - name: is_incident_missing_details + data_type: boolean + description: | + Flag to indicate if the incident is currently missing details. + This is used to limit the data tests coverage. + data_tests: + - not_null + - name: is_submission_complete data_type: boolean description: "Flag to indicate if the submission is complete." @@ -69,6 +77,7 @@ models: - relationships: to: ref('stg_core__user') field: id_user + where: not is_incident_missing_details - name: host_account_name data_type: text @@ -105,6 +114,7 @@ models: - relationships: to: ref('stg_core__user') field: id_user + where: not is_incident_missing_details - name: guest_email data_type: text @@ -146,13 +156,14 @@ models: - dbt_expectations.expect_column_values_to_be_between: min_value: 0 strictly: false + where: not is_incident_missing_details - name: deposit_retained_currency data_type: text description: "Currency of the deposit retained." data_tests: - not_null: - where: "deposit_retained_amount_in_txn_currency > 0" + where: "deposit_retained_amount_in_txn_currency > 0 and not is_incident_missing_details" - name: has_guest_contributed_to_cost data_type: boolean @@ -172,13 +183,14 @@ models: - dbt_expectations.expect_column_values_to_be_between: min_value: 0 strictly: false + where: not is_incident_missing_details - name: guest_contribution_currency data_type: text description: "Currency of the guest contribution." data_tests: - not_null: - where: "guest_contribution_amount_in_txn_currency > 0" + where: "guest_contribution_amount_in_txn_currency > 0 and not is_incident_missing_details" - name: is_guest_contacted_about_damage data_type: boolean @@ -193,10 +205,12 @@ models: data_type: numeric description: "Superhog code that uniquely identifies a single accommodation." data_tests: - - not_null + - not_null: + where: "not is_incident_missing_details" - relationships: to: ref('stg_core__accommodation') field: id_accommodation + where: not is_incident_missing_details - name: accommodation_name data_type: text @@ -226,25 +240,29 @@ models: data_type: timestamp without time zone description: "Timestamp of the check-in date in UTC of the booking." data_tests: - - not_null + - not_null: + where: "not is_incident_missing_details" - name: check_in_date_utc data_type: date description: "Date of the check-in date in UTC of the booking." data_tests: - - not_null + - not_null: + where: "not is_incident_missing_details" - name: check_out_at_utc data_type: timestamp without time zone description: "Timestamp of the check-out date in UTC of the booking." data_tests: - - not_null + - not_null: + where: "not is_incident_missing_details" - name: check_out_date_utc data_type: date description: "Date of the check-out date in UTC of the booking." data_tests: - - not_null + - not_null: + where: "not is_incident_missing_details" - name: id_booking data_type: numeric @@ -270,6 +288,7 @@ models: - dbt_expectations.expect_column_values_to_be_between: min_value: 0 strictly: false + where: "not is_incident_missing_details" - name: booking_services data_type: text @@ -301,7 +320,7 @@ models: description: "Timestamp of the calculation." data_tests: - not_null: - where: "calculated_payout_amount_in_txn_currency > 0" + where: "calculated_payout_amount_in_txn_currency > 0 and not is_incident_missing_details" - name: protection_name data_type: text @@ -323,6 +342,7 @@ models: - dbt_expectations.expect_column_values_to_be_between: min_value: 0 strictly: false + where: "not is_incident_missing_details" - name: upper_protection_limit_usd data_type: numeric @@ -331,6 +351,7 @@ models: - dbt_expectations.expect_column_values_to_be_between: min_value: 0 strictly: false + where: "not is_incident_missing_details" - name: calculated_payout_amount_in_txn_currency data_type: numeric @@ -339,13 +360,14 @@ models: - dbt_expectations.expect_column_values_to_be_between: min_value: 0 strictly: false + where: "not is_incident_missing_details" - name: calculated_payout_currency data_type: text description: "Currency of the calculated payout amount." data_tests: - not_null: - where: "calculated_payout_amount_in_txn_currency > 0" + where: "calculated_payout_amount_in_txn_currency > 0 and not is_incident_missing_details" - name: calculated_payout_amount_in_usd data_type: numeric @@ -354,6 +376,7 @@ models: - dbt_expectations.expect_column_values_to_be_between: min_value: 0 strictly: false + where: "not is_incident_missing_details" - name: submitted_payout_amount_in_txn_currency data_type: numeric @@ -362,13 +385,14 @@ models: - dbt_expectations.expect_column_values_to_be_between: min_value: 0 strictly: false + where: "not is_incident_missing_details" - name: submitted_payout_currency data_type: text description: "Currency of the submitted payout amount." data_tests: - not_null: - where: "submitted_payout_amount_in_txn_currency > 0" + where: "submitted_payout_amount_in_txn_currency > 0 and not is_incident_missing_details" - name: submitted_payout_amount_in_usd data_type: numeric @@ -377,6 +401,7 @@ models: - dbt_expectations.expect_column_values_to_be_between: min_value: 0 strictly: false + where: "not is_incident_missing_details" - name: calculated_guest_charge_amount_in_txn_currency data_type: numeric @@ -385,13 +410,14 @@ models: - dbt_expectations.expect_column_values_to_be_between: min_value: 0 strictly: false + where: "not is_incident_missing_details" - name: calculated_guest_charge_currency data_type: text description: "Currency of the calculated guest charge amount." data_tests: - not_null: - where: "calculated_guest_charge_amount_in_txn_currency > 0" + where: "calculated_guest_charge_amount_in_txn_currency > 0 and not is_incident_missing_details" - name: calculated_guest_charge_amount_in_usd data_type: numeric @@ -400,6 +426,7 @@ models: - dbt_expectations.expect_column_values_to_be_between: min_value: 0 strictly: false + where: "not is_incident_missing_details" - name: submitted_guest_charge_amount_in_txn_currency data_type: numeric @@ -408,13 +435,14 @@ models: - dbt_expectations.expect_column_values_to_be_between: min_value: 0 strictly: false + where: "not is_incident_missing_details" - name: submitted_guest_charge_currency data_type: text description: "Currency of the submitted guest charge amount." data_tests: - not_null: - where: "submitted_guest_charge_amount_in_txn_currency > 0" + where: "submitted_guest_charge_amount_in_txn_currency > 0 and not is_incident_missing_details" - name: submitted_guest_charge_amount_in_usd data_type: numeric @@ -423,6 +451,7 @@ models: - dbt_expectations.expect_column_values_to_be_between: min_value: 0 strictly: false + where: "not is_incident_missing_details" - name: third_party_claim_submitted data_type: boolean @@ -440,6 +469,7 @@ models: - dbt_expectations.expect_column_values_to_be_between: min_value: 0 strictly: false + where: "not is_incident_missing_details" - name: third_party_claim_currency data_type: text @@ -447,7 +477,7 @@ models: compensation from another platform." data_tests: - not_null: - where: "third_party_claim_amount_in_txn_currency > 0" + where: "third_party_claim_amount_in_txn_currency > 0 and not is_incident_missing_details" - name: cosmos_db_timestamp_utc data_type: timestamp diff --git a/models/staging/resolutions/stg_resolutions__incidents.sql b/models/staging/resolutions/stg_resolutions__incidents.sql index 1787baa..cc77e30 100644 --- a/models/staging/resolutions/stg_resolutions__incidents.sql +++ b/models/staging/resolutions/stg_resolutions__incidents.sql @@ -3,9 +3,13 @@ -- Many of them don't have the same necessary fields as the current model. -- The same happens with the 'ManualFormWeb' document_version. {% set old_or_manual_documents = ("OLDCLAIMTOINCIDENTMODEL", "MANUALFORMWEB") %} + -- Test or cancelled incidents are not considered in the reporting. {% set tests_or_cancelled_incidents = "ARCHIVED" %} +-- Some incidents have insufficient details which might create data quality issues. +{% set insufficient_details_incidents = "INSUFFICIENT DETAILS" %} + with raw_incident as (select * from {{ source("resolutions", "incident") }}), deduped_verifications as ( @@ -16,6 +20,8 @@ select {{ adapter.quote("documents") }} ->> 'id' as id_incident, {{ adapter.quote("documents") }} ->> 'VerificationId' as id_verification, {{ adapter.quote("documents") }} ->> 'CurrentStatusName' as current_status_name, + upper({{ adapter.quote("documents") }} ->> 'CurrentStatusName') + = '{{ insufficient_details_incidents }}' as is_incident_missing_details, ({{ adapter.quote("documents") }} ->> 'IsSubmissionComplete')::boolean as is_submission_complete, {{ adapter.quote("documents") }} ->> 'CurrentAgentName' as current_agent_name,