Merged PR 4887: Adds audit helper + finishes KPIs Refactor Stage 1

# Description

Adds audit helper.
Removes computation of Total and Retained Revenue from cross models.

I've tested audit helper and the trick with the hashes, as discussed in the demo.

# Checklist

- [X] The edited models and dependants run properly with production data.
- [X] The edited models are sufficiently documented.
- [X] The edited models contain PK tests, and I've ran and passed them. **Except for today's outlier detector!**
- [X] I have checked for DRY opportunities with other models and docs.
- [X] I've picked the right materialization for the affected models.

# Other

- [ ] Check if a full-refresh is required after this PR is merged.

Related work items: #28946

models/intermediate/cross/int_monthly_aggregated_metrics_history_by_deal.sql

@@ -73,6 +73,11 @@ with
         select *
         from {{ ref("int_kpis__agg_monthly_host_resolutions") }}
         where dimension in ('by_deal') and dimension_value <> 'UNSET'
+    ),
+    total_and_retained_revenue as (
+        select *
+        from {{ ref("int_kpis__agg_monthly_total_and_retained_revenue") }}
+        where dimension in ('by_deal') and dimension_value <> 'UNSET'
     )
 
 select
@@ -182,8 +187,7 @@ select
     as host_resolution_amount_paid_per_created_booking,
     {{
         return_capped_value(
-            "cast(host_resolutions.xero_host_resolution_payment_count as decimal)
+            "cast(host_resolutions.xero_host_resolution_payment_count as decimal)/created_bookings.created_bookings",
-            / created_bookings.created_bookings",
             -1,
             1,
         )
@@ -203,12 +207,7 @@ select
     guest_payments.total_guest_payments_in_gbp,
 
     -- TOTAL REVENUE --
-    nullif(
+    total_and_retained_revenue.total_revenue_in_gbp,
-        coalesce(guest_payments.total_guest_payments_in_gbp, 0)
-        + coalesce(invoiced_revenue.xero_operator_net_fees_in_gbp, 0)
-        + coalesce(invoiced_revenue.xero_apis_net_fees_in_gbp, 0),
-        0
-    ) as total_revenue_in_gbp,
 
     -- GUEST REVENUE AND PAYMENTS WEIGHTED METRICS --
     guest_payments.total_guest_payments_in_gbp / nullif(
@@ -219,76 +218,31 @@ select
     ) as guest_payments_per_paid_guest_journey,
 
     -- TOTAL REVENUE WEIGHTED METRICS --
-    (
+    coalesce(total_and_retained_revenue.total_revenue_in_gbp, 0)
-        coalesce(guest_payments.total_guest_payments_in_gbp, 0)
-        + coalesce(invoiced_revenue.xero_operator_net_fees_in_gbp, 0)
-        + coalesce(invoiced_revenue.xero_apis_net_fees_in_gbp, 0)
-    )
     / nullif(created_bookings.created_bookings, 0) as total_revenue_per_created_booking,
-    (
+    coalesce(total_and_retained_revenue.total_revenue_in_gbp, 0) / nullif(
-        coalesce(guest_payments.total_guest_payments_in_gbp, 0)
-        + coalesce(invoiced_revenue.xero_operator_net_fees_in_gbp, 0)
-        + coalesce(invoiced_revenue.xero_apis_net_fees_in_gbp, 0)
-    ) / nullif(
         created_guest_journeys.created_guest_journeys, 0
     ) as total_revenue_per_created_guest_journey,
-    (
+    coalesce(total_and_retained_revenue.total_revenue_in_gbp, 0) / nullif(
-        coalesce(guest_payments.total_guest_payments_in_gbp, 0)
-        + coalesce(invoiced_revenue.xero_operator_net_fees_in_gbp, 0)
-        + coalesce(invoiced_revenue.xero_apis_net_fees_in_gbp, 0)
-    ) / nullif(
         listings.listings_booked_in_month, 0
     ) as total_revenue_per_listings_booked_in_month,
 
-    -- REVENUE RETAINED --
+    -- INCOME RETAINED --
-    nullif(
+    total_and_retained_revenue.revenue_retained_in_gbp,
-        coalesce(guest_payments.total_guest_payments_in_gbp, 0)
-        + coalesce(invoiced_revenue.xero_operator_net_fees_in_gbp, 0)
-        + coalesce(invoiced_revenue.xero_apis_net_fees_in_gbp, 0)
-        + coalesce(invoiced_revenue.xero_waiver_paid_back_to_host_in_gbp, 0),
-        0
-    ) as revenue_retained_in_gbp,
     {{
         return_capped_value(
-            "nullif(
+            "total_and_retained_revenue.revenue_retained_in_gbp/total_and_retained_revenue.total_revenue_in_gbp",
-                coalesce(guest_payments.total_guest_payments_in_gbp, 0)
-                + coalesce(invoiced_revenue.xero_operator_net_fees_in_gbp, 0)
-                + coalesce(invoiced_revenue.xero_apis_net_fees_in_gbp, 0)
-                + coalesce(invoiced_revenue.xero_waiver_paid_back_to_host_in_gbp, 0), 
-                0) / 
-             nullif(
-                coalesce(guest_payments.total_guest_payments_in_gbp, 0)
-                + coalesce(invoiced_revenue.xero_operator_net_fees_in_gbp, 0)
-                + coalesce(invoiced_revenue.xero_apis_net_fees_in_gbp, 0),
-                0)",
             -1,
             1,
         )
     }}
     as revenue_retained_ratio,
 
-    -- REVENUE RETAINED POST RESOLUTIONS--
+    -- INCOME RETAINED POST RESOLUTIONS--
-    nullif(
+    total_and_retained_revenue.revenue_retained_post_resolutions_in_gbp,
-        coalesce(guest_payments.total_guest_payments_in_gbp, 0)
-        + coalesce(invoiced_revenue.xero_operator_net_fees_in_gbp, 0)
-        + coalesce(invoiced_revenue.xero_apis_net_fees_in_gbp, 0)
-        + coalesce(invoiced_revenue.xero_waiver_paid_back_to_host_in_gbp, 0)
-        + coalesce(host_resolutions.xero_host_resolution_amount_paid_in_gbp, 0),
-        0
-    ) as revenue_retained_post_resolutions_in_gbp,
     {{
         return_capped_value(
-            "nullif(                 
+            "total_and_retained_revenue.revenue_retained_post_resolutions_in_gbp/total_and_retained_revenue.total_revenue_in_gbp",
-                coalesce(guest_payments.total_guest_payments_in_gbp, 0)                 
-                + coalesce(invoiced_revenue.xero_operator_net_fees_in_gbp, 0)                 
-                + coalesce(invoiced_revenue.xero_apis_net_fees_in_gbp, 0)                 
-                + coalesce(invoiced_revenue.xero_waiver_paid_back_to_host_in_gbp, 0)                
-                + coalesce(host_resolutions.xero_host_resolution_amount_paid_in_gbp, 0),                 
-                0) / 
-             nullif(coalesce(guest_payments.total_guest_payments_in_gbp, 0)
-                + coalesce(invoiced_revenue.xero_operator_net_fees_in_gbp, 0)
-                + coalesce(invoiced_revenue.xero_apis_net_fees_in_gbp, 0),
-                0)",
             -1,
             1,
         )
@@ -347,3 +301,7 @@ left join
     host_resolutions
     on d.date = host_resolutions.end_date
     and d.dimension_value = host_resolutions.dimension_value
+left join
+    total_and_retained_revenue
+    on d.date = total_and_retained_revenue.end_date
+    and d.dimension_value = total_and_retained_revenue.dimension_value

models/intermediate/cross/int_mtd_vs_previous_year_metrics.sql

@@ -150,6 +150,19 @@ with
             dimension in ('global', 'by_number_of_listings', 'by_billing_country', 'by_business_scope')
             and dimension_value <> 'UNSET'
     ),
+    total_and_retained_revenue as (
+        select *
+        from {{ ref("int_kpis__agg_mtd_total_and_retained_revenue") }}
+        where
+            dimension in ('global', 'by_number_of_listings', 'by_billing_country', 'by_business_scope')
+            and dimension_value <> 'UNSET'
+        union all
+        select *
+        from {{ ref("int_kpis__agg_monthly_total_and_retained_revenue") }}
+        where
+            dimension in ('global', 'by_number_of_listings', 'by_billing_country', 'by_business_scope')
+            and dimension_value <> 'UNSET'
+    ),
     int_monthly_churn_metrics as (select * from {{ ref("int_monthly_churn_metrics") }}),
     int_kpis__agg_dates_main_kpis as (
         select *
@@ -289,12 +302,7 @@ with
             guest_payments.total_guest_payments_in_gbp,
 
             -- TOTAL REVENUE --
-            nullif(
+            total_and_retained_revenue.total_revenue_in_gbp,
-                coalesce(guest_payments.total_guest_payments_in_gbp, 0)
-                + coalesce(invoiced_revenue.xero_operator_net_fees_in_gbp, 0)
-                + coalesce(invoiced_revenue.xero_apis_net_fees_in_gbp, 0),
-                0
-            ) as total_revenue_in_gbp,
 
             -- GUEST REVENUE AND PAYMENTS WEIGHTED METRICS --
             guest_payments.total_guest_payments_in_gbp / nullif(
@@ -305,32 +313,16 @@ with
             ) as guest_payments_per_paid_guest_journey,
 
             -- TOTAL REVENUE WEIGHTED METRICS --
-            (
+            coalesce(total_and_retained_revenue.total_revenue_in_gbp,0) / nullif(
-                coalesce(guest_payments.total_guest_payments_in_gbp, 0)
-                + coalesce(invoiced_revenue.xero_operator_net_fees_in_gbp, 0)
-                + coalesce(invoiced_revenue.xero_apis_net_fees_in_gbp, 0)
-            ) / nullif(
                 created_bookings.created_bookings, 0
             ) as total_revenue_per_created_booking,
-            (
+            coalesce(total_and_retained_revenue.total_revenue_in_gbp,0) / nullif(
-                coalesce(guest_payments.total_guest_payments_in_gbp, 0)
-                + coalesce(invoiced_revenue.xero_operator_net_fees_in_gbp, 0)
-                + coalesce(invoiced_revenue.xero_apis_net_fees_in_gbp, 0)
-            ) / nullif(
                 created_guest_journeys.created_guest_journeys, 0
             ) as total_revenue_per_created_guest_journey,
-            (
+            coalesce(total_and_retained_revenue.total_revenue_in_gbp,0) / nullif(
-                coalesce(guest_payments.total_guest_payments_in_gbp, 0)
-                + coalesce(invoiced_revenue.xero_operator_net_fees_in_gbp, 0)
-                + coalesce(invoiced_revenue.xero_apis_net_fees_in_gbp, 0)
-            ) / nullif(
                 deals.deals_booked_in_month, 0
             ) as total_revenue_per_deals_booked_in_month,
-            (
+            coalesce(total_and_retained_revenue.total_revenue_in_gbp,0) / nullif(
-                coalesce(guest_payments.total_guest_payments_in_gbp, 0)
-                + coalesce(invoiced_revenue.xero_operator_net_fees_in_gbp, 0)
-                + coalesce(invoiced_revenue.xero_apis_net_fees_in_gbp, 0)
-            ) / nullif(
                 listings.listings_booked_in_month, 0
             ) as total_revenue_per_listings_booked_in_month,
 
@@ -342,26 +334,11 @@ with
             churn.listings_booked_in_month_churn_average_contribution,
 
             -- INCOME RETAINED --
-            nullif(
+            total_and_retained_revenue.revenue_retained_in_gbp,
-                coalesce(guest_payments.total_guest_payments_in_gbp, 0)
-                + coalesce(invoiced_revenue.xero_operator_net_fees_in_gbp, 0)
-                + coalesce(invoiced_revenue.xero_apis_net_fees_in_gbp, 0)
-                + coalesce(invoiced_revenue.xero_waiver_paid_back_to_host_in_gbp, 0),
-                0
-            ) as revenue_retained_in_gbp,
             {{
                 return_capped_value(
-                    "nullif(
+                    "total_and_retained_revenue.revenue_retained_in_gbp 
-                        coalesce(guest_payments.total_guest_payments_in_gbp, 0)
+                    / total_and_retained_revenue.total_revenue_in_gbp",
-                        + coalesce(invoiced_revenue.xero_operator_net_fees_in_gbp, 0)
-                        + coalesce(invoiced_revenue.xero_apis_net_fees_in_gbp, 0)
-                        + coalesce(invoiced_revenue.xero_waiver_paid_back_to_host_in_gbp, 0)
-                        ,0) 
-                    / nullif(
-                        coalesce(guest_payments.total_guest_payments_in_gbp, 0)                         
-                        + coalesce(invoiced_revenue.xero_operator_net_fees_in_gbp, 0)                         
-                        + coalesce(invoiced_revenue.xero_apis_net_fees_in_gbp, 0)
-                        ,0)",
                     -1,
                     1,
                 )
@@ -369,28 +346,11 @@ with
             as revenue_retained_ratio,
 
             -- INCOME RETAINED POST RESOLUTIONS--
-            nullif(
+            total_and_retained_revenue.revenue_retained_post_resolutions_in_gbp,
-                coalesce(guest_payments.total_guest_payments_in_gbp, 0)
-                + coalesce(invoiced_revenue.xero_operator_net_fees_in_gbp, 0)
-                + coalesce(invoiced_revenue.xero_apis_net_fees_in_gbp, 0)
-                + coalesce(invoiced_revenue.xero_waiver_paid_back_to_host_in_gbp, 0)
-                + coalesce(host_resolutions.xero_host_resolution_amount_paid_in_gbp, 0),
-                0
-            ) as revenue_retained_post_resolutions_in_gbp,
             {{
                 return_capped_value(
-                    "nullif(
+                    "total_and_retained_revenue.revenue_retained_post_resolutions_in_gbp 
-                        coalesce(guest_payments.total_guest_payments_in_gbp, 0)
+                    / total_and_retained_revenue.total_revenue_in_gbp",
-                        + coalesce(invoiced_revenue.xero_operator_net_fees_in_gbp, 0)
-                        + coalesce(invoiced_revenue.xero_apis_net_fees_in_gbp, 0)
-                        + coalesce(invoiced_revenue.xero_waiver_paid_back_to_host_in_gbp, 0)
-                        + coalesce(host_resolutions.xero_host_resolution_amount_paid_in_gbp, 0)
-                        ,0) 
-                    / nullif(
-                        coalesce(guest_payments.total_guest_payments_in_gbp, 0)
-                        + coalesce(invoiced_revenue.xero_operator_net_fees_in_gbp, 0)
-                        + coalesce(invoiced_revenue.xero_apis_net_fees_in_gbp, 0),
-                        0)",
                     -1,
                     1,
                 )
@@ -477,6 +437,11 @@ with
             on d.date = onboarding_mrr_revenue.date
             and d.dimension = onboarding_mrr_revenue.dimension
             and d.dimension_value = onboarding_mrr_revenue.dimension_value
+        left join 
+            total_and_retained_revenue
+            on d.date = total_and_retained_revenue.end_date
+            and d.dimension = total_and_retained_revenue.dimension
+            and d.dimension_value = total_and_retained_revenue.dimension_value
     )
 
 select

package-lock.yml

@@ -5,6 +5,8 @@ packages:
     version: 0.9.0
   - package: dbt-labs/dbt_utils
     version: 1.2.0
+  - package: dbt-labs/audit_helper
+    version: 0.12.1
   - package: calogica/dbt_date
     version: 0.8.1
-sha1_hash: ceec21d8037429db57330b6f23cfdc761bbb7698
+sha1_hash: ef447e4c6ef3a63e0f8fa3d3ec7db2f888bb0930

packages.yml

@@ -4,4 +4,6 @@ packages:
   - package: calogica/dbt_expectations
     version: [">=0.9.0", "<0.10.0"]
   - package: dbt-labs/dbt_utils
-    version: 1.2.0
+    version: 1.2.0
+  - package: dbt-labs/audit_helper
+    version: 0.12.1