Merged PR 4504: Onboarding MRR + Visualisation changes

# Description

Changes:
* Adds Onboarding MRR
* Refactors exclusion code for ongoing month / invoicing cycle
* Adds sign format on relative differences

# Checklist

- [X] The edited models and dependants run properly with production data.
- [X] The edited models are sufficiently documented.
- [X] The edited models contain PK tests, and I've ran and passed them.
- [X] I have checked for DRY opportunities with other models and docs.
- [X] I've picked the right materialization for the affected models.

# Other

- [ ] Check if a full-refresh is required after this PR is merged.

Related work items: #27609, #27805

models/intermediate/cross/int_ytd_mtd_aggregated_main_metrics_overview.sql

@@ -7,7 +7,8 @@
         "previous_year_MTD": "previous_year_mtd_total_revenue_in_gbp",
         "current_YTD": "current_ytd_total_revenue_in_gbp",
         "previous_YTD": "previous_ytd_total_revenue_in_gbp",
-        "requires_invoicing_data": true,
+        "display_exclusion": "INVOICING",
+        "increment_sign_format": "POSITIVE",
     },
     {
         "id_metric": 2,
@@ -17,7 +18,8 @@
         "previous_year_MTD": "previous_year_mtd_revenue_retained_post_resolutions_in_gbp",
         "current_YTD": "current_ytd_revenue_retained_post_resolutions_in_gbp",
         "previous_YTD": "previous_ytd_revenue_retained_post_resolutions_in_gbp",
-        "requires_invoicing_data": true,
+        "display_exclusion": "INVOICING",
+        "increment_sign_format": "POSITIVE",
     },
     {
         "id_metric": 3,
@@ -27,7 +29,8 @@
         "previous_year_MTD": "previous_year_mtd_total_guest_payments_in_gbp",
         "current_YTD": "current_ytd_total_guest_payments_in_gbp",
         "previous_YTD": "previous_ytd_total_guest_payments_in_gbp",
-        "requires_invoicing_data": false,
+        "display_exclusion": "NONE",
+        "increment_sign_format": "POSITIVE",
     },
     {
         "id_metric": 4,
@@ -37,7 +40,8 @@
         "previous_year_MTD": "previous_year_mtd_xero_operator_net_fees_in_gbp",
         "current_YTD": "current_ytd_xero_operator_net_fees_in_gbp",
         "previous_YTD": "previous_ytd_xero_operator_net_fees_in_gbp",
-        "requires_invoicing_data": true,
+        "display_exclusion": "INVOICING",
+        "increment_sign_format": "POSITIVE",
     },
     {
         "id_metric": 5,
@@ -47,7 +51,8 @@
         "previous_year_MTD": "previous_year_mtd_xero_apis_net_fees_in_gbp",
         "current_YTD": "current_ytd_xero_apis_net_fees_in_gbp",
         "previous_YTD": "previous_ytd_xero_apis_net_fees_in_gbp",
-        "requires_invoicing_data": true,
+        "display_exclusion": "INVOICING",
+        "increment_sign_format": "POSITIVE",
     },
     {
         "id_metric": 6,
@@ -57,7 +62,8 @@
         "previous_year_MTD": "previous_year_mtd_billable_bookings",
         "current_YTD": "current_ytd_billable_bookings",
         "previous_YTD": "previous_ytd_billable_bookings",
-        "requires_invoicing_data": false,
+        "display_exclusion": "NONE",
+        "increment_sign_format": "POSITIVE",
     },
     {
         "id_metric": 7,
@@ -67,7 +73,8 @@
         "previous_year_MTD": "previous_year_mtd_live_deals",
         "current_YTD": "current_ytd_live_deals",
         "previous_YTD": "previous_ytd_live_deals",
-        "requires_invoicing_data": false,
+        "display_exclusion": "NONE",
+        "increment_sign_format": "POSITIVE",
     },
     {
         "id_metric": 8,
@@ -77,7 +84,8 @@
         "previous_year_MTD": "previous_year_mtd_new_deals",
         "current_YTD": "current_ytd_new_deals",
         "previous_YTD": "previous_ytd_new_deals",
-        "requires_invoicing_data": false,
+        "display_exclusion": "NONE",
+        "increment_sign_format": "POSITIVE",
     },
     {
         "id_metric": 9,
@@ -87,7 +95,8 @@
         "previous_year_MTD": "previous_year_mtd_churning_deals",
         "current_YTD": "current_ytd_churning_deals",
         "previous_YTD": "previous_ytd_churning_deals",
-        "requires_invoicing_data": false,
+        "display_exclusion": "NONE",
+        "increment_sign_format": "NEGATIVE",
     },
     {
         "id_metric": 10,
@@ -97,7 +106,8 @@
         "previous_year_MTD": "previous_year_mtd_waiver_payout_rate",
         "current_YTD": "current_ytd_waiver_payout_rate",
         "previous_YTD": "previous_ytd_waiver_payout_rate",
-        "requires_invoicing_data": true,
+        "display_exclusion": "INVOICING",
+        "increment_sign_format": "NEGATIVE",
     },
     {
         "id_metric": 11,
@@ -107,7 +117,8 @@
         "previous_year_MTD": "previous_year_mtd_resolutions_payout_rate",
         "current_YTD": "current_ytd_resolutions_payout_rate",
         "previous_YTD": "previous_ytd_resolutions_payout_rate",
-        "requires_invoicing_data": true,
+        "display_exclusion": "INVOICING",
+        "increment_sign_format": "NEGATIVE",
     },
     {
         "id_metric": 12,
@@ -117,7 +128,8 @@
         "previous_year_MTD": "previous_year_mtd_operator_revenue_per_billable_booking",
         "current_YTD": "current_ytd_operator_revenue_per_billable_booking",
         "previous_YTD": "previous_ytd_operator_revenue_per_billable_booking",
-        "requires_invoicing_data": true,
+        "display_exclusion": "INVOICING",
+        "increment_sign_format": "POSITIVE",
     },
     {
         "id_metric": 13,
@@ -127,7 +139,8 @@
         "previous_year_MTD": "previous_year_mtd_waiver_revenue_per_billable_booking",
         "current_YTD": "current_ytd_waiver_revenue_per_billable_booking",
         "previous_YTD": "previous_ytd_waiver_revenue_per_billable_booking",
-        "requires_invoicing_data": false,
+        "display_exclusion": "NONE",
+        "increment_sign_format": "POSITIVE",
     },
     {
         "id_metric": 14,
@@ -137,7 +150,8 @@
         "previous_year_MTD": "-1*previous_year_mtd_xero_waiver_paid_back_to_host_in_gbp",
         "current_YTD": "-1*current_ytd_xero_waiver_paid_back_to_host_in_gbp",
         "previous_YTD": "-1*previous_ytd_xero_waiver_paid_back_to_host_in_gbp",
-        "requires_invoicing_data": true,
+        "display_exclusion": "INVOICING",
+        "increment_sign_format": "NEGATIVE",
     },
     {
         "id_metric": 15,
@@ -147,7 +161,8 @@
         "previous_year_MTD": "-1*previous_year_mtd_xero_host_resolution_amount_paid_in_gbp",
         "current_YTD": "-1*current_ytd_xero_host_resolution_amount_paid_in_gbp",
         "previous_YTD": "-1*previous_ytd_xero_host_resolution_amount_paid_in_gbp",
-        "requires_invoicing_data": true,
+        "display_exclusion": "INVOICING",
+        "increment_sign_format": "NEGATIVE",
     },
     {
         "id_metric": 16,
@@ -157,7 +172,19 @@
         "previous_year_MTD": "previous_year_mtd_total_revenue_churn_rate",
         "current_YTD": "current_ytd_total_revenue_churn_rate",
         "previous_YTD": "previous_ytd_total_revenue_churn_rate",
-        "requires_invoicing_data": false,
+        "display_exclusion": "ONGOING_MONTH",
+        "increment_sign_format": "NEGATIVE",
+    },
+    {
+        "id_metric": 17,
+        "name": "Onboarding MRR",
+        "current_month_MTD": "current_month_mtd_onboarding_mrr",
+        "previous_month_EOM": "previous_month_eom_onboarding_mrr",
+        "previous_year_MTD": "previous_year_mtd_onboarding_mrr",
+        "current_YTD": "current_ytd_onboarding_mrr",
+        "previous_YTD": "previous_ytd_onboarding_mrr",
+        "display_exclusion": "ONGOING_MONTH",
+        "increment_sign_format": "POSITIVE",
     },
 ] %}
 with
@@ -179,7 +206,8 @@ with
                 {{ metric.id_metric }} as id_metric,
                 -- quotation marks added because text format
                 '{{ metric.name }}' as metric_name,
-                {{ metric.requires_invoicing_data }} as requires_invoicing_data,
+                '{{ metric.display_exclusion }}' as display_exclusion,
+                '{{ metric.increment_sign_format }}' as increment_sign_format,
                 {{ metric.current_month_MTD }} as current_month_mtd,
                 {{ metric.previous_month_EOM }} as previous_month_eom,
                 {{ metric.previous_year_MTD }} as previous_year_mtd,
@@ -206,30 +234,63 @@ with
                 union all
             {% endif %}
         {% endfor %}
+    ),
+    metrics_with_targets as (
+        select
+            metrics.*,
+            -- EOM target --
+            targets.target_eom_value as target_eom_value,
+            metrics.current_month_mtd
+            - targets.target_eom_value as diff_current_month_mtd_vs_eom_target,
+            metrics.current_month_mtd / nullif(targets.target_eom_value, 0)
+            - 1 as rel_diff_current_month_mtd_vs_eom_target,
+            -- YTD target --
+            targets.target_ytd_value as target_ytd_value,
+            metrics.current_year_ytd
+            - targets.target_ytd_value as diff_current_ytd_vs_ytd_target,
+            metrics.current_year_ytd / nullif(targets.target_ytd_value, 0)
+            - 1 as rel_diff_current_ytd_vs_ytd_target,
+            -- EOFY target --
+            targets.target_eofy_value as target_eofy_value,
+            metrics.current_year_ytd
+            - targets.target_eofy_value as diff_current_ytd_vs_eofy_target,
+            metrics.current_year_ytd / nullif(
+                targets.target_eofy_value, 0
+            ) as achievement_rate_current_ytd_vs_eofy_target
+        from main_metrics_aggregation as metrics
+        left join
+            stg_seed__main_metrics_targets as targets
+            on metrics.id_metric = targets.id_metric
+            and date_trunc('month', metrics.date)
+            = date_trunc('month', targets.target_date)
     )
 select
-    metrics.*,
+    metrics_with_targets.*,
-    -- EOM target --
+    -- Add sign to relative differences --
-    targets.target_eom_value as target_eom_value,
+    case
-    metrics.current_month_mtd
+        when increment_sign_format = 'NEGATIVE'
-    - targets.target_eom_value as diff_current_month_mtd_vs_eom_target,
+        then rel_diff_current_month_mtd_vs_previous_month_eom * -1
-    metrics.current_month_mtd / nullif(targets.target_eom_value, 0)
+        else rel_diff_current_month_mtd_vs_previous_month_eom
-    - 1 as rel_diff_current_month_mtd_vs_eom_target,
+    end as rel_diff_with_sign_current_month_mtd_vs_previous_month_eom,
-    -- YTD target --
+    case
-    targets.target_ytd_value as target_ytd_value,
+        when increment_sign_format = 'NEGATIVE'
-    metrics.current_year_ytd
+        then rel_diff_current_month_mtd_vs_previous_year_mtd * -1
-    - targets.target_ytd_value as diff_current_ytd_vs_ytd_target,
+        else rel_diff_current_month_mtd_vs_previous_year_mtd
-    metrics.current_year_ytd / nullif(targets.target_ytd_value, 0)
+    end as rel_diff_with_sign_current_month_mtd_vs_previous_year_mtd,
-    - 1 as rel_diff_current_ytd_vs_ytd_target,
+    case
-    -- EOFY target --
+        when increment_sign_format = 'NEGATIVE'
-    targets.target_eofy_value as target_eofy_value,
+        then rel_diff_current_ytd_vs_previous_ytd * -1
-    metrics.current_year_ytd
+        else rel_diff_current_ytd_vs_previous_ytd
-    - targets.target_eofy_value as diff_current_ytd_vs_eofy_target,
+    end as rel_diff_with_sign_current_ytd_vs_previous_ytd,
-    metrics.current_year_ytd / nullif(
+    case
-        targets.target_eofy_value, 0
+        when increment_sign_format = 'NEGATIVE'
-    ) as achievement_rate_current_ytd_vs_eofy_target
+        then rel_diff_current_month_mtd_vs_eom_target * -1
-from main_metrics_aggregation as metrics
+        else rel_diff_current_month_mtd_vs_eom_target
-left join
+    end as rel_diff_with_sign_current_month_mtd_vs_eom_target,
-    stg_seed__main_metrics_targets as targets
+    case
-    on metrics.id_metric = targets.id_metric
+        when increment_sign_format = 'NEGATIVE'
-    and date_trunc('month', metrics.date) = date_trunc('month', targets.target_date)
+        then rel_diff_current_ytd_vs_ytd_target * -1
+        else rel_diff_current_ytd_vs_ytd_target
+    end as rel_diff_with_sign_current_ytd_vs_ytd_target
+
+from metrics_with_targets

models/intermediate/cross/int_ytd_mtd_main_metrics_overview.sql

@@ -40,6 +40,7 @@ with
             as current_month_mtd_total_revenue_churn_12m,
             total_revenue_global_preceding_12_months
             as current_month_mtd_total_revenue_global_12m,
+            expected_mrr as current_month_mtd_onboarding_mrr,
 
             -- Previous Year (12 months ago), Month To Date Metrics --
             previous_year_total_revenue_in_gbp
@@ -66,6 +67,7 @@ with
             as previous_year_mtd_total_revenue_churn_12m,
             previous_year_total_revenue_global_preceding_12_months
             as previous_year_mtd_total_revenue_global_12m,
+            previous_year_expected_mrr as previous_year_mtd_onboarding_mrr,
 
             -- Previous Month, End Of Month Metrics --
             lag(total_revenue_in_gbp) over (
@@ -109,7 +111,10 @@ with
             ) as previous_month_eom_total_revenue_churn_12m,
             lag(total_revenue_global_preceding_12_months) over (
                 partition by dimension, dimension_value order by date
-            ) as previous_month_eom_total_revenue_global_12m
+            ) as previous_month_eom_total_revenue_global_12m,
+            lag(expected_mrr) over (
+                partition by dimension, dimension_value order by date
+            ) as previous_month_eom_onboarding_mrr
 
         from int_mtd_vs_previous_year_metrics
         where
@@ -188,6 +193,11 @@ with
                 order by date
                 rows between unbounded preceding and current row
             ) as current_ytd_total_revenue_global_12m,
+            sum(current_month_mtd_onboarding_mrr) over (
+                partition by financial_year, dimension, dimension_value
+                order by date
+                rows between unbounded preceding and current row
+            ) as current_ytd_onboarding_mrr,
             -- Specific treatment for live_deals as it is a counter
             current_month_mtd_live_deals as current_ytd_live_deals,
 
@@ -257,6 +267,11 @@ with
                 order by date
                 rows between unbounded preceding and current row
             ) as previous_ytd_total_revenue_global_12m,
+            sum(previous_year_mtd_onboarding_mrr) over (
+                partition by financial_year, dimension, dimension_value
+                order by date
+                rows between unbounded preceding and current row
+            ) as previous_ytd_onboarding_mrr,
             -- Specific treatment for live_deals as it is a counter
             previous_year_mtd_live_deals as previous_ytd_live_deals
 

models/intermediate/cross/schema.yml

@@ -1989,6 +1989,11 @@ models:
         description: |
           Total revenue churn rate for the current month MTD.
 
+      - name: current_month_mtd_onboarding_mrr
+        data_type: numeric
+        description: |
+          Total expected Onboarding MRR for the current month MTD.
+
       - name: previous_year_mtd_billable_bookings
         data_type: integer
         description: |
@@ -2099,6 +2104,12 @@ models:
           Total revenue churn rate for the previous year 
           (12 months ago) MTD.
 
+      - name: previous_year_mtd_onboarding_mrr
+        data_type: numeric
+        description: |
+          Total expected Onboarding MRR for the previous year 
+          (12 months ago) MTD.
+
       - name: current_ytd_billable_bookings
         data_type: integer
         description: |
@@ -2207,6 +2218,11 @@ models:
         description: |
           Total revenue churn rate for the current financial year YTD.
 
+      - name: current_ytd_onboarding_mrr
+        data_type: numeric
+        description: |
+          Total expected Onboarding MRR for the current financial year YTD.
+
       - name: previous_ytd_billable_bookings
         data_type: integer
         description: |
@@ -2316,6 +2332,12 @@ models:
           Total revenue churn rate for the previous 
           financial year YTD.
 
+      - name: previous_ytd_onboarding_mrr
+        data_type: numeric
+        description: |
+          Total expected Onboarding MRR for the previous 
+          financial year YTD.
+
       - name: previous_month_eom_billable_bookings
         data_type: integer
         description: |
@@ -2432,6 +2454,12 @@ models:
           Total revenue churn rate for the previous month, 
           at the end of the month.
 
+      - name: previous_month_eom_onboarding_mrr
+        data_type: numeric
+        description: |
+          Total expected Onboarding MRR for the previous month, 
+          at the end of the month.
+
   - name: int_ytd_mtd_aggregated_main_metrics_overview
     description: |
       This model provides a high-level overview of the main metrics for the month-to-date 
@@ -2510,13 +2538,19 @@ models:
         data_tests:
           - not_null
 
-      - name: requires_invoicing_data
+      - name: display_exclusion
-        data_type: boolean
+        data_type: string
         description: |
-          Flag to indicate if the metric requires invoicing data to be calculated.
+          Category to indicate if the metric requires a certain exclusion due 
+          to relying on not timely information.
           This will limit the display for reporting purposes.
         data_tests:
           - not_null
+          - accepted_values:
+              values:
+                - NONE
+                - INVOICING
+                - ONGOING_MONTH
 
       - name: current_month_mtd
         data_type: numeric
@@ -2630,3 +2664,38 @@ models:
         description: |
           Achievement rate between the current year YTD and the EOFY target. It can be null
           if the target is not available.
+
+      - name: rel_diff_with_sign_current_month_mtd_vs_previous_month_eom
+        data_type: numeric
+        description: |
+          Relative difference between the current month MTD and the previous month EOM, 
+          with a sign to represent if the relative difference is good (positive) or bad 
+          (negative) for our business.
+
+      - name: rel_diff_with_sign_current_month_mtd_vs_previous_year_mtd
+        data_type: numeric
+        description: |
+          Relative difference between the current month MTD and the previous year MTD, 
+          with a sign to represent if the relative difference is good (positive) or bad 
+          (negative) for our business.
+
+      - name: rel_diff_with_sign_current_ytd_vs_previous_ytd
+        data_type: numeric
+        description: |
+          Relative difference between the current year YTD and the previous year YTD, 
+          with a sign to represent if the relative difference is good (positive) or bad 
+          (negative) for our business.
+
+      - name: rel_diff_with_sign_current_month_mtd_vs_eom_target
+        data_type: numeric
+        description: |
+          Relative difference between the current month MTD and the EOM target, 
+          with a sign to represent if the relative difference is good (positive) or bad 
+          (negative) for our business.
+
+      - name: rel_diff_with_sign_current_ytd_vs_ytd_target
+        data_type: numeric
+        description: |
+          Relative difference between the current year YTD and the YTD target, 
+          with a sign to represent if the relative difference is good (positive) or bad 
+          (negative) for our business.

models/reporting/general/schema.yml

@@ -2131,3 +2131,38 @@ models:
         description: |
           Achievement rate between the current year YTD and the EOFY target. It can be null
           if the target is not available.
+
+      - name: rel_diff_with_sign_current_month_mtd_vs_previous_month_eom
+        data_type: numeric
+        description: |
+          Relative difference between the current month MTD and the previous month EOM, 
+          with a sign to represent if the relative difference is good (positive) or bad 
+          (negative) for our business.
+
+      - name: rel_diff_with_sign_current_month_mtd_vs_previous_year_mtd
+        data_type: numeric
+        description: |
+          Relative difference between the current month MTD and the previous year MTD, 
+          with a sign to represent if the relative difference is good (positive) or bad 
+          (negative) for our business.
+
+      - name: rel_diff_with_sign_current_ytd_vs_previous_ytd
+        data_type: numeric
+        description: |
+          Relative difference between the current year YTD and the previous year YTD, 
+          with a sign to represent if the relative difference is good (positive) or bad 
+          (negative) for our business.
+
+      - name: rel_diff_with_sign_current_month_mtd_vs_eom_target
+        data_type: numeric
+        description: |
+          Relative difference between the current month MTD and the EOM target, 
+          with a sign to represent if the relative difference is good (positive) or bad 
+          (negative) for our business.
+
+      - name: rel_diff_with_sign_current_ytd_vs_ytd_target
+        data_type: numeric
+        description: |
+          Relative difference between the current year YTD and the YTD target, 
+          with a sign to represent if the relative difference is good (positive) or bad 
+          (negative) for our business.

models/reporting/general/ytd_mtd_aggregated_main_metrics_overview.sql

@@ -3,8 +3,9 @@ with
         select * from {{ ref("int_ytd_mtd_aggregated_main_metrics_overview") }}
     ),
     latest_dates_per_financial_year as (
-        select dimension, financial_year, id_metric, max(date) as latest_available_date
+        select
-        from int_ytd_mtd_aggregated_main_metrics_overview
+            dimension, financial_year, id_metric, max(m.date) as latest_available_date
+        from int_ytd_mtd_aggregated_main_metrics_overview m
         where
             (
                 (
@@ -12,20 +13,17 @@ with
                     -- invoicing cycle and it is before the 20th of the month, if it
                     -- is the 20th of the month or after, only exclude the current
                     -- month.
-                    requires_invoicing_data = true
+                    display_exclusion = 'INVOICING'
                     and {{ is_date_before_20th_of_previous_month("date") }}
                 )
-                -- Keep all history for the rest of metrics
+                or (
-                or requires_invoicing_data = false
+                    -- Handle exclusion for Churn/MRR metrics: do not show them in the
-            )
+                    -- current month.
-            -- Handle exclusion for Churn/MRR metrics: do not show them in the current
+                    display_exclusion = 'ONGOING_MONTH'
-            -- month.
+                    and date_trunc('month', m.date) > date_trunc('month', current_date)
-            and not (
-                (
-                    lower(metric_name) like '%revenue%churn%rate%'
-                    or lower(metric_name) like '%onboarding%mrr%'
                 )
-                and date_trunc('month', "date") = date_trunc('month', current_date)
+                -- Keep all history for the rest of metrics
+                or display_exclusion = 'NONE'
             )
         group by dimension, financial_year, id_metric
     )
@@ -65,7 +63,17 @@ select
     m.target_eofy_value as target_eofy_value,
     m.diff_current_ytd_vs_eofy_target as diff_current_ytd_vs_eofy_target,
     m.achievement_rate_current_ytd_vs_eofy_target
-    as achievement_rate_current_ytd_vs_eofy_target
+    as achievement_rate_current_ytd_vs_eofy_target,
+    m.rel_diff_with_sign_current_month_mtd_vs_previous_month_eom
+    as rel_diff_with_sign_current_month_mtd_vs_previous_month_eom,
+    m.rel_diff_with_sign_current_month_mtd_vs_previous_year_mtd
+    as rel_diff_with_sign_current_month_mtd_vs_previous_year_mtd,
+    m.rel_diff_with_sign_current_ytd_vs_previous_ytd
+    as rel_diff_with_sign_current_ytd_vs_previous_ytd,
+    m.rel_diff_with_sign_current_month_mtd_vs_eom_target
+    as rel_diff_with_sign_current_month_mtd_vs_eom_target,
+    m.rel_diff_with_sign_current_ytd_vs_ytd_target
+    as rel_diff_with_sign_current_ytd_vs_ytd_target
 from int_ytd_mtd_aggregated_main_metrics_overview m
 inner join
     latest_dates_per_financial_year ld