Merged PR 4395: Propagates business scope into Deal/Listing metrics

# Description

Changes:
* Propagates business scope, based on deal, for Deal and Listing metrics. This already handles the daily metric and the daily aggregation.
* Modifies lifecycle_daily_deal to depend on dimension_deals and compute API segmentation.
* Creates new metric: Live Deals, that includes New, Active and Reactivated. This will be needed for YTD/MTD overview.

# Checklist

- [X] The edited models and dependants run properly with production data.
- [X] The edited models are sufficiently documented.
- [X] The edited models contain PK tests, and I've ran and passed them.
- [X] I have checked for DRY opportunities with other models and docs.
- [X] I've picked the right materialization for the affected models.

# Other

- [ ] Check if a full-refresh is required after this PR is merged.

Related work items: #27356

macros/business_kpis_configuration.sql

@@ -145,6 +145,8 @@ Provides a general assignement for the Dimensions available for each KPI
         "STARTED_GUEST_JOURNEYS",
         "INVOICED_REVENUE",
         "HOST_RESOLUTIONS",
+        "LISTINGS",
+        "DEALS",
     ] %}
         {% set additional_dimensions = additional_dimensions + [dim_business_scope()] %}
     {% endif %}

models/intermediate/kpis/int_kpis__agg_daily_deals.sql

@@ -22,7 +22,8 @@
         sum(reactivated_deals) as reactivated_deals,
         sum(deals_booked_in_month) as deals_booked_in_month,
         sum(deals_booked_in_6_months) as deals_booked_in_6_months,
-        sum(deals_booked_in_12_months) as deals_booked_in_12_months
+        sum(deals_booked_in_12_months) as deals_booked_in_12_months,
+        sum(live_deals) as live_deals
     from {{ ref("int_kpis__dimension_dates") }} d
     left join {{ ref("int_kpis__metric_daily_deals") }} as mdd on d.date = mdd.date
     group by 1, 2, 3, 4, 5, 6

models/intermediate/kpis/int_kpis__lifecycle_daily_deal.sql

@@ -8,7 +8,7 @@ with
     int_core__bookings as (select * from {{ ref("int_core__bookings") }}),
     int_core__user_host as (select * from {{ ref("int_core__user_host") }}),
     int_kpis__dimension_dates as (select * from {{ ref("int_kpis__dimension_dates") }}),
-    int_hubspot__deal as (select * from {{ ref("int_hubspot__deal") }}),
+    int_kpis__dimension_deals as (select * from {{ ref("int_kpis__dimension_deals") }}),
 
     hubspot_deal_offboardings as (
         -- At the moment it's not possible to account for deal reactivation within
@@ -21,9 +21,9 @@ with
         -- discussing with our colleagues from sales / am teams. In this discussion
         -- period, the stage could be different than cancelled, but still not
         -- reactivated.
-        select id_deal, cancellation_date_utc
+        select id_deal, hubspot_deal_cancellation_date_utc as cancellation_date_utc
-        from int_hubspot__deal
+        from int_kpis__dimension_deals
-        where cancellation_date_utc is not null
+        where hubspot_deal_cancellation_date_utc is not null
     ),
     booked_days_per_deal as (
         select
@@ -34,37 +34,37 @@ with
             ) as previous_booked_date
         from int_core__bookings icb
         inner join int_core__user_host icuh on icb.id_user_host = icuh.id_user_host
+        -- Ensure only Platform deals are considered (exclude API deals)
+        inner join
+            int_kpis__dimension_deals ikdd
+            on icuh.id_deal = ikdd.id_deal
+            and ikdd.client_type = 'PLATFORM'
         where icuh.id_deal is not null
         group by icuh.id_deal, icb.created_date_utc
     ),
     deals as (
         select
-            coalesce(hd.id_deal, h.id_deal) as id_deal,
+            ikdd.id_deal,
-            min(coalesce(hd.live_date_utc, h.created_date_utc)) as created_date_utc
+            ikdd.client_type,
-        from int_hubspot__deal hd
+            ikdd.effective_deal_start_date_utc as created_date_utc
-        full outer join
+        from int_kpis__dimension_deals ikdd
-            int_core__user_host h
-            on hd.id_deal = h.id_deal
-        where hd.id_deal is not null
-        and h.id_deal is not null
-        group by 1
     ),
     deal_historic_booking_dates as (
         select
             d.date,
-            h.id_deal,
+            ikdd.id_deal,
-            min(h.created_date_utc) as creation_date_utc,
+            min(ikdd.client_type) as client_type,
+            min(ikdd.created_date_utc) as creation_date_utc,
             min(b.created_date_utc) as first_time_booked_date_utc,
             max(b.created_date_utc) as last_time_booked_date_utc,
             max(b.previous_booked_date) as second_to_last_time_booked_date_utc
         from int_kpis__dimension_dates d
-        inner join deals h on d.date >= h.created_date_utc
+        inner join deals ikdd on d.date >= ikdd.created_date_utc
         left join
             booked_days_per_deal b
-            on h.id_deal = b.id_deal
+            on ikdd.id_deal = b.id_deal
             and d.date >= b.created_date_utc
-        where h.id_deal is not null
+        group by d.date, ikdd.id_deal
-        group by d.date, h.id_deal
     ),
     deal_historic_features as (
         select
@@ -75,6 +75,7 @@ with
             hhbf.last_time_booked_date_utc,
             hhbf.second_to_last_time_booked_date_utc,
             hdo.cancellation_date_utc,
+            case when hhbf.client_type = 'API' then true else false end as is_api_deal,
             case
                 when hhbf.date >= hdo.cancellation_date_utc then true else false
             end as deal_has_been_offboarded,
@@ -158,18 +159,28 @@ select
         -- Additionally, the deal has not been offboarded in hubspot.
         when deal_was_created_this_month and not deal_has_been_offboarded
         then '01-New'
-        -- 02-Never Booked: The deal has been created before this month and has not
+        -- 02-Never Booked: The deal is not API, has been created before this month
-        -- had any booking. Additionally, the deal has not been offboarded in hubspot.
+        -- and has not had any booking. Additionally, the deal has not been offboarded
+        -- in hubspot.
         when
             not deal_has_at_least_one_booking
             and not deal_was_created_this_month
             and not deal_has_been_offboarded
+            and not is_api_deal
         then '02-Never Booked'
-        -- 04-Active: The deal has had at least 1 booking in its history and it's
+        -- 04-Active: 
+        -- The deal is API, is not New and has not been offboarded
+        -- The deal is not API and has had at least 1 booking in its history and it's
         -- been less than 12 months since the last booking and has not been offboarded
-        -- in hubspot and is not reactivated and is not FTB
         when
-            deal_has_at_least_one_booking
+            -- API deals --
+            is_api_deal
+            and not deal_was_created_this_month
+            and not deal_has_been_offboarded
+            -- Platform deals --
+            or (
+                not is_api_deal
+                and deal_has_at_least_one_booking
                 and not deal_was_created_this_month
                 and has_been_booked_within_last_12_months
                 and not deal_has_been_offboarded
@@ -178,20 +189,26 @@ select
                     had_previous_booking_more_than_12_months_before_the_last
                     and has_been_booked_within_current_month
                 )
+            )
         then '04-Active'
-        -- 05-Churning: The deal has been offboarded this month. Alternatively, The
+        -- 05-Churning: The deal has been offboarded this month.
-        -- deal has been booked at least once and it's been 12 months since the last
+        -- Alternatively, The
+        -- deal has been booked at least once and it's been 12 months since
+        -- the last
         -- booking
         when
             (
                 deal_has_at_least_one_booking
                 and last_booking_was_12_months_ago
                 and not deal_has_been_offboarded
+                and not is_api_deal
             )
             or deal_was_offboarded_this_month
         then '05-Churning'
-        -- 06-Inactive: The deal has been offboarded in the past but not this month.
+        -- 06-Inactive: The deal has been offboarded in the past but not this
-        -- Alternatively, the deal is not offboarded and the deal has been booked at
+        -- month.
+        -- Alternatively, the deal is not offboarded and the deal has been
+        -- booked at
         -- least once and it's been more than 12 months since the last booking. 
         when
             (
@@ -199,15 +216,18 @@ select
                 and not has_been_booked_within_last_12_months
                 and not last_booking_was_12_months_ago
                 and not deal_has_been_offboarded
+                and not is_api_deal
             )
             or (deal_has_been_offboarded and not deal_was_offboarded_this_month)
         then '06-Inactive'
-        -- 07-Reactivated: The deal is not offboarded but was churned/inactive, and
+        -- 07-Reactivated: The deal is not offboarded but was
+        -- churned/inactive, and
         -- now has had a new booking this month
         when
             had_previous_booking_more_than_12_months_before_the_last
             and has_been_booked_within_current_month
             and not deal_has_been_offboarded
+            and not is_api_deal
         then '07-Reactivated'
         else null
     end as deal_lifecycle_state,

models/intermediate/kpis/int_kpis__metric_daily_deals.sql

@@ -5,18 +5,33 @@
             "date",
             "main_billing_country_iso_3_per_deal",
             "active_accommodations_per_deal_segmentation",
+            "business_scope",
         ],
     )
 }}
 select
     -- Unique Key --
     ldl.date,
+    case
+        when ikdd.client_type = 'API'
+        then 'API'
+        when ikdd.client_type = 'PLATFORM'
+        then
+            case
+                when
+                    icnddsd.id_deal is not null
+                    and ldl.date >= icnddsd.min_user_in_new_dash_since_date_utc
+                then 'New Dash'
+                else 'Old Dash'
+            end
+        else 'UNSET'
+    end as business_scope,
     coalesce(
-        icd.main_billing_country_iso_3_per_deal, 'UNSET'
+        ikdd.main_billing_country_iso_3_per_deal, 'UNSET'
     ) as main_billing_country_iso_3_per_deal,
     case
         when ldl.deal_lifecycle_state = '01-New'
-        then coalesce(dd.hubspot_listing_segmentation, 'UNSET')
+        then coalesce(ikdd.hubspot_listing_segmentation, 'UNSET')
         else coalesce(icmas.active_accommodations_per_deal_segmentation, 'UNSET')
     end as active_accommodations_per_deal_segmentation,
     -- Metrics --
@@ -58,12 +73,24 @@ select
     ) as deals_booked_in_6_months,
     sum(
         case when has_been_booked_within_last_12_months then 1 else 0 end
-    ) as deals_booked_in_12_months
+    ) as deals_booked_in_12_months,
+    count(
+        distinct
+        case
+            when
+                ldl.deal_lifecycle_state
+                in ('01-New', '02-Never Booked', '04-Active', '07-Reactivated')
+            then ldl.id_deal
+            else null
+        end
+    ) as live_deals
 from {{ ref("int_kpis__lifecycle_daily_deal") }} as ldl
-left join {{ ref("int_core__deal") }} as icd on ldl.id_deal = icd.id_deal
+left join {{ ref("int_kpis__dimension_deals") }} as ikdd on ldl.id_deal = ikdd.id_deal
-left join {{ ref("int_kpis__dimension_deals") }} as dd on ldl.id_deal = dd.id_deal
 left join
     {{ ref("int_kpis__dimension_daily_accommodation") }} as icmas
     on ldl.id_deal = icmas.id_deal
     and ldl.date = icmas.date
-group by 1, 2, 3
+left join
+    {{ ref("int_core__new_dash_deal_since_date") }} as icnddsd
+    on ldl.id_deal = icnddsd.id_deal
+group by 1, 2, 3, 4

models/intermediate/kpis/int_kpis__metric_daily_listings.sql

@@ -1,16 +1,35 @@
 {{
     config(
         materialized="table",
-        unique_key=["date", "id_deal"],
+        unique_key=["date", "id_deal", "business_scope"],
     )
 }}
 select
     -- Unique Key --
     lda.date,
     coalesce(icuh.id_deal, 'UNSET') as id_deal,
+    case
+        -- New Dash users with a deal after move date
+        when
+            icuh.is_user_in_new_dash = true
+            and icuh.is_missing_id_deal = false
+            and icuh.user_in_new_dash_since_date_utc >= lda.date
+        then 'New Dash'
+        -- Old Dash users if user is not in the new dash or before move date
+        when
+            icuh.is_user_in_new_dash = false
+            or (
+                icuh.is_user_in_new_dash = true
+                and icuh.is_missing_id_deal = false
+                and not icuh.user_in_new_dash_since_date_utc >= lda.date
+            )
+        then 'Old Dash'
+        -- Rest of the cases, includes KYG Lite users
+        else 'UNSET'
+    end as business_scope,
     -- Dimensions --
     coalesce(
-        icd.main_billing_country_iso_3_per_deal, 'UNSET'
+        icuh.main_billing_country_iso_3_per_deal, 'UNSET'
     ) as main_billing_country_iso_3_per_deal,
     coalesce(
         icmas.active_accommodations_per_deal_segmentation, 'UNSET'
@@ -80,9 +99,8 @@ left join
     on icatu.id_accommodation = lda.id_accommodation
 left join
     {{ ref("int_core__user_host") }} as icuh on icatu.id_user_owner = icuh.id_user_host
-left join {{ ref("int_core__deal") }} as icd on icuh.id_deal = icd.id_deal
 left join
     {{ ref("int_kpis__dimension_daily_accommodation") }} as icmas
-    on icd.id_deal = icmas.id_deal
+    on icuh.id_deal = icmas.id_deal
     and lda.date = icmas.date
-group by 1, 2, 3, 4
+group by 1, 2, 3, 4, 5

models/intermediate/kpis/schema.yml

@@ -5075,6 +5075,7 @@ models:
       in this case: 
       - date, 
       - main_billing_country_iso_3_per_deal,
+      - business_scope,
       - active_accommodations_per_deal_segmentation
 
     data_tests:
@@ -5082,6 +5083,7 @@ models:
           combination_of_columns:
             - date
             - main_billing_country_iso_3_per_deal
+            - business_scope
             - active_accommodations_per_deal_segmentation
 
     columns:
@@ -5091,6 +5093,19 @@ models:
         data_tests:
           - not_null
 
+      - name: business_scope
+        data_type: string
+        description: |
+          Business scope identifying the metric source.
+        data_tests:
+          - not_null
+          - accepted_values:
+              values:
+                - "Old Dash"
+                - "New Dash"
+                - "API"
+                - "UNSET"
+
       - name: active_accommodations_per_deal_segmentation
         data_type: string
         description: |
@@ -5159,6 +5174,12 @@ models:
         description: |
           Count of deals booked within the past 12 months in a given date and per specified dimension.
 
+      - name: live_deals
+        data_type: bigint
+        description: |
+          Count of live deals in a given date and per specified dimension. 
+          This accounts for New Deals, Active Deals and Reactivated Deals.
+
   - name: int_kpis__agg_daily_deals
     description: |
       This model computes the dimension aggregation for 
@@ -5205,12 +5226,14 @@ models:
                 - deals_booked_in_month
                 - deals_booked_in_6_months
                 - deals_booked_in_12_months
+                - live_deals
 
           - accepted_values:
               values:
                 - global
                 - by_number_of_listings
                 - by_billing_country
+                - by_business_scope
 
       - name: dimension_value
         data_type: string
@@ -5287,6 +5310,12 @@ models:
         description: |
           Count of deals booked within the past 12 months for a given date, dimension and value.
 
+      - name: live_deals
+        data_type: bigint
+        description: |
+          Count of live deals in a given date and per specified dimension. 
+          This accounts for New Deals, Never Booked Deals, Active Deals and Reactivated Deals.
+
   - name: int_kpis__metric_daily_listings
     description: |
       This model computes the Daily Listing metrics at the deepest granularity.
@@ -5299,6 +5328,7 @@ models:
       The unique key corresponds to the deepest granularity of the model,
       in this case: 
       - date, 
+      - business_scope
       - id_deal
 
     data_tests:
@@ -5306,6 +5336,7 @@ models:
           combination_of_columns:
             - date
             - id_deal
+            - business_scope
 
     columns:
       - name: date
@@ -5320,6 +5351,19 @@ models:
         data_tests:
           - not_null
 
+      - name: business_scope
+        data_type: string
+        description: |
+          Business scope identifying the metric source.
+        data_tests:
+          - not_null
+          - accepted_values:
+              values:
+                - "Old Dash"
+                - "New Dash"
+                - "API"
+                - "UNSET"
+
       - name: active_accommodations_per_deal_segmentation
         data_type: string
         description: |
@@ -5443,6 +5487,7 @@ models:
                 - by_number_of_listings
                 - by_billing_country
                 - by_deal
+                - by_business_scope
 
       - name: dimension_value
         data_type: string