counterweight/data-dwh-dbt-project
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merged PR 3455: Cleans old Xero KPIs models

Browse source 
# Description

Cleans Xero models for the old KPIs flow, temporary tests and schema entries

# Checklist

- [X] The edited models and dependants run properly with production data.
- [NA] The edited models are sufficiently documented.
- [X] The edited models contain PK tests, and I've ran and passed them.
- [NA] I have checked for DRY opportunities with other models and docs.
- [NA] I've picked the right materialization for the affected models.

# Other

- [ ] Check if a full-refresh is required after this PR is merged.

Related work items: #23760
This commit is contained in:
Oriol Roqué Paniagua 2024-11-07 11:48:15 +00:00
parent 8c23f91242
commit 38c7cc10df
5 changed files with 0 additions and 684 deletions
Download patch file Download diff file Expand all files Collapse all files

159
models/intermediate/xero/int_xero__monthly_invoicing_history_by_deal.sql
View file

@ -1,159 +0,0 @@
/*
This model provides monthly metrics for those hosts that have a deal assigned,
on anything related to invoicing metrics. This includes Operator (Host) fees,
waiver payments, APIs and Host Resolutions.
*/
{% set relevant_document_statuses = "('PAID', 'AUTHORISED')" %}
{% set booking_fee_items = "('EU BOOKING FEE','ZAR BOOKINGS','BOOKING FEE - NON-UK','USD BOOKINGS','CAD BOOKINGS','BOOKING FEE - UK','AUD BOOKINGS')" %}
{% set listing_fee_items = "('USD LISTINGS','LISTING FEE - NON UK','ZAR LISTINGS','CAD LISTINGS','LISTING FEE - UK','AUD LISTINGS','EU LISTINGS')" %}
{% set waiver_items = "('DAMAGE WAVER', 'DAMAGE WAIVER')" %}
{% set verification_fee_items = "('VERIFICATION FEE')" %}
{% set resolutions_host_payment_account_name = "('RESOLUTIONS - HOST PAYMENT')" %}
{% set e_deposit_account_name = "('E-DEPOSIT FEES')" %}
{% set guesty_account_name = "('GUESTY FEES', 'GUESTY ADMINISTRATION FEE')" %}
{{ config(materialized="table", unique_key="date") }}
with
int_xero__sales_denom_mart as (
select * from {{ ref("int_xero__sales_denom_mart") }}
),
int_dates_by_deal as (select * from {{ ref("int_dates_by_deal") }}),
int_xero__bank_transaction_line_items as (
select * from {{ ref("int_xero__bank_transaction_line_items") }}
),
int_xero__bank_transactions as (
select * from {{ ref("int_xero__bank_transactions") }}
),
int_xero__contacts as (select * from {{ ref("int_xero__contacts") }}),
resolution_host_payment as (
select
date_trunc('month', bt.transaction_date_utc)::date as first_day_month,
c.id_deal,
sum(
btli.line_amount_wo_taxes_in_gbp
) as xero_host_resolution_amount_paid_in_gbp,
count(distinct bt.id_bank_transaction) as xero_host_resolution_payment_count
from int_xero__bank_transactions bt
inner join
int_xero__bank_transaction_line_items btli
on bt.id_bank_transaction = btli.id_bank_transaction
inner join int_xero__contacts c on c.id_contact = bt.id_contact
where upper(btli.account_name) in {{ resolutions_host_payment_account_name }}
group by 1, 2
),
apis_net_fees as (
select
date_trunc('month', sdm.document_issued_date_utc)::date as first_day_month,
id_deal,
sum(
case
when upper(sdm.account_name) in {{ e_deposit_account_name }}
then sdm.line_amount_wo_taxes_in_gbp
else 0
end
) as xero_e_deposit_net_fees_in_gbp,
sum(
case
when upper(sdm.account_name) in {{ guesty_account_name }}
then sdm.line_amount_wo_taxes_in_gbp
else 0
end
) as xero_guesty_net_fees_in_gbp
from int_xero__sales_denom_mart sdm
where
upper(sdm.document_status) in {{ relevant_document_statuses }}
and (
upper(sdm.account_name) in {{ e_deposit_account_name }}
or upper(sdm.account_name) in {{ guesty_account_name }}
)
group by 1, 2
),
host_net_fees as (
select
date_trunc('month', sdm.document_issued_date_utc)::date as first_day_month,
id_deal,
sum(
case
when upper(sdm.item_code) in {{ booking_fee_items }}
then sdm.line_amount_wo_taxes_in_gbp
else 0
end
) as xero_booking_net_fees_in_gbp,
sum(
case
when upper(sdm.item_code) in {{ listing_fee_items }}
then sdm.line_amount_wo_taxes_in_gbp
else 0
end
) as xero_listing_net_fees_in_gbp,
sum(
case
when upper(sdm.item_code) in {{ verification_fee_items }}
then sdm.line_amount_wo_taxes_in_gbp
else 0
end
) as xero_verification_net_fees_in_gbp,
sum(
case
when upper(sdm.item_code) in {{ waiver_items }}
then sdm.line_amount_wo_taxes_in_gbp
else 0
end
) as xero_waiver_paid_back_to_host_in_gbp
from int_xero__sales_denom_mart sdm
where
upper(sdm.document_status) in {{ relevant_document_statuses }}
and (
upper(sdm.item_code) in {{ booking_fee_items }}
or upper(sdm.item_code) in {{ listing_fee_items }}
or upper(sdm.item_code) in {{ verification_fee_items }}
or upper(sdm.item_code) in {{ waiver_items }}
)
group by 1, 2
)
-- Final aggregation of subqueries --
select
d.year,
d.month,
d.day,
d.date,
d.id_deal,
-- HOST/OPERATOR --
hnf.xero_booking_net_fees_in_gbp,
hnf.xero_listing_net_fees_in_gbp,
hnf.xero_verification_net_fees_in_gbp,
nullif(
coalesce(hnf.xero_booking_net_fees_in_gbp, 0)
+ coalesce(hnf.xero_listing_net_fees_in_gbp, 0)
+ coalesce(hnf.xero_verification_net_fees_in_gbp, 0),
0
) as xero_operator_net_fees_in_gbp,
-- APIs --
anf.xero_e_deposit_net_fees_in_gbp,
anf.xero_guesty_net_fees_in_gbp,
nullif(
coalesce(anf.xero_e_deposit_net_fees_in_gbp, 0)
+ coalesce(anf.xero_guesty_net_fees_in_gbp, 0),
0
) as xero_apis_net_fees_in_gbp,
-- WAIVERS PAID BACK TO HOST --
hnf.xero_waiver_paid_back_to_host_in_gbp,
-- HOST RESOLUTIONS --
rhp.xero_host_resolution_amount_paid_in_gbp,
rhp.xero_host_resolution_payment_count
from int_dates_by_deal d
left join
resolution_host_payment rhp
on d.first_day_month = rhp.first_day_month
and d.id_deal = rhp.id_deal
left join
apis_net_fees anf
on d.first_day_month = anf.first_day_month
and d.id_deal = anf.id_deal
left join
host_net_fees hnf
on d.first_day_month = hnf.first_day_month
and d.id_deal = hnf.id_deal

239
models/intermediate/xero/int_xero__mtd_invoicing_metrics.sql
View file

@ -1,239 +0,0 @@
/*
This model provides Month-To-Date (MTD) based on anything related to
invoicing metrics. This includes Operator (Host) fees, waiver payments,
APIs and Host Resolutions.
*/
{% set dimensions = get_kpi_dimensions() %}
{% set relevant_document_statuses = "('PAID', 'AUTHORISED')" %}
{% set booking_fee_items = "('EU BOOKING FEE','ZAR BOOKINGS','BOOKING FEE - NON-UK','USD BOOKINGS','CAD BOOKINGS','BOOKING FEE - UK','AUD BOOKINGS')" %}
{% set listing_fee_items = "('USD LISTINGS','LISTING FEE - NON UK','ZAR LISTINGS','CAD LISTINGS','LISTING FEE - UK','AUD LISTINGS','EU LISTINGS')" %}
{% set waiver_items = "('DAMAGE WAVER', 'DAMAGE WAIVER')" %}
{% set verification_fee_items = "('VERIFICATION FEE')" %}
{% set resolutions_host_payment_account_name = "('RESOLUTIONS - HOST PAYMENT')" %}
{% set e_deposit_account_name = "('E-DEPOSIT FEES')" %}
{% set guesty_account_name = "('GUESTY FEES', 'GUESTY ADMINISTRATION FEE')" %}
{{ config(materialized="table", unique_key=["date", "dimension", "dimension_value"]) }}
with
int_xero__sales_denom_mart as (
select * from {{ ref("int_xero__sales_denom_mart") }}
),
int_dates_mtd as (select * from {{ ref("int_dates_mtd") }}),
int_xero__bank_transaction_line_items as (
select * from {{ ref("int_xero__bank_transaction_line_items") }}
),
int_xero__bank_transactions as (
select * from {{ ref("int_xero__bank_transactions") }}
),
int_core__mtd_accommodation_segmentation as (
select * from {{ ref("int_core__mtd_accommodation_segmentation") }}
),
int_dates_mtd_by_dimension as (
select * from {{ ref("int_dates_mtd_by_dimension") }}
),
int_xero__contacts as (select * from {{ ref("int_xero__contacts") }}),
int_core__deal as (select * from {{ ref("int_core__deal") }}),
resolution_host_payment as (
{% for dimension in dimensions %}
select
d.date,
{{ dimension.dimension }} as dimension,
{{ dimension.dimension_value }} as dimension_value,
sum(
btli.line_amount_wo_taxes_in_gbp
) as xero_host_resolution_amount_paid_in_gbp,
count(
distinct bt.id_bank_transaction
) as xero_host_resolution_payment_count
from int_dates_mtd d
inner join
int_xero__bank_transactions bt
on date_trunc('month', bt.transaction_date_utc)::date = d.first_day_month
and extract(day from bt.transaction_date_utc) <= d.day
inner join
int_xero__bank_transaction_line_items btli
on bt.id_bank_transaction = btli.id_bank_transaction
and upper(btli.account_name)
in {{ resolutions_host_payment_account_name }}
{% if dimension.dimension == "'by_number_of_listings'" %}
inner join int_xero__contacts c on c.id_contact = bt.id_contact
inner join
int_core__mtd_accommodation_segmentation mas
on c.id_deal = mas.id_deal
and d.date = mas.date
{% elif dimension.dimension == "'by_billing_country'" %}
inner join int_xero__contacts c on c.id_contact = bt.id_contact
inner join
int_core__deal ud
on c.id_deal = ud.id_deal
and ud.main_billing_country_iso_3_per_deal is not null
{% endif %}
group by 1, 2, 3
{% if not loop.last %}
union all
{% endif %}
{% endfor %}
),
apis_net_fees as (
{% for dimension in dimensions %}
select
d.date,
{{ dimension.dimension }} as dimension,
{{ dimension.dimension_value }} as dimension_value,
sum(
case
when upper(sdm.account_name) in {{ e_deposit_account_name }}
then sdm.line_amount_wo_taxes_in_gbp
else 0
end
) as xero_e_deposit_net_fees_in_gbp,
sum(
case
when upper(sdm.account_name) in {{ guesty_account_name }}
then sdm.line_amount_wo_taxes_in_gbp
else 0
end
) as xero_guesty_net_fees_in_gbp
from int_dates_mtd d
inner join
int_xero__sales_denom_mart sdm
on date_trunc('month', sdm.document_issued_date_utc) = d.first_day_month
and extract(day from sdm.document_issued_date_utc) <= d.day
{% if dimension.dimension == "'by_number_of_listings'" %}
inner join
int_core__mtd_accommodation_segmentation mas
on sdm.id_deal = mas.id_deal
and d.date = mas.date
{% elif dimension.dimension == "'by_billing_country'" %}
inner join
int_core__deal ud
on sdm.id_deal = ud.id_deal
and ud.main_billing_country_iso_3_per_deal is not null
{% endif %}
where
upper(sdm.document_status) in {{ relevant_document_statuses }}
and (
upper(sdm.account_name) in {{ e_deposit_account_name }}
or upper(sdm.account_name) in {{ guesty_account_name }}
)
group by 1, 2, 3
{% if not loop.last %}
union all
{% endif %}
{% endfor %}
),
host_net_fees as (
{% for dimension in dimensions %}
select
d.date,
{{ dimension.dimension }} as dimension,
{{ dimension.dimension_value }} as dimension_value,
sum(
case
when upper(sdm.item_code) in {{ booking_fee_items }}
then sdm.line_amount_wo_taxes_in_gbp
else 0
end
) as xero_booking_net_fees_in_gbp,
sum(
case
when upper(sdm.item_code) in {{ listing_fee_items }}
then sdm.line_amount_wo_taxes_in_gbp
else 0
end
) as xero_listing_net_fees_in_gbp,
sum(
case
when upper(sdm.item_code) in {{ verification_fee_items }}
then sdm.line_amount_wo_taxes_in_gbp
else 0
end
) as xero_verification_net_fees_in_gbp,
sum(
case
when upper(sdm.item_code) in {{ waiver_items }}
then sdm.line_amount_wo_taxes_in_gbp
else 0
end
) as xero_waiver_paid_back_to_host_in_gbp
from int_dates_mtd d
inner join
int_xero__sales_denom_mart sdm
on date_trunc('month', sdm.document_issued_date_utc) = d.first_day_month
and extract(day from sdm.document_issued_date_utc) <= d.day
{% if dimension.dimension == "'by_number_of_listings'" %}
inner join
int_core__mtd_accommodation_segmentation mas
on sdm.id_deal = mas.id_deal
and d.date = mas.date
{% elif dimension.dimension == "'by_billing_country'" %}
inner join
int_core__deal ud
on sdm.id_deal = ud.id_deal
and ud.main_billing_country_iso_3_per_deal is not null
{% endif %}
where
upper(sdm.document_status) in {{ relevant_document_statuses }}
and (
upper(sdm.item_code) in {{ booking_fee_items }}
or upper(sdm.item_code) in {{ listing_fee_items }}
or upper(sdm.item_code) in {{ verification_fee_items }}
or upper(sdm.item_code) in {{ waiver_items }}
)
group by 1, 2, 3
{% if not loop.last %}
union all
{% endif %}
{% endfor %}
)
-- Final aggregation of subqueries --
select
d.year,
d.month,
d.day,
d.date,
d.dimension,
d.dimension_value,
d.is_end_of_month,
d.is_current_month,
-- HOST/OPERATOR --
hnf.xero_booking_net_fees_in_gbp,
hnf.xero_listing_net_fees_in_gbp,
hnf.xero_verification_net_fees_in_gbp,
nullif(
coalesce(hnf.xero_booking_net_fees_in_gbp, 0)
+ coalesce(hnf.xero_listing_net_fees_in_gbp, 0)
+ coalesce(hnf.xero_verification_net_fees_in_gbp, 0),
0
) as xero_operator_net_fees_in_gbp,
-- APIs --
anf.xero_e_deposit_net_fees_in_gbp,
anf.xero_guesty_net_fees_in_gbp,
nullif(
coalesce(anf.xero_e_deposit_net_fees_in_gbp, 0)
+ coalesce(anf.xero_guesty_net_fees_in_gbp, 0),
0
) as xero_apis_net_fees_in_gbp,
-- WAIVERS PAID BACK TO HOST --
hnf.xero_waiver_paid_back_to_host_in_gbp,
-- HOST RESOLUTIONS --
rhp.xero_host_resolution_amount_paid_in_gbp,
rhp.xero_host_resolution_payment_count
from int_dates_mtd_by_dimension d
left join
resolution_host_payment rhp
on rhp.date = d.date
and rhp.dimension = d.dimension
and rhp.dimension_value = d.dimension_value
left join
apis_net_fees anf
on anf.date = d.date
and anf.dimension = d.dimension
and anf.dimension_value = d.dimension_value
left join
host_net_fees hnf
on hnf.date = d.date
and hnf.dimension = d.dimension
and hnf.dimension_value = d.dimension_value

65
models/intermediate/xero/schema.yml
View file

@ -259,68 +259,3 @@ models:
- name: contact_name
data_type: character varying
description: ""
- name: int_xero__monthly_invoicing_history_by_deal
description: |
This model contains the historic information regarding invoicing metrics for each deal id.
It's used for the business KPIs in the view by deal id. Data is aggregated at the last
day of the month, or up to yesterday if it's the current month.
tests:
- dbt_utils.unique_combination_of_columns:
combination_of_columns:
- date
- id_deal
deprecation_date: 2024-11-30
columns:
- name: date
data_type: date
description: The last day of the month or yesterday for historic invoicing metrics.
tests:
- not_null
- name: id_deal
data_type: character varying
description: Id of the deal associated to the host.
tests:
- not_null
- name: int_xero__mtd_invoicing_metrics
description: |
This model contains the historic information regarding the invoiced and credited metrics
in an aggregated manner.
It's used for the business KPIs. Data is aggregated at the last day of the month and in the
days necessary for the Month-to-Date computation of the current month.
deprecation_date: 2024-11-30
tests:
- dbt_utils.unique_combination_of_columns:
combination_of_columns:
- date
- dimension
- dimension_value
columns:
- name: date
data_type: date
description: The date for the month-to-date invoicing-related metrics.
tests:
- not_null
- name: dimension
data_type: string
description: The dimension or granularity of the metrics.
tests:
- accepted_values:
values:
- global
- by_number_of_listings
- by_billing_country
- name: dimension_value
data_type: string
description: The value or segment available for the selected dimension.
tests:
- not_null