diff --git a/models/intermediate/core/int_core__mtd_guest_journey_metrics.sql b/models/intermediate/core/int_core__mtd_guest_journey_metrics.sql index 5bfb0a5..ea98536 100644 --- a/models/intermediate/core/int_core__mtd_guest_journey_metrics.sql +++ b/models/intermediate/core/int_core__mtd_guest_journey_metrics.sql @@ -2,7 +2,6 @@ This model provides Month-To-Date (MTD) based on Guest Journey metrics. */ - {% set dimensions = get_kpi_dimensions() %} {{ config(materialized="table", unique_key=["date", "dimension", "dimension_value"]) }} @@ -19,7 +18,9 @@ with select * from {{ ref("int_core__mtd_accommodation_segmentation") }} ), int_dates_mtd as (select * from {{ ref("int_dates_mtd") }}), - int_dates_mtd_by_dimension as (select * from {{ ref("int_dates_mtd_by_dimension") }}), + int_dates_mtd_by_dimension as ( + select * from {{ ref("int_dates_mtd_by_dimension") }} + ), first_payment_per_verification_request as ( select @@ -46,11 +47,13 @@ with and extract(day from vr.created_date_utc) <= d.day {% if dimension.dimension == "'by_number_of_listings'" %} inner join int_core__user_host u on vr.id_user_host = u.id_user_host - inner join int_core__mtd_accommodation_segmentation mas + inner join + int_core__mtd_accommodation_segmentation mas on u.id_deal = mas.id_deal and d.date = mas.date {% elif dimension.dimension == "'by_billing_country'" %} - inner join int_core__user_host u + inner join + int_core__user_host u on vr.id_user_host = u.id_user_host and u.main_billing_country_iso_3_per_deal is not null {% endif %} @@ -71,15 +74,19 @@ with from int_dates_mtd d inner join int_core__verification_requests vr - on date_trunc('month', vr.verification_estimated_started_date_utc)::date = d.first_day_month - and extract(day from vr.verification_estimated_started_date_utc) <= d.day + on date_trunc('month', vr.verification_estimated_started_date_utc)::date + = d.first_day_month + and extract(day from vr.verification_estimated_started_date_utc) + <= d.day {% if dimension.dimension == "'by_number_of_listings'" %} inner join int_core__user_host u on vr.id_user_host = u.id_user_host - inner join int_core__mtd_accommodation_segmentation mas + inner join + int_core__mtd_accommodation_segmentation mas on u.id_deal = mas.id_deal and d.date = mas.date {% elif dimension.dimension == "'by_billing_country'" %} - inner join int_core__user_host u + inner join + int_core__user_host u on vr.id_user_host = u.id_user_host and u.main_billing_country_iso_3_per_deal is not null {% endif %} @@ -100,15 +107,21 @@ with from int_dates_mtd d inner join int_core__verification_requests vr - on date_trunc('month', vr.verification_estimated_completed_date_utc)::date = d.first_day_month - and extract(day from vr.verification_estimated_completed_date_utc) <= d.day + on date_trunc( + 'month', vr.verification_estimated_completed_date_utc + )::date + = d.first_day_month + and extract(day from vr.verification_estimated_completed_date_utc) + <= d.day {% if dimension.dimension == "'by_number_of_listings'" %} inner join int_core__user_host u on vr.id_user_host = u.id_user_host - inner join int_core__mtd_accommodation_segmentation mas + inner join + int_core__mtd_accommodation_segmentation mas on u.id_deal = mas.id_deal and d.date = mas.date {% elif dimension.dimension == "'by_billing_country'" %} - inner join int_core__user_host u + inner join + int_core__user_host u on vr.id_user_host = u.id_user_host and u.main_billing_country_iso_3_per_deal is not null {% endif %} @@ -129,17 +142,24 @@ with from int_dates_mtd d inner join first_payment_per_verification_request p - on date_trunc('month', p.first_payment_paid_date_utc)::date = d.first_day_month + on date_trunc('month', p.first_payment_paid_date_utc)::date + = d.first_day_month and extract(day from p.first_payment_paid_date_utc) <= d.day {% if dimension.dimension == "'by_number_of_listings'" %} - inner join int_core__verification_requests vr on vr.id_verification_request = p.id_verification_request + inner join + int_core__verification_requests vr + on vr.id_verification_request = p.id_verification_request inner join int_core__user_host u on vr.id_user_host = u.id_user_host - inner join int_core__mtd_accommodation_segmentation mas + inner join + int_core__mtd_accommodation_segmentation mas on u.id_deal = mas.id_deal and d.date = mas.date {% elif dimension.dimension == "'by_billing_country'" %} - inner join int_core__verification_requests vr on vr.id_verification_request = p.id_verification_request - inner join int_core__user_host u + inner join + int_core__verification_requests vr + on vr.id_verification_request = p.id_verification_request + inner join + int_core__user_host u on vr.id_user_host = u.id_user_host and u.main_billing_country_iso_3_per_deal is not null {% endif %} @@ -173,22 +193,23 @@ select cast(pym.paid_guest_journeys as decimal) / coym.completed_guest_journeys as payment_rate_guest_journey from int_dates_mtd_by_dimension d -left join - created_year_month cym +left join + created_year_month cym on cym.date = d.date - and cym.dimension = d.dimension + and cym.dimension = d.dimension and cym.dimension_value = d.dimension_value -left join - started_year_month sym +left join + started_year_month sym on d.date = sym.date - and d.dimension = sym.dimension + and d.dimension = sym.dimension and d.dimension_value = sym.dimension_value -left join +left join completed_year_month coym on d.date = coym.date - and d.dimension = coym.dimension + and d.dimension = coym.dimension and d.dimension_value = coym.dimension_value -left join paid_year_month pym +left join + paid_year_month pym on d.date = pym.date - and d.dimension = pym.dimension + and d.dimension = pym.dimension and d.dimension_value = pym.dimension_value diff --git a/models/intermediate/core/int_core__mtd_guest_payments_metrics.sql b/models/intermediate/core/int_core__mtd_guest_payments_metrics.sql index 5bece0d..e93a9e0 100644 --- a/models/intermediate/core/int_core__mtd_guest_payments_metrics.sql +++ b/models/intermediate/core/int_core__mtd_guest_payments_metrics.sql @@ -62,7 +62,8 @@ with from int_dates_mtd d inner join int_core__verification_payments vp - on date_trunc('month', vp.payment_paid_date_utc)::date = d.first_day_month + on date_trunc('month', vp.payment_paid_date_utc)::date + = d.first_day_month and extract(day from vp.payment_paid_date_utc) <= d.day {% if dimension.dimension == "'by_number_of_listings'" %} inner join int_core__user_host u on vp.id_user_host = u.id_user_host @@ -71,7 +72,9 @@ with on u.id_deal = mas.id_deal and d.date = mas.date {% elif dimension.dimension == "'by_billing_country'" %} - inner join int_core__user_host u on vp.id_user_host = u.id_user_host + inner join + int_core__user_host u + on vp.id_user_host = u.id_user_host and u.main_billing_country_iso_3_per_deal is not null {% endif %} where upper(vp.payment_status) = {{ var("paid_payment_state") }} diff --git a/models/intermediate/core/int_core__verification_payments.sql b/models/intermediate/core/int_core__verification_payments.sql index ac4bb23..617cec6 100644 --- a/models/intermediate/core/int_core__verification_payments.sql +++ b/models/intermediate/core/int_core__verification_payments.sql @@ -7,7 +7,9 @@ with select * from {{ ref("stg_core__verification_payment_type") }} ), stg_core__verification as (select * from {{ ref("stg_core__verification") }}), - stg_core__verification_request as (select * from {{ ref("stg_core__verification_request") }}), + stg_core__verification_request as ( + select * from {{ ref("stg_core__verification_request") }} + ), stg_core__payment as (select * from {{ ref("stg_core__payment") }}), stg_core__payment_status as (select * from {{ ref("stg_core__payment_status") }}), int_simple_exchange_rates as (select * from {{ ref("int_simple_exchange_rates") }}) @@ -52,6 +54,6 @@ left join on vtp.payment_due_date_utc = r.rate_date_utc and p.currency = r.from_currency and r.to_currency = 'GBP' -left join - stg_core__verification_request vr - on v.id_verification_request = vr.id_verification_request \ No newline at end of file +left join + stg_core__verification_request vr + on v.id_verification_request = vr.id_verification_request diff --git a/models/intermediate/core/int_core__verification_payments_v2.sql b/models/intermediate/core/int_core__verification_payments_v2.sql new file mode 100644 index 0000000..67406cf --- /dev/null +++ b/models/intermediate/core/int_core__verification_payments_v2.sql @@ -0,0 +1,158 @@ +{{ config(materialized="table") }} + +{% set vat_applicable_services = "('Waiver', 'Fee', 'CheckInCover')" %} + +with + stg_core__verification_to_payment as ( + select * from {{ ref("stg_core__verification_to_payment") }} + ), + stg_core__verification_payment_type as ( + select * from {{ ref("stg_core__verification_payment_type") }} + ), + stg_core__verification as (select * from {{ ref("stg_core__verification") }}), + stg_core__verification_request as ( + select * from {{ ref("stg_core__verification_request") }} + ), + stg_core__payment as (select * from {{ ref("stg_core__payment") }}), + stg_core__payment_status as (select * from {{ ref("stg_core__payment_status") }}), + int_simple_exchange_rates as (select * from {{ ref("int_simple_exchange_rates") }}), + int_core__unified_user as (select * from {{ ref("int_core__unified_user") }}), + stg_seed__guest_services_vat_rates_by_country as ( + select * from {{ ref("stg_seed__guest_services_vat_rates_by_country") }} + ), + vat_details as ( + select + vtp.id_verification_to_payment, + coalesce(vat.vat_rate, 0) as vat_rate, + case + when vpt.verification_payment_type in {{ vat_applicable_services }} + then true + else false + end as is_service_subject_to_vat, + case + when vpt.verification_payment_type not in {{ vat_applicable_services }} + then false + when vat.vat_rate = 0 + then false + when uu.billing_country_iso_3 is null + then false + when vat.vat_rate < 1 and vat.vat_rate > 0 + then true + else false + end as is_vat_taxed, + (uu.billing_country_iso_3 is null) as is_missing_user_country, + ( + uu.billing_country_iso_3 is not null and vat.alpha_3 is null + ) as is_missing_vat_rate_for_country, + (uu.is_deleted = true) as are_user_details_deleted, + -- This final case isolates null VAT rates that are not caused + -- by the previous columns. The idea is: if any of the previous + -- have happened, that's ok because there are known exceptions. + -- But if the VAT rate is missing and it's not for any of those + -- reasons, we have some unhandled issue. + case + when uu.billing_country_iso_3 is null + then false + when uu.is_deleted = true + then false + when uu.billing_country_iso_3 is not null and vat.alpha_3 is null + then false + when vat.vat_rate is null + then true + else false + end as is_missing_vat_details_without_known_cause + from stg_core__verification_to_payment vtp + left join + stg_core__verification_payment_type vpt + on vtp.id_verification_payment_type = vpt.id_verification_payment_type + left join int_core__unified_user uu on vtp.id_guest_user = uu.id_user + left join + stg_seed__guest_services_vat_rates_by_country vat + on uu.billing_country_iso_3 = vat.alpha_3 + + ) +select + vtp.id_verification_to_payment, + vtp.id_payment, + vtp.is_refundable, + vtp.created_at_utc, + vtp.updated_at_utc, + vtp.payment_due_at_utc, + vtp.payment_due_date_utc, + p.paid_at_utc as payment_paid_at_utc, + p.paid_date_utc as payment_paid_date_utc, + p.payment_reference, + vtp.refund_due_at_utc, + vtp.refund_due_date_utc, + p.refunded_at_utc as payment_refunded_at_utc, + p.refunded_date_utc as payment_refunded_date_utc, + p.refund_payment_reference, + -- Host User identifier is included to speed up + -- KPIs execution, even though the host itself + -- has nothing to do with the guest payments. + -- --------------------------------------------- + -- Pablo here, I promise I'll find a way to improve performance and get rid + -- of this uglyness. Oh god, it hurts. + vr.id_user_host, + vtp.id_guest_user, + vtp.id_verification, + v.id_verification_request, + vpt.verification_payment_type, + p.currency, + p.amount as total_amount_in_txn_currency, + (p.amount * r.rate)::decimal(19, 4) as total_amount_in_gbp, + /* + Helping comment for logic below. + Given that guest payments are tax inclusive, the tax (column + tax_amount_in_txn_currency) is calculated as: + paid by guest + tax = paid by guest - ( ------------- ) + 1 + VAT Rate + + The amount without tax (column amount_without_taxes_in_txn_currency) gets + calculated as: + paid by guest + amount without tax = ( ------------- ) + 1 + VAT Rate + */ + ( + (p.amount - (p.amount / (1 + vat.vat_rate))) + * vat.is_service_subject_to_vat::int -- Multiplying by this makes amount 0 if not taxable + )::decimal(19, 4) as tax_amount_in_txn_currency, + ( + (p.amount - (p.amount / (1 + vat.vat_rate))) + * vat.is_service_subject_to_vat::int + * r.rate + )::decimal(19, 4) as tax_amount_in_gbp, + (p.amount / (1 + vat.vat_rate))::decimal( + 19, 4 + ) as amount_without_taxes_in_txn_currency, + ((p.amount / (1 + vat.vat_rate)) * r.rate)::decimal( + 19, 4 + ) as amount_without_taxes_in_gbp, + vat.vat_rate, + vat.is_service_subject_to_vat, + vat.is_vat_taxed, + vat.is_missing_user_country, + vat.are_user_details_deleted, + vat.is_missing_vat_rate_for_country, + vat.is_missing_vat_details_without_known_cause, + ps.payment_status, + p.notes +from stg_core__verification_to_payment vtp +left join stg_core__payment p on vtp.id_payment = p.id_payment +left join stg_core__verification v on vtp.id_verification = v.id_verification +left join + stg_core__verification_payment_type vpt + on vtp.id_verification_payment_type = vpt.id_verification_payment_type +left join stg_core__payment_status ps on p.id_payment_status = ps.id_payment_status +left join + int_simple_exchange_rates r + on vtp.payment_due_date_utc = r.rate_date_utc + and p.currency = r.from_currency + and r.to_currency = 'GBP' +left join + stg_core__verification_request vr + on v.id_verification_request = vr.id_verification_request +left join + vat_details vat on vat.id_verification_to_payment = vtp.id_verification_to_payment diff --git a/models/intermediate/core/schema.yml b/models/intermediate/core/schema.yml index a877015..9a47223 100644 --- a/models/intermediate/core/schema.yml +++ b/models/intermediate/core/schema.yml @@ -819,12 +819,16 @@ models: date before a starting date. - name: int_core__verification_payments + latest_version: 1 description: >- A simplified table that holds guest journey payments with details around when they happen, what service was being paid, what was the related verification request, etc. Currency rates are converted to GBP with our simple exchange rates view. + + Guest taxes get calculated here. You can find out more about Guest Tax + calculation here: https://www.notion.so/knowyourguest-superhog/Guest-Services-Taxes-How-to-calculate-a5ab4c049d61427fafab669dbbffb3a2?pvs=4 columns: - name: id_verification_to_payment data_type: bigint @@ -884,14 +888,14 @@ models: data_type: bigint - name: verification_payment_type data_type: character varying - - name: amount_in_txn_currency - data_type: numeric - tests: - - not_null - name: currency data_type: character varying tests: - not_null + - name: amount_in_txn_currency + data_type: numeric + tests: + - not_null - name: amount_in_gbp data_type: numeric tests: @@ -900,7 +904,165 @@ models: data_type: character varying - name: notes data_type: character varying + versions: + - v: 1 + deprecation_date: 2024-10-15 00:00:00.00+00:00 + - v: 2 + columns: + - name: total_amount_in_txn_currency + data_type: numeric + description: | + The total amount due created by the interaction, in the currency + of the transaction. + + Should we refund the payment, this is also the amount we will give + back to the guest. + tests: + - not_null + - name: total_amount_in_gbp + data_type: numeric + description: | + The total amount due created by the interaction, in GBP. + + Should we refund the payment, this is the GBP equivalent of the + amount we will give back to the guest, but we won't be paying in + GBP unless the original payment was in GBP. + tests: + - not_null + - name: tax_amount_in_txn_currency + data_type: numeric + description: | + The tax amount applicable to this transaction, in the currency of + the transaction. + + If the transaction accrues no taxes, will be 0. + tests: + - not_null + - name: tax_amount_in_gbp + data_type: numeric + description: | + The tax amount applicable to this transaction, in GBP. + + If the transaction accrues no taxes, will be 0. + tests: + - not_null + - name: amount_without_taxes_in_txn_currency + data_type: numeric + description: | + The total amount minus taxes, in the currency of the transaction. + + This is what should be considered net-of-taxes revenue for + Superhog. + + If the transaction accrues no taxes, will be equal to the field + total_amount_in_txn_currency. + tests: + - not_null + - name: amount_without_taxes_in_gbp + data_type: numeric + description: | + The total amount minus taxes, in GBP. + + This is what should be considered net-of-taxes revenue for + Superhog. + + If the transaction accrues no taxes, will be equal to the field + total_amount_in_txn_currency. + tests: + - not_null + - name: vat_rate + data_type: numeric + description: | + The applicable VAT rate to this payment. This is inferred from (1) + which service is the payment related to and (2) what's the billing + country of the guest. + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_between: + min_value: 0 + max_value: 0.99 # If we ever have a 100% tax rate... Let's riot working please + strictly: false + + - name: is_service_subject_to_vat + data_type: boolean + description: | + Whether the related payment is subject to VAT. For instance, + deposit payments are not. + tests: + - not_null + + - name: is_vat_taxed + data_type: boolean + description: | + Syntactic sugar to indicate if there's any VAT on this payment. + Will be true if so, false if not for any reason (guest country has + no VAT, the payment is for a deposit, etc.) + tests: + - not_null + + - name: is_missing_user_country + data_type: boolean + description: | + True if, for some reason, the user doesn't have an informed + country. + + The only known, justified reason for this is that the user was + deleted, along with the billing details. + + If this turns true in any other case, you should really find out + why the guest doesn't have a billing country. + + # should be uncommented once this ticket gets solved: + #tests: + # - not_null + # - accepted_values: + # values: + # - false + # where: are_user_details_deleted != true or are_user_details_deleted is not null + + - name: is_missing_vat_rate_for_country + data_type: boolean + description: | + True if the user country is informed, but no VAT rates were found + for it. + + This has to be a joining issue, since our database for VAT rates + covers all the countries in the world. We simply assign a 0% rate + to countries where we don't collect taxes. + + If this turns true in any other case, you should really find out + what's happening. + + # should be uncommented once this ticket gets solved: + #tests: + # - not_null + # - accepted_values: + # values: + # - false + + - name: are_user_details_deleted + data_type: boolean + description: | + True if the user has been deleted, which is a possible explanation + for why there might be no country informed. + + - name: is_missing_vat_details_without_known_cause + data_type: boolean + description: | + True if the VAT rate is missing as a fallback for any + other reason beyond the other one specified in the table. + + If this turns true, you have an unhandled problem and you should + fix it. + + tests: + - not_null + - accepted_values: + values: + - false + - include: all + exclude: [amount_in_txn_currency, amount_in_gbp] - name: int_core__country description: | This model contains information regarding countries, such as codes,