counterweight/data-dwh-dbt-project
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merged PR 5173: First version of Verification Product Payments

Browse source 
# Description

New model for Verification Product Payments (Waiver, Deposit, Fee + not assigned set as Unknown).
This includes:
* Similar upper case and coalesce for payment status and verification product name
* Rename of paway_minimum_commission_local_curr as it was not clear (this needs to be handled separately)
* Waiver payaway stuff
* No tax handling

Audit passed with following considerations:

```
-- THIS GOES INTO AN AUDIT FILE
{% set old_query %}
  select
    *
  from dwh_hybrid.intermediate.int_core__verification_product_payments
{% endset %}

{% set new_query %}
  select
    id_verification_to_payment,
    id_payment,
    is_refundable,
    created_at_utc,
    updated_at_utc,
    payment_due_at_utc,
    payment_due_date_utc,
    payment_paid_at_utc,
    payment_paid_date_utc,
    payment_reference,
    refund_due_at_utc,
    refund_due_date_utc,
    payment_refunded_at_utc,
    payment_refunded_date_utc,
    refund_payment_reference,
    id_user_host,
    id_guest_user as id_user_guest,
    id_verification,
    id_verification_request,
    coalesce(
        upper(verification_payment_type), 'UNKNOWN'
    ) as verification_product_name,
    currency,
    total_amount_in_txn_currency,
    total_amount_in_gbp,
    is_host_taking_waiver_risk,
    payaway_percentage,
    payaway_minimum_commission_local_curr as payaway_minimum_commission_in_txn_currency,
    amount_due_to_host_in_txn_currency,
    amount_due_to_host_in_gbp,
    superhog_fee_in_txn_currency,
    superhog_fee_in_gbp,
    coalesce(upper(payment_status),'UNKNOWN') as payment_status,
    notes
  from dwh_hybrid.intermediate.int_core__verification_payments_v2
  where (verification_payment_type != 'CheckInCover' or verification_payment_type is null)
{% endset %}

{{
    audit_helper.compare_and_classify_query_results(
        old_query,
        new_query,
        primary_key_columns=["id_verification_to_payment"],
        columns=[
            "id_verification_to_payment",
            "id_payment",
            "is_refundable",
            "created_at_utc",
            "updated_at_utc",
            "payment_due_at_utc",
            "payment_due_date_utc",
            "payment_paid_at_utc",
            "payment_paid_date_utc",
            "payment_reference",
            "refund_due_at_utc",
            "refund_due_date_utc",
            "payment_refunded_at_utc",
            "payment_refunded_date_utc",
            "refund_payment_reference",
            "id_user_host",
            "id_user_guest",
            "id_verification",
            "id_verification_request",
            "verification_product_name",
            "currency",
            "total_amount_in_txn_currency",
            "total_amount_in_gbp",
            "is_host_taking_waiver_risk",
            "payaway_percentage",
            "payaway_minimum_commission_in_txn_currency",
            "amount_due_to_host_in_txn_currency",
            "amount_due_to_host_in_gbp",
            "superhog_fee_in_txn_currency",
        ...
This commit is contained in:
Oriol Roqué Paniagua 2025-05-09 11:27:00 +00:00
parent ac73138cf1
commit 77c3a02466
4 changed files with 373 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

3
macros/tests/is_null.sql Normal file
View file

@ -0,0 +1,3 @@
{% test is_null(model, column_name) %}
select * from {{ model }} where {{ column_name }} is not null
{% endtest %}

2
models/intermediate/core/int_core__guest_product_payments.sql
View file

@ -77,7 +77,7 @@ with
select * select *
from from
guest_product_payments_from_verification_payments guest_product_payments_from_verification_payments
-- This model is expecting to have provide Guest Product Payment data for -- This model is expecting to provide Guest Product Payment data for
-- both old Check-in Cover and new Guest Product models. At the moment, -- both old Check-in Cover and new Guest Product models. At the moment,
-- the data is only provided for the "old" Check-in Cover. Once the Guest Product -- the data is only provided for the "old" Check-in Cover. Once the Guest Product
-- models are in place, we expect to have here a new CTE that is unioned with the -- models are in place, we expect to have here a new CTE that is unioned with the

82
models/intermediate/core/int_core__verification_product_payments.sql Normal file
View file

@ -0,0 +1,82 @@
{% set check_in_cover_as_id_verification_payment = "5" %}
{{ config(materialized="table", unique_key="id_verification_product_payment") }}
with
stg_core__verification_to_payment as (
select * from {{ ref("stg_core__verification_to_payment") }}
),
stg_core__verification_payment_type as (
select * from {{ ref("stg_core__verification_payment_type") }}
),
stg_core__verification as (select * from {{ ref("stg_core__verification") }}),
stg_core__verification_request as (
select * from {{ ref("stg_core__verification_request") }}
),
stg_core__payment as (select * from {{ ref("stg_core__payment") }}),
stg_core__payment_status as (select * from {{ ref("stg_core__payment_status") }}),
int_simple_exchange_rates as (select * from {{ ref("int_simple_exchange_rates") }}),
int_core__waiver_fees as (select * from {{ ref("int_core__waiver_fees") }})
select
vtp.id_verification_to_payment as id_verification_product_payment,
vtp.id_payment,
vtp.id_verification_to_payment,
vtp.is_refundable,
vtp.created_at_utc,
vtp.updated_at_utc,
vtp.payment_due_at_utc,
vtp.payment_due_date_utc,
p.paid_at_utc as payment_paid_at_utc,
p.paid_date_utc as payment_paid_date_utc,
p.payment_reference,
vtp.refund_due_at_utc,
vtp.refund_due_date_utc,
p.refunded_at_utc as payment_refunded_at_utc,
p.refunded_date_utc as payment_refunded_date_utc,
p.refund_payment_reference,
vr.id_user_host,
vtp.id_guest_user as id_user_guest,
vtp.id_verification,
v.id_verification_request,
-- Some old payments did not record the verification payment type
-- These are tagged as 'UNKNOWN' in the model
coalesce(
upper(vpt.verification_payment_type), 'UNKNOWN'
) as verification_product_name,
p.currency,
p.amount as total_amount_in_txn_currency,
(p.amount * r.rate)::decimal(19, 4) as total_amount_in_gbp,
wf.is_host_taking_waiver_risk,
wf.payaway_percentage,
-- The currency available in Waiver Fees model is in the actual transaction
-- currency, not the host one. Local currency is not clear what it's actually
-- referring to. This should be modified in the upstream model and all
-- dependencies.
wf.payaway_minimum_commission_local_curr
as payaway_minimum_commission_in_txn_currency,
wf.amount_due_to_host_in_txn_currency,
wf.amount_due_to_host_in_gbp,
wf.superhog_fee_in_txn_currency,
wf.superhog_fee_in_gbp,
upper(coalesce(ps.payment_status, 'UNKNOWN')) as payment_status,
p.notes
from stg_core__verification_to_payment vtp
left join
stg_core__verification_payment_type vpt
on vtp.id_verification_payment_type = vpt.id_verification_payment_type
left join stg_core__payment p on vtp.id_payment = p.id_payment
inner join stg_core__payment_status ps on p.id_payment_status = ps.id_payment_status
inner join
int_simple_exchange_rates r
on vtp.payment_due_date_utc = r.rate_date_utc
and p.currency = r.from_currency
and r.to_currency = 'GBP'
left join stg_core__verification v on vtp.id_verification = v.id_verification
left join
stg_core__verification_request vr
on v.id_verification_request = vr.id_verification_request
left join int_core__waiver_fees wf on vtp.id_payment = wf.id_payment
-- Exclude 'Old' Check-in Hero that were considered as Verification Payment
where
vtp.id_verification_payment_type != {{ check_in_cover_as_id_verification_payment }}
or vtp.id_verification_payment_type is null

287
models/intermediate/core/schema.yml
View file

@ -6269,3 +6269,290 @@ models:
description: | description: |
Notes or comments associated with the payment. Notes or comments associated with the payment.
It can be null if there are no notes. It can be null if there are no notes.
- name: int_core__verification_product_payments
description: |
A model that holds verification products payments (Waiver, Deposit)
with details around when they happen, what service was being paid,
what was the related verification request, etc.
Currency rates are converted to GBP with our simple exchange rates view.
Only amounts with taxes are included in this model.
Note that Guest Products are excluded from this model, as they are not
considered verification products.
columns:
- name: id_verification_product_payment
data_type: bigint
description: |
Unique identifier for the verification product payment.
It corresponds to the id_verification_to_payment.
data_tests:
- unique
- not_null
- name: id_payment
data_type: bigint
description: |
Unique ID for the payment itself.
data_tests:
- unique
- not_null
- name: id_verification_to_payment
data_type: bigint
description: |
Identifier for the relation between the payment verification
and the payment.
- name: is_refundable
data_type: boolean
description: |
Indicates whether the payment is refundable or not.
- name: created_at_utc
data_type: timestamp without time zone
description: |
Timestamp of when the payment was created.
It cannot be null.
data_tests:
- not_null
- name: updated_at_utc
data_type: timestamp without time zone
description: |
Timestamp of when the payment was last updated.
It cannot be null.
data_tests:
- not_null
- name: payment_due_at_utc
data_type: timestamp without time zone
description: |
Timestamp of when the payment is due.
It cannot be null.
data_tests:
- not_null
- name: payment_due_date_utc
data_type: date
description: |
Date of when the payment is due.
It cannot be null.
data_tests:
- not_null
- name: payment_paid_at_utc
data_type: timestamp without time zone
description: |
Timestamp of when the payment was paid.
It can be null if the payment has not been paid yet.
- name: payment_paid_date_utc
data_type: date
description: |
Date of when the payment was paid.
It can be null if the payment has not been paid yet.
- name: payment_reference
data_type: character varying
description: |
Reference code associated with the payment.
It can be null if the payment has not been paid yet.
- name: refund_due_at_utc
data_type: timestamp without time zone
description: |
Timestamp of when the refund is due.
It can be null if the payment has not been refunded.
- name: refund_due_date_utc
data_type: date
description: |
Date of when the refund is due.
It can be null if the payment has not been refunded.
- name: payment_refunded_at_utc
data_type: timestamp without time zone
description: |
Timestamp of when the payment was refunded.
It can be null if the payment has not been refunded.
- name: payment_refunded_date_utc
data_type: date
description: |
Date of when the payment was refunded.
It can be null if the payment has not been refunded.
- name: refund_payment_reference
data_type: character varying
description: |
Reference code associated with the refund payment.
It can be null if the payment has not been refunded.
- name: id_user_host
data_type: character varying
description: |
Unique identifier of the Host linked to the Verification
Request that has a payment.
It can be null for same-day payments due to different freshness
of the sources used to build this model.
data_tests:
- not_null:
where: "date(created_at_utc) < current_date"
- name: id_user_guest
data_type: character varying
description: |
Unique identifier of the Guest linked to the Verification
Request that has a payment.
It cannot be null.
data_tests:
- not_null
- name: id_verification
data_type: bigint
description: |
Unique identifier of the Verification that has a payment.
It cannot be null.
data_tests:
- not_null
- name: id_verification_request
data_type: bigint
description: |
Unique identifier of the Verification Request that has a payment.
It can be null for same-day payments due to different freshness
of the sources used to build this model.
data_tests:
- not_null:
where: "date(created_at_utc) < current_date"
- name: verification_product_name
data_type: character varying
description: |
Name of the Verification Product associated with the payment.
This corresponds to the internal name, rather than the display name.
It cannot be null.
data_tests:
- not_null
- accepted_values:
values:
- DEPOSIT
- FEE
- WAIVER
- UNKNOWN
- name: currency
data_type: character varying
description: |
The ISO 4217 currency code (e.g., GBP, USD, EUR) in which the payment
was originally made.
It can be null for same-day payments due to different freshness
of the sources used to build this model.
data_tests:
- not_null:
where: "date(created_at_utc) < current_date"
- name: total_amount_in_txn_currency
data_type: numeric
description: |
The total amount of the payment in the payment currency.
This includes taxes if applicable.
- name: total_amount_in_gbp
data_type: numeric
description: |
The total amount of the payment in GBP.
This includes taxes if applicable.
- name: is_host_taking_waiver_risk
data_type: boolean
description: |
Boolean indicating whether the host is taking the risk.
It is always null if the payment is not a Waiver payment.
data_tests:
- is_null:
where: "verification_product_name <> 'WAIVER'"
- name: payaway_percentage
data_type: numeric
description: |
Percentage of the payment that goes to Truvi.
It is always null if the payment is not a Waiver payment.
data_tests:
- is_null:
where: "verification_product_name <> 'WAIVER'"
- name: payaway_minimum_commission_in_txn_currency
data_type: numeric
description: |
Minimum commission amount in the payment currency.
It is always null if the payment is not a Waiver payment.
data_tests:
- is_null:
where: "verification_product_name <> 'WAIVER'"
- name: amount_due_to_host_in_txn_currency
data_type: numeric
description: |
The amount payable to the host in the payment currency.
This includes taxes if applicable.
It is always null if the payment is not a Waiver payment.
data_tests:
- is_null:
where: "verification_product_name <> 'WAIVER'"
- name: amount_due_to_host_in_gbp
data_type: numeric
description: |
The amount payable to the host in GBP.
This includes taxes if applicable.
It is always null if the payment is not a Waiver payment.
data_tests:
- is_null:
where: "verification_product_name <> 'WAIVER'"
- name: superhog_fee_in_txn_currency
data_type: numeric
description: |
The service fee charged by Truvi in the payment currency.
This includes taxes if applicable.
It is always null if the payment is not a Waiver payment.
data_tests:
- is_null:
where: "verification_product_name <> 'WAIVER'"
- name: superhog_fee_in_gbp
data_type: numeric
description: |
The service fee charged by Truvi in GBP.
This includes taxes if applicable.
It is always null if the payment is not a Waiver payment.
data_tests:
- is_null:
where: "verification_product_name <> 'WAIVER'"
- name: payment_status
data_type: character varying
description: |
Current status of the payment (e.g., PAID).
It cannot be null.
data_tests:
- accepted_values:
values:
- PAID
- PAID MANUALLY
- CANCELLED
- REFUNDED
- REFUND FAILED
- FAILED WITH RETRY
- FAILED
- UNKNOWN
- name: notes
data_type: character varying
description: |
Notes or comments associated with the payment.
It can be null if there are no notes.