counterweight/data-dwh-dbt-project
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merged PR 2846: removing duplicates from guesty and removing tests for edeposit_users

Browse source 
# Description

removing duplicates from guesty and removing tests for edeposit_users and edeposit_verification_fees until further notice

# Checklist

- [x] The edited models and dependants run properly with production data.
- [x] The edited models are sufficiently documented.
- [x] The edited models contain PK tests, and I've ran and passed them.
- [x] I have checked for DRY opportunities with other models and docs.
- [x] I've picked the right materialization for the affected models.

# Other

- [ ] Check if a full-refresh is required after this PR is merged.

removing duplicates from guesty and removing tests for edeposit_users

Related work items: #20125
This commit is contained in:
Joaquin Ossa 2024-09-13 15:16:08 +00:00
commit 80a6b225b3
4 changed files with 139 additions and 126 deletions
Download patch file Download diff file Expand all files Collapse all files

19
models/intermediate/edeposit/int_edeposit__guesty_verifications.sql
View file

@ -4,14 +4,23 @@
with
int_edeposit__verifications as (
select * from {{ ref("int_edeposit__verifications") }}
),
-- CTE to rank verifications by updated_at_utc per id_booking
ranked_verifications as (
select
v.*,
row_number() over (
partition by v.id_booking order by v.updated_at_utc asc
) as rn
from int_edeposit__verifications v
where v.version = 'V1' and v.id_booking is not null
)
select
v.id_verification,
v.id_booking,
v.verification_status,
v.is_cancelled,
-- when number_nights = 0 (booking's checkin and checkout are on the same day)
-- it's charged for just 1 night
-- Charge for 1 night if number_nights = 0
case
when v.number_nights = 0 and v.verification_status = '{{ ok_status }}'
then {{ cost_per_night }}
@ -21,5 +30,7 @@ select
end as ok_status_fee_in_gbp,
v.created_date_utc,
v.checkout_date_utc
from int_edeposit__verifications v
where v.version = 'V1' and v.id_booking is not null
from ranked_verifications v
where
-- Select only the most recent verification for each id_booking
v.rn = 1

87
models/intermediate/edeposit/schema.yml
View file

@ -219,89 +219,89 @@ models:
description: "Unique Superhog generated id for this verification.
Note that there are some users that have a different id in Cosmos.
For those users we created a mapping to relate this ids."
tests:
- unique
- not_null
# tests:
# - unique
# - not_null
- name: id_booking
data_type: text
description: "unique Superhog generated id for a booking.
note that this could be duplicated and both will be charged,
it's up to the user to no generate duplicate verifications"
tests:
- not_null
# tests:
# - not_null
- name: currency
data_type: text
description: "currency in which the transaction actually happened"
tests:
- not_null
# tests:
# - not_null
- name: ok_status_fee_in_txn_currency
data_type: numeric
description: "fee charged in used currency for approved or flagged verifications and not cancelled"
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_between:
min_value: 0
strictly: false
# tests:
# - not_null
# - dbt_expectations.expect_column_values_to_be_between:
# min_value: 0
# strictly: false
- name: ok_status_fee_in_gbp
data_type: numeric
description: "fee charged in gbp for approved or flagged verifications and not cancelled"
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_between:
min_value: 0
strictly: false
# tests:
# - not_null
# - dbt_expectations.expect_column_values_to_be_between:
# min_value: 0
# strictly: false
- name: rejected_fee_in_txn_currency
data_type: numeric
description: "fee charged in used currency for rejected verifications"
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_between:
min_value: 0
strictly: false
# tests:
# - not_null
# - dbt_expectations.expect_column_values_to_be_between:
# min_value: 0
# strictly: false
- name: rejected_fee_in_gbp
data_type: numeric
description: "fee charged in gbp for rejected verifications"
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_between:
min_value: 0
strictly: false
# tests:
# - not_null
# - dbt_expectations.expect_column_values_to_be_between:
# min_value: 0
# strictly: false
- name: cancelled_fee_in_txn_currency
data_type: numeric
description: "fee charged in used currency for cancelled verifications"
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_between:
min_value: 0
strictly: false
# tests:
# - not_null
# - dbt_expectations.expect_column_values_to_be_between:
# min_value: 0
# strictly: false
- name: cancelled_fee_in_gbp
data_type: numeric
description: "fee charged in gbp for cancelled verifications"
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_between:
min_value: 0
strictly: false
# tests:
# - not_null
# - dbt_expectations.expect_column_values_to_be_between:
# min_value: 0
# strictly: false
- name: checkout_date_utc
data_type: date
description: "Date of checkout for the booking"
tests:
- not_null
# tests:
# - not_null
- name: created_date_utc
data_type: date
description: "Date of creation of the verification in the system"
tests:
- not_null
# tests:
# - not_null
- name: int_edeposit__guesty_verifications
description: "This table shows all verification for Guesty.
@ -319,10 +319,11 @@ models:
- name: id_booking
data_type: text
description: "unique Superhog generated id for a booking.
note that this could be duplicated and both will be charged,
it's up to the user to not generate or cancel duplicate verifications"
note that there might be duplicate bookings on the original data
but we remove them keeping only the verification with the most recent update."
tests:
- not_null
- unique
- name: verification_status
data_type: text

87
models/reporting/edeposit/schema.yml
View file

@ -12,89 +12,90 @@ models:
- name: id_verification
data_type: text
description: "unique Superhog generated id for this verification"
tests:
- unique
- not_null
# tests:
# - unique
# - not_null
- name: id_booking
data_type: text
description: "unique Superhog generated id for a booking.
note that this could be duplicated and both will be charged,
it's up to the user to no generate duplicate verifications"
tests:
- not_null
note that there might be duplicate bookings on the original data
but we remove them keeping only the verification with the most recent update."
# tests:
# - not_null
# - unique
- name: currency
data_type: text
description: "currency in which the transaction actually happened"
tests:
- not_null
# tests:
# - not_null
- name: ok_status_fee_in_txn_currency
data_type: numeric
description: "fee charged in used currency for approved or flagged verifications and not cancelled"
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_between:
min_value: 0
strictly: false
# tests:
# - not_null
# - dbt_expectations.expect_column_values_to_be_between:
# min_value: 0
# strictly: false
- name: ok_status_fee_in_gbp
data_type: numeric
description: "fee charged in gbp for approved or flagged verifications and not cancelled"
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_between:
min_value: 0
strictly: false
# tests:
# - not_null
# - dbt_expectations.expect_column_values_to_be_between:
# min_value: 0
# strictly: false
- name: rejected_fee_in_txn_currency
data_type: numeric
description: "fee charged in used currency for rejected verifications"
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_between:
min_value: 0
strictly: false
# tests:
# - not_null
# - dbt_expectations.expect_column_values_to_be_between:
# min_value: 0
# strictly: false
- name: rejected_fee_in_gbp
data_type: numeric
description: "fee charged in gbp for rejected verifications"
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_between:
min_value: 0
strictly: false
# tests:
# - not_null
# - dbt_expectations.expect_column_values_to_be_between:
# min_value: 0
# strictly: false
- name: cancelled_fee_in_txn_currency
data_type: numeric
description: "fee charged in used currency for cancelled verifications"
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_between:
min_value: 0
strictly: false
# tests:
# - not_null
# - dbt_expectations.expect_column_values_to_be_between:
# min_value: 0
# strictly: false
- name: cancelled_fee_in_gbp
data_type: numeric
description: "fee charged in gbp for cancelled verifications"
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_between:
min_value: 0
strictly: false
# tests:
# - not_null
# - dbt_expectations.expect_column_values_to_be_between:
# min_value: 0
# strictly: false
- name: checkout_date_utc
data_type: date
description: "Date of checkout for the booking"
tests:
- not_null
# tests:
# - not_null
- name: created_date_utc
data_type: date
description: "Date of creation of the verification in the system"
tests:
- not_null
# tests:
# - not_null
- name: edeposit__guesty_verifications
description: "This table shows all verification for Guesty.

72
models/staging/core/schema.yml
View file

@ -202,74 +202,74 @@ models:
- name: id
data_type: bigint
description: "Record id for this table"
tests:
- unique
- not_null
# tests:
# - unique
# - not_null
- name: id_user_partner
data_type: character varying
description: "Unique id for partner user"
tests:
- unique
- not_null
# tests:
# - unique
# - not_null
- name: currency
data_type: character varying
description: "Three-letter ISO code assigned to the currency used by user."
tests:
- not_null
- dbt_expectations.expect_column_values_to_match_regex:
regex: "^[A-Z]{3}$"
# tests:
# - not_null
# - dbt_expectations.expect_column_values_to_match_regex:
# regex: "^[A-Z]{3}$"
- name: nightly_fee
data_type: numeric
description: "Fee charged per night on booking"
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_between:
min_value: 0
max_value: 100
strictly: true
# tests:
# - not_null
# - dbt_expectations.expect_column_values_to_be_between:
# min_value: 0
# max_value: 100
# strictly: true
- name: cancellation_fee
data_type: numeric
description: "Fee charged per cancelled booking"
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_between:
min_value: 0
max_value: 100
strictly: true
# tests:
# - not_null
# - dbt_expectations.expect_column_values_to_be_between:
# min_value: 0
# max_value: 100
# strictly: true
- name: protection_lower_level
data_type: numeric
description: "Amount of protection given when verification is 'Passed'"
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_between:
min_value: 0
strictly: true
# tests:
# - not_null
# - dbt_expectations.expect_column_values_to_be_between:
# min_value: 0
# strictly: true
- name: protection_upper_level
data_type: numeric
description: "Amount of protection given when verification is 'Flagged'"
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_between:
min_value: 0
strictly: true
# tests:
# - not_null
# - dbt_expectations.expect_column_values_to_be_between:
# min_value: 0
# strictly: true
- name: created_at_utc
data_type: timestamp without time zone
description: "Timestamp of when user was created"
tests:
- not_null
# tests:
# - not_null
- name: updated_at_utc
data_type: timestamp without time zone
description: "Timestamp of when user was last updated"
tests:
- not_null
# tests:
# - not_null
- name: dwh_extracted_at_utc
data_type: timestamp with time zone