counterweight/data-dwh-dbt-project
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'master' of ssh.dev.azure.com:v3/guardhog/Data/data-dwh-dbt-project

Browse source
This commit is contained in:
Pablo Martin 2024-12-23 12:19:55 +01:00
commit 5fd69b7101
24 changed files with 1208 additions and 93 deletions
Download patch file Download diff file Expand all files Collapse all files

10
macros/tests/at_least_one_null.sql Normal file
View file

@ -0,0 +1,10 @@
{% test at_least_one_null(model, columns) %}
select *
from {{ model }}
where
{% for column in columns %}
{{ column }} is not null {% if not loop.last %} and {% endif %}
{% endfor %}
-- If all columns are not null, the row violates the test
{% endtest %}

3
macros/tests/is_first_day_of_month.sql Normal file
View file

@ -0,0 +1,3 @@
{% test is_first_day_of_month(model, column_name) %}
select * from {{ model }} where extract(day from {{ column_name }}) != 1
{% endtest %}

10
macros/tests/is_last_day_of_month.sql Normal file
View file

@ -0,0 +1,10 @@
{% test is_last_day_of_month(model, column_name) %}
select *
from {{ model }}
where
{{ column_name }} != (
date_trunc('MONTH', {{ column_name }})
+ interval '1 MONTH'
- interval '1 DAY'
)
{% endtest %}

18
models/intermediate/check_in_hero/int_check_in_hero__checkins.sql
View file

@ -1,18 +1,17 @@
{% set api_name = "CHECKINHEROAPI" %}
with
stg_check_in_hero__checkins as (
select * from {{ ref("stg_check_in_hero__checkins") }}
),
stg_core__apim_user as (select * from {{ ref("stg_core__apim_user") }}),
stg_core__apim_user_type as (select * from {{ ref("stg_core__apim_user_type") }}),
stg_core__user as (select * from {{ ref("stg_core__user") }})
int_core__check_in_hero_users as (
select * from {{ ref("int_core__check_in_hero_users") }}
)
select
c.id_record,
c.id_user_partner,
c.id_reservation,
c.id_accommodation,
chu.id_currency,
c.guest_last_name,
c.guest_first_name,
c.guest_email,
@ -31,7 +30,7 @@ select
c.listing_town,
c.listing_country,
c.listing_postcode,
u.company_name,
chu.company_name,
c.user_email,
c.updated_at_utc,
c.updated_date_utc,
@ -39,9 +38,4 @@ select
c.cosmos_creation_date_utc,
c.created_date_utc
from stg_check_in_hero__checkins c
inner join stg_core__apim_user au on c.id_user_partner = au.id_apim_user
inner join
stg_core__apim_user_type aut
on au.id_apim_user_type = aut.id_apim_user_type
and upper(aut.user_type_name) = '{{ api_name }}'
left join stg_core__user u on u.id_user = au.id_apim_user
inner join int_core__check_in_hero_users chu on c.id_user_partner = chu.id_apim_user

10
models/intermediate/check_in_hero/schema.yml
View file

@ -31,6 +31,16 @@ models:
tests:
- not_null
- name: id_currency
data_type: bigint
description: "Id of the currency, works as a foreign key to the
currency table"
tests:
- not_null
- relationships:
to: ref('stg_core__currency')
field: id_currency
- name: guest_last_name
data_type: text
description: Last name of the guest checking in.

26
models/intermediate/core/int_core__check_in_hero_users.sql Normal file
View file

@ -0,0 +1,26 @@
{% set api_name = "CHECKINHEROAPI" %}
with
stg_core__apim_user as (select * from {{ ref("stg_core__apim_user") }}),
stg_core__apim_user_type as (select * from {{ ref("stg_core__apim_user_type") }}),
stg_core__user as (select * from {{ ref("stg_core__user") }})
select
au.id_apim_user,
au.id_apim_user_type,
u.id_account_currency as id_currency,
au.json_document_user_data ->> 'DealId' as id_deal,
au.json_document_user_data ->> 'AccountType' as account_type,
au.json_document_user_data ->> 'ClientMarkup' as client_markup,
(au.json_document_user_data ->> 'BookingFee')::decimal
as booking_fee_in_local_currency,
u.company_name,
au.created_at_utc,
au.created_date_utc,
au.updated_at_utc,
au.updated_date_utc
from stg_core__apim_user au
inner join
stg_core__apim_user_type aut
on au.id_apim_user_type = aut.id_apim_user_type
and upper(aut.user_type_name) = '{{ api_name }}'
inner join stg_core__user u on au.id_apim_user = u.id_user

47
models/intermediate/core/int_core__screen_and_protect_users.sql Normal file
View file

@ -0,0 +1,47 @@
{% set api_name = "SCREENANDPROTECTAPI" %}
with
stg_core__apim_user as (select * from {{ ref("stg_core__apim_user") }}),
stg_core__apim_user_type as (select * from {{ ref("stg_core__apim_user_type") }}),
stg_core__user as (select * from {{ ref("stg_core__user") }})
select
au.id_apim_user,
au.id_apim_user_type,
u.id_account_currency as id_currency,
au.json_document_user_data ->> 'DealId' as id_deal,
au.json_document_user_data ->> 'AccountType' as account_type,
au.json_document_user_data ->> 'ClientMarkup' as client_markup,
au.json_document_user_data ->> 'TechEmailAddress' as tech_email_address,
au.json_document_user_data ->> 'InvoicingEmailAddress' as invoicing_email_address,
case
when (au.json_document_user_data ->> 'FlaggedNotProtected') = 'false'
then true
else false
end as is_protected,
(au.json_document_user_data ->> 'PriceIncrease')::decimal as price_increase,
(au.json_document_user_data ->> 'PriceIncreaseStartDate')::date
as price_increase_start_date_utc,
(au.json_document_user_data ->> 'MonthlyVolumeDiscount')::decimal
as monthly_volume_discount,
(au.json_document_user_data ->> 'ThresholdApprovedBookingVolume')::integer
as threshold_approved_booking_volume,
(au.json_document_user_data ->> 'MonthlyVolumeDiscountStartDate')::date
as monthly_volume_discount_start_date_utc,
(au.json_document_user_data ->> 'MonthlyVolumeDiscountEndDate')::date
as monthly_volume_discount_end_date_utc,
(au.json_document_user_data ->> 'MonthlyGeneralDiscount')::decimal
as monthly_general_discount,
(au.json_document_user_data ->> 'MonthlyGeneralDiscountStartDate')::date
as monthly_general_discount_start_date_utc,
(au.json_document_user_data ->> 'MonthlyGeneralDiscountEndDate')::date
as monthly_general_discount_end_date_utc,
au.created_at_utc,
au.created_date_utc,
au.updated_at_utc,
au.updated_date_utc
from stg_core__apim_user au
inner join
stg_core__apim_user_type aut
on au.id_apim_user_type = aut.id_apim_user_type
and upper(aut.user_type_name) = '{{ api_name }}'
inner join stg_core__user u on au.id_apim_user = u.id_user

12
models/intermediate/core/int_core__unified_user.sql
View file

@ -1,11 +1,15 @@
{% set test_account_type_name = "('TEST')" %}
with
stg_core__user as (select * from {{ ref("stg_core__user") }}),
stg_core__superhog_user as (select * from {{ ref("stg_core__superhog_user") }}),
stg_core__currency as (select * from {{ ref("stg_core__currency") }}),
stg_core__country as (select * from {{ ref("stg_core__country") }})
stg_core__country as (select * from {{ ref("stg_core__country") }}),
stg_core__account_type as (select * from {{ ref("stg_core__account_type") }})
select
u.id_user,
u.id_account_type,
act.account_type_display_name as account_type,
u.id_billing_country,
co.country_name as billing_country_name,
co.iso_2 as billing_country_iso_2,
@ -42,8 +46,12 @@ select
su.number_of_properties,
su.id_superhog_verified_set,
su.platform_comms_recipient,
su.id_user_verification_status
su.id_user_verification_status,
case
when act.account_type_name in {{ test_account_type_name }} then true else false
end as is_test_account
from stg_core__user as u
inner join stg_core__superhog_user as su on u.id_user = su.id_superhoguser
left join stg_core__currency cu on u.id_account_currency = cu.id_currency
left join stg_core__country co on u.id_billing_country = co.id_country
left join stg_core__account_type act on u.id_account_type = act.id_account_type

3
models/intermediate/core/int_core__user_host.sql
View file

@ -30,7 +30,8 @@ with
)
select
uu.id_user as id_user_host,
uu.id_account_type,
uu.account_type,
uu.is_test_account,
uu.id_billing_country,
uu.billing_country_name,
uu.billing_country_iso_2,

388
models/intermediate/core/schema.yml
View file

@ -178,6 +178,144 @@ models:
tests:
- not_null
- unique
- name: id_account_type
data_type: bigint
description: |
The ID of the account type. Might be null and not necessarily up to date.
- name: account_type
data_type: character varying
description: |
The name of the account type. Might be null and not necessarily up to date.
- name: id_billing_country
data_type: bigint
description: The ID of the billing country. Can be null.
- name: billing_country_name
data_type: character varying
description: The name of the billing country. Can be null.
- name: billing_country_iso_2
data_type: char(2)
description: The ISO 3166-1 alpha-2 code of the billing country. Can be null.
- name: billing_country_iso_3
data_type: char(3)
description: The ISO 3166-1 alpha-3 code of the billing country. Can be null.
- name: account_currency_iso4217
data_type: char(3)
description: The ISO 4217 code of the account currency. Can be null.
- name: user_code
data_type: bigint
description: The user code.
- name: first_name
data_type: character varying
description: The first name of the user.
- name: last_name
data_type: character varying
description: The last name of the user.
- name: email
data_type: character varying
description: The email of the user.
- name: phone_number
data_type: character varying
description: The phone number of the user.
- name: title
data_type: character varying
description: The title of the user.
- name: id_deal
data_type: character varying
description: |
The ID of the deal of the user. It only applies to client accounts.
Can be null.
- name: is_deleted
data_type: boolean
description: |
True if the user has been deleted. False otherwise.
- name: joined_at_utc
data_type: timestamp
description: The timestamp when the user joined.
- name: joined_date_utc
data_type: date
description: The date when the user joined.
- name: user_name
data_type: character varying
description: The user name.
- name: code_prefix
data_type: character varying
description: The code prefix.
- name: billing_town
data_type: character varying
description: The billing town.
- name: company_name
data_type: character varying
description: The company name.
- name: is_email_confirmed
data_type: boolean
description: |
True if the email of the user has been confirmed. False otherwise.
- name: is_lockout_enabled
data_type: boolean
description: |
True if the lockout has been enabled. False otherwise.
- name: billing_postcode
data_type: character varying
description: The billing postcode.
- name: is_twofactor_enabled
data_type: boolean
description: |
True if the two-factor authentication has been enabled. False otherwise.
- name: access_failed_count
data_type: bigint
description: The access failed count.
- name: lockout_end_date_utc
data_type: timestamp
description: The lockout end date.
- name: avatar
data_type: character varying
description: The avatar of the user.
- name: id_airbnb
data_type: character varying
description: The ID of the Airbnb.
- name: airbnb_url
data_type: character varying
description: The Airbnb URL.
- name: created_date_utc
data_type: timestamp
description: The timestamp when the user was created. The field name needs to be fixed.
- name: updated_date_utc
data_type: timestamp
description: The timestamp when the user was updated. The field name needs to be fixed.
- name: verified_at_utc
data_type: timestamp
description: The timestamp when the user was verified.
- name: verified_date_utc
data_type: date
description: The date when the user was verified.
- name: flag_as_problem
data_type: boolean
description: |
True if the user has been flagged as a problem. False otherwise.
- name: number_of_properties
data_type: bigint
description: |
The number of properties according to the backend. For a better
figure we recommend using Hubspot data. For a real figure
of properties actually being active in our business, we recommend
using other sources of data.
- name: id_superhog_verified_set
data_type: bigint
description: The ID of the Superhog verified set.
- name: platform_comms_recipient
data_type: boolean
description: |
True if the user is a platform comms recipient. False otherwise.
- name: id_user_verification_status
data_type: bigint
description: The ID of the user verification status.
- name: is_test_account
data_type: boolean
description: |
True if the user is a test account. False otherwise.
Note that there might be other users that are test accounts
that are not controlled by this field. However it provides
a first level of filtering.
- name: int_core__vr_check_in_cover
description: "This tables holds information on verification requests
@ -1738,10 +1876,17 @@ models:
tests:
- not_null
- unique
- name: id_account_type
data_type: integer
- name: account_type
data_type: string
description: |
Account type ID. Can be null and might be not up-to-date.
Name of the account type. Can be null and might be not up-to-date.
- name: is_test_account
data_type: boolean
description: |
Flag to identify if the user is a test account.
Can be null and might be not up-to-date. Be aware that
this might not include all test accounts, but it provides
a first level of filtering.
- name: id_billing_country
data_type: integer
description: |
@ -4427,3 +4572,240 @@ models:
description: |
Check-in Cover fees revenue paid without taxes in GBP.
Can be null.
- name: int_core__screen_and_protect_users
description: |
"Contains information about Screen & Protect API users.
It includes all user details related with the service."
columns:
- name: id_apim_user
data_type: character varying
description: "Identifier of the User. Acts as primary
key for this table."
tests:
- not_null
- unique
- name: id_apim_user_type
data_type: bigint
description: "Identifier of the type of user."
- name: id_currency
data_type: bigint
description: "Id of the currency, works as a foreign key to the
currency table"
tests:
- not_null
- relationships:
to: ref('stg_core__currency')
field: id_currency
- name: id_deal
data_type: text
description: ""
- name: account_type
data_type: text
description: ""
- name: client_markup
data_type: text
description: ""
- name: tech_email_address
data_type: text
description: ""
- name: invoicing_email_address
data_type: text
description: ""
- name: is_protected
data_type: boolean
description: |
Indicates if the user's bookings are protected or not.
tests:
- not_null
- name: price_increase
data_type: numeric
description: The percentage or value of the price increase
applied to the user's account.
tests:
- dbt_expectations.expect_column_values_to_be_between:
min_value: 0
strictly: true
- name: price_increase_start_date_utc
data_type: date
description: The date when the price increase becomes effective.
This is the first day of the month.
tests:
- is_first_day_of_month
- name: monthly_volume_discount
data_type: numeric
description: The discount percentage or value offered based on the
volume of bookings achieved within a month.
No user can have more than one discount per month.
tests:
- dbt_expectations.expect_column_values_to_be_between:
min_value: 0
max_value: 100
strictly: true
- name: threshold_approved_booking_volume
data_type: numeric
description: The minimum number of bookings required to qualify for
the monthly volume discount.
tests:
- dbt_expectations.expect_column_values_to_be_between:
min_value: 0
strictly: true
- name: monthly_volume_discount_start_date_utc
data_type: date
description: The date when the monthly volume discount period begins.
This is the first day of the month.
tests:
- is_first_day_of_month
- name: monthly_volume_discount_end_date_utc
data_type: date
description: The date when the monthly volume discount period ends.
This is the last day of the month.
tests:
- is_last_day_of_month
- name: monthly_general_discount
data_type: numeric
description: The general discount percentage or value applied to all
bookings within the applicable period.
No user can have more than one discount per month.
tests:
- dbt_expectations.expect_column_values_to_be_between:
min_value: 0
max_value: 100
strictly: true
- name: monthly_general_discount_start_date_utc
data_type: date
description: The date when the general discount period begins.
This is the first day of the month.
tests:
- is_first_day_of_month
- name: monthly_general_discount_end_date_utc
data_type: date
description: The date when the general discount period ends.
This is the last day of the month.
tests:
- is_last_day_of_month
- name: created_at_utc
data_type: timestamp
description: |
Timestamp of when this user was created.
tests:
- not_null
- name: created_date_utc
data_type: date
description: |
Date of when this user was created.
tests:
- not_null
- name: updated_at_utc
data_type: timestamp
description: |
Timestamp of when this user was last updated.
tests:
- not_null
- name: updated_date_utc
data_type: date
description: |
Date of when this user was last updated.
tests:
- not_null
- name: int_core__check_in_hero_users
description: |
"Contains information about CheckIn Hero API users.
It includes all user details related with the service."
columns:
- name: id_apim_user
data_type: character varying
description: "Identifier of the User. Acts as primary
key for this table."
tests:
- not_null
- unique
- name: id_apim_user_type
data_type: bigint
description: "Identifier of the type of user."
- name: id_currency
data_type: bigint
description: "Id of the currency, works as a foreign key to the
currency table"
tests:
- not_null
- relationships:
to: ref('stg_core__currency')
field: id_currency
- name: id_deal
data_type: text
description: ""
- name: account_type
data_type: text
description: ""
- name: client_markup
data_type: text
description: ""
- name: booking_fee_in_local_currency
data_type: numeric
description: "Fee charged to the user for each booking in the local currency."
tests:
- dbt_expectations.expect_column_values_to_be_between:
min_value: 0
max_value: 1000
strictly: true
- name: company_name
data_type: text
description: Company name of partner user.
- name: created_at_utc
data_type: timestamp
description: |
Timestamp of when this user was created.
tests:
- not_null
- name: created_date_utc
data_type: date
description: |
Date of when this user was created.
tests:
- not_null
- name: updated_at_utc
data_type: timestamp
description: |
Timestamp of when this user was last updated.
tests:
- not_null
- name: updated_date_utc
data_type: date
description: |
Date of when this user was last updated.
tests:
- not_null

42
models/intermediate/screen_and_protect/int_screen_and_protect__verification_requests.sql
View file

@ -1,30 +1,33 @@
{% set api_name = "SCREENANDPROTECTAPI" %}
with
stg_screen_and_protect__verification_requests as (
select * from {{ ref("stg_screen_and_protect__verification_requests") }}
),
stg_core__apim_user as (select * from {{ ref("stg_core__apim_user") }}),
stg_core__apim_user_type as (select * from {{ ref("stg_core__apim_user_type") }})
int_core__screen_and_protect_users as (
select * from {{ ref("int_core__screen_and_protect_users") }}
)
select
-- note that these ids are not the same as the ones found in Core DWH
-- they are completely unrelated
vr.id_verification,
vr.id_booking,
vr.id_user_partner,
vr.id_accommodation,
case
when (au.json_document_user_data ->> 'FlaggedNotProtected') = 'false'
then true
else false
end as is_protected,
spu.id_currency,
spu.is_protected,
vr.protection_type,
vr.protection_starting_level,
vr.protection_basic_amount,
vr.protection_extended_amount,
vr.protection_starting_amount_in_local_currency,
vr.protection_basic_amount_in_local_currency,
vr.protection_extended_amount_in_local_currency,
vr.pet_protection,
vr.verification_status,
vr.verification_status_reason,
spu.price_increase,
spu.price_increase_start_date_utc,
spu.monthly_volume_discount,
spu.threshold_approved_booking_volume,
spu.monthly_volume_discount_start_date_utc,
spu.monthly_volume_discount_end_date_utc,
spu.monthly_general_discount,
spu.monthly_general_discount_start_date_utc,
spu.monthly_general_discount_end_date_utc,
vr.email_flag,
vr.phone_flag,
vr.watch_list,
@ -53,12 +56,9 @@ select
vr.status_updated_date_utc,
vr.updated_at_utc,
vr.updated_date_utc,
vr.cosmos_creation_at_utc,
vr.cosmos_creation_date_utc,
vr.created_date_utc
vr.creation_at_utc,
vr.creation_date_utc,
vr.cosmos_created_date_utc
from stg_screen_and_protect__verification_requests vr
inner join stg_core__apim_user au on vr.id_user_partner = au.id_apim_user
inner join
stg_core__apim_user_type aut
on au.id_apim_user_type = aut.id_apim_user_type
and upper(aut.user_type_name) = '{{ api_name }}'
int_core__screen_and_protect_users spu on vr.id_user_partner = spu.id_apim_user

119
models/intermediate/screen_and_protect/schema.yml
View file

@ -6,6 +6,11 @@ models:
Records of verification requests from the Screen and Protect API. The
table tracks verification requests, their outcomes, and related metadata
about guests, listings, and partners.
tests:
- at_least_one_null:
columns:
- monthly_volume_discount
- monthly_general_discount
columns:
- name: id_verification
data_type: text
@ -30,6 +35,16 @@ models:
data_type: text
description: Identifier for the accommodation related to the booking.
- name: id_currency
data_type: bigint
description: "Id of the currency, works as a foreign key to the
currency table"
tests:
- not_null
- relationships:
to: ref('stg_core__currency')
field: id_currency
- name: is_protected
data_type: boolean
description: |
@ -50,30 +65,33 @@ models:
- "DAMAGE WAIVER"
- "PET PROTECTION"
- name: protection_starting_level
- name: protection_starting_amount_in_local_currency
data_type: numeric
description: Field used for protection type "STANDALONE PROTECTION" to
indicate the starting level of protection.
In local currency.
tests:
- dbt_expectations.expect_column_values_to_be_between:
min_value: 0
max_value: 100000
strictly: true
- name: protection_basic_amount
- name: protection_basic_amount_in_local_currency
data_type: numeric
description: Field used for protection type "SCREEN & PROTECT",
"BASIC PROTECTION" or "DAMAGE WAIVER" to show the basic amount of protection.
In local currency.
tests:
- dbt_expectations.expect_column_values_to_be_between:
min_value: 0
max_value: 100000
strictly: true
- name: protection_extended_amount
- name: protection_extended_amount_in_local_currency
data_type: numeric
description: Field used for protection type "SCREEN & PROTECT",
or "STANDALONE PROTECTION" to show the basic amount of protection.
In local currency.
tests:
- dbt_expectations.expect_column_values_to_be_between:
min_value: 0
@ -95,6 +113,81 @@ models:
- "FLAGGED"
- "REJECTED"
- name: price_increase
data_type: numeric
description: The percentage or value of the price increase
applied to the user's account.
tests:
- dbt_expectations.expect_column_values_to_be_between:
min_value: 0
strictly: true
- name: price_increase_start_date_utc
data_type: date
description: The date when the price increase becomes effective.
This is the first day of the month.
tests:
- is_first_day_of_month
- name: monthly_volume_discount
data_type: numeric
description: The discount percentage or value offered based on the
volume of bookings achieved within a month.
No user can have more than one discount per month.
tests:
- dbt_expectations.expect_column_values_to_be_between:
min_value: 0
max_value: 100
strictly: true
- name: threshold_approved_booking_volume
data_type: numeric
description: The minimum number of bookings required to qualify for
the monthly volume discount.
tests:
- dbt_expectations.expect_column_values_to_be_between:
min_value: 0
strictly: true
- name: monthly_volume_discount_start_date_utc
data_type: date
description: The date when the monthly volume discount period begins.
This is the first day of the month.
tests:
- is_first_day_of_month
- name: monthly_volume_discount_end_date_utc
data_type: date
description: The date when the monthly volume discount period ends.
This is the last day of the month.
tests:
- is_last_day_of_month
- name: monthly_general_discount
data_type: numeric
description: The general discount percentage or value applied to all
bookings within the applicable period.
No user can have more than one discount per month.
tests:
- dbt_expectations.expect_column_values_to_be_between:
min_value: 0
max_value: 100
strictly: true
- name: monthly_general_discount_start_date_utc
data_type: date
description: The date when the general discount period begins.
This is the first day of the month.
tests:
- is_first_day_of_month
- name: monthly_general_discount_end_date_utc
data_type: date
description: The date when the general discount period ends.
This is the last day of the month.
tests:
- is_last_day_of_month
- name: verification_status_reason
data_type: text
description: Reason for the verification status.
@ -225,23 +318,23 @@ models:
tests:
- not_null
- name: cosmos_creation_at_utc
- name: creation_at_utc
data_type: timestamp without time zone
description: |
Timestamp of when the verification request was created in Cosmos DB.
Timestamp of when the reservation was created.
tests:
- not_null
- name: cosmos_creation_date_utc
- name: creation_date_utc
data_type: date
description: |
Date of when the reservation was created.
tests:
- not_null
- name: cosmos_created_date_utc
data_type: date
description: |
Date of when the verification request was created in Cosmos DB.
tests:
- not_null
- name: created_date_utc
data_type: date
description: |
Date when the reservation was created.
tests:
- not_null

16
models/reporting/exposures.yml
View file

@ -335,3 +335,19 @@ exposures:
owner:
name: Joaquin Ossa
email: joaquin.ossa@superhog.com
- name: fx_rates_integration_with_billing_db
label: FX Rates integration with Billing DB
type: application
maturity: high
url: https://www.notion.so/knowyourguest-superhog/Currency-Rates-for-apps-integration-1600446ff9c9804faa66f982f294e6e8?pvs=4
description: |
We currently push our currency rates data into the BillingDB with Airbyte.
Application runtimes request currency rates data from there.
depends_on:
- ref('daily_currency_exchange_rates')
owner:
name: Pablo Martin
email: pablo.martin@superhog.com

29
models/reporting/screen_and_protect/schema.yml
View file

@ -50,30 +50,33 @@ models:
- "DAMAGE WAIVER"
- "PET PROTECTION"
- name: protection_starting_level
- name: protection_starting_amount_in_local_currency
data_type: numeric
description: Field used for protection type "STANDALONE PROTECTION" to
indicate the starting level of protection.
In local currency.
tests:
- dbt_expectations.expect_column_values_to_be_between:
min_value: 0
max_value: 100000
strictly: true
- name: protection_basic_amount
- name: protection_basic_amount_in_local_currency
data_type: numeric
description: Field used for protection type "SCREEN & PROTECT",
"BASIC PROTECTION" or "DAMAGE WAIVER" to show the basic amount of protection.
In local currency.
tests:
- dbt_expectations.expect_column_values_to_be_between:
min_value: 0
max_value: 100000
strictly: true
- name: protection_extended_amount
- name: protection_extended_amount_in_local_currency
data_type: numeric
description: Field used for protection type "SCREEN & PROTECT",
or "STANDALONE PROTECTION" to show the basic amount of protection.
In local currency.
tests:
- dbt_expectations.expect_column_values_to_be_between:
min_value: 0
@ -225,23 +228,23 @@ models:
tests:
- not_null
- name: cosmos_creation_at_utc
- name: creation_at_utc
data_type: timestamp without time zone
description: |
Timestamp of when the verification request was created in Cosmos DB.
Timestamp of when the reservation was created.
tests:
- not_null
- name: cosmos_creation_date_utc
- name: creation_date_utc
data_type: date
description: |
Date of when the reservation was created.
tests:
- not_null
- name: cosmos_created_date_utc
data_type: date
description: |
Date of when the verification request was created in Cosmos DB.
tests:
- not_null
- name: created_date_utc
data_type: date
description: |
Date when the reservation was created.
tests:
- not_null

17
models/reporting/screen_and_protect/screen_and_protect__verification_requests.sql
View file

@ -3,17 +3,18 @@ with
select * from {{ ref("int_screen_and_protect__verification_requests") }}
)
select
-- note that these ids are not the same as the ones found in Core DWH
-- they are completely unrelated
id_verification as id_verification,
id_booking as id_booking,
id_user_partner as id_user_partner,
id_accommodation as id_accommodation,
is_protected as is_protected,
protection_type as protection_type,
protection_starting_level as protection_starting_level,
protection_basic_amount as protection_basic_amount,
protection_extended_amount as protection_extended_amount,
protection_starting_amount_in_local_currency
as protection_starting_amount_in_local_currency,
protection_basic_amount_in_local_currency
as protection_basic_amount_in_local_currency,
protection_extended_amount_in_local_currency
as protection_extended_amount_in_local_currency,
pet_protection as pet_protection,
verification_status as verification_status,
verification_status_reason as verification_status_reason,
@ -45,7 +46,7 @@ select
status_updated_date_utc as status_updated_date_utc,
updated_at_utc as updated_at_utc,
updated_date_utc as updated_date_utc,
cosmos_creation_at_utc as cosmos_creation_at_utc,
cosmos_creation_date_utc as cosmos_creation_date_utc,
created_date_utc as created_date_utc
creation_at_utc as creation_at_utc,
creation_date_utc as creation_date_utc,
cosmos_created_date_utc as cosmos_created_date_utc
from int_screen_and_protect__verification_requests

10
models/staging/core/_core_sources.yml
View file

@ -267,3 +267,13 @@ sources:
identifier: ApimUser
- name: ApimUserType
identifier: ApimUserType
- name: BasicProtection
identifier: BasicProtection
- name: DamageWaiver
identifier: DamageWaiver
- name: ScreenAndProtect
identifier: ScreenAndProtect
- name: StandaloneProtection
identifier: StandaloneProtection
- name: AccountType
identifier: AccountType

400
models/staging/core/schema.yml
View file

@ -1443,3 +1443,403 @@ models:
Timestamp of when this record was extracted into DWH.
tests:
- not_null
- name: stg_core__basic_protection
description: "Contains the basic protection fees and protection amounts
for different protection options and currencies."
tests:
- dbt_expectations.expect_column_pair_values_A_to_be_greater_than_B:
column_A: end_at_utc
column_B: start_at_utc
or_equal: True
columns:
- name: id_basic_protection
data_type: bigint
description: "Unique identifier of the basic protection.
Acts as the primary key for this table."
tests:
- not_null
- unique
- name: long_stay_fee_in_local_currency
data_type: numeric
description: "Nightly fee in local currency for long stay bookings.
Bookings of more than 30 days are considered long."
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_between:
min_value: 0
max_value: 1000
strictly: true
- name: short_stay_fee_in_local_currency
data_type: numeric
description: "Nightly fee in local currency for short stay bookings"
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_between:
min_value: 0
max_value: 1000
strictly: true
- name: id_currency
data_type: bigint
description: "Id of the currency, works as a foreign key to the
currency table"
tests:
- not_null
- relationships:
to: ref('stg_core__currency')
field: id_currency
- name: protection_basic_amount_in_local_currency
data_type: bigint
description: "Amount in local currency of basic protection coverage"
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_between:
min_value: 0
max_value: 1000000
strictly: true
- name: start_at_utc
data_type: timestamp
description: "Timestamp of when this basic protection record is active"
tests:
- not_null
- name: start_date_utc
data_type: date
description: "Date of when this basic protection record is active"
tests:
- not_null
- name: end_at_utc
data_type: timestamp
description:
"Timestamp of when this basic protection record is no longer active.
It can be null."
- name: end_date_utc
data_type: date
description:
"Date of when this basic protection record is no longer active.
It can be null."
- name: dwh_extracted_at_utc
data_type: timestamp
description: |
Timestamp of when this record was extracted into DWH.
tests:
- not_null
- name: stg_core__damage_waiver
description: "Contains the damage waiver fees and protection amounts
for different protection options and currencies."
tests:
- dbt_expectations.expect_column_pair_values_A_to_be_greater_than_B:
column_A: end_at_utc
column_B: start_at_utc
or_equal: True
columns:
- name: id_damage_waiver
data_type: bigint
description: "Unique identifier of the damage waiver.
Acts as the primary key for this table."
tests:
- not_null
- unique
- name: booking_fee_in_local_currency
data_type: numeric
description: "Nightly fee in local currency for bookings"
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_between:
min_value: 0
max_value: 1000
strictly: true
- name: id_currency
data_type: bigint
description: "Id of the currency, works as a foreign key to the
currency table"
tests:
- not_null
- relationships:
to: ref('stg_core__currency')
field: id_currency
- name: protection_basic_amount_in_local_currency
data_type: bigint
description: "Amount in local currency of basic protection coverage"
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_between:
min_value: 0
max_value: 1000000
strictly: true
- name: start_at_utc
data_type: timestamp
description: "Timestamp of when this basic protection record is active"
tests:
- not_null
- name: start_date_utc
data_type: date
description: "Date of when this basic protection record is active"
tests:
- not_null
- name: end_at_utc
data_type: timestamp
description:
"Timestamp of when this basic protection record is no longer active.
It can be null."
- name: end_date_utc
data_type: date
description:
"Date of when this basic protection record is no longer active.
It can be null."
- name: dwh_extracted_at_utc
data_type: timestamp
description: |
Timestamp of when this record was extracted into DWH.
tests:
- not_null
- name: stg_core__screen_and_protect
description: "Contains the screen and protect fees and protection amounts
for different protection options and currencies."
tests:
- dbt_expectations.expect_column_pair_values_A_to_be_greater_than_B:
column_A: end_at_utc
column_B: start_at_utc
or_equal: True
columns:
- name: id_screen_and_protect
data_type: bigint
description: "Unique identifier of the Screen and Protect record.
Acts as the primary key for this table."
tests:
- not_null
- unique
- name: long_stay_fee_in_local_currency
data_type: numeric
description: "Nightly fee in local currency for long stay bookings.
Bookings of more than 30 days are considered long."
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_between:
min_value: 0
max_value: 1000
strictly: true
- name: short_stay_fee_in_local_currency
data_type: numeric
description: "Nightly fee in local currency for short stay bookings."
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_between:
min_value: 0
max_value: 1000
strictly: true
- name: id_currency
data_type: bigint
description: "Id of the currency, works as a foreign key
to the currency table."
tests:
- not_null
- relationships:
to: ref('stg_core__currency')
field: id_currency
- name: protection_basic_amount_in_local_currency
data_type: bigint
description: "Amount in local currency of basic protection coverage."
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_between:
min_value: 0
max_value: 1000000
strictly: true
- name: protection_extended_amount_in_local_currency
data_type: bigint
description: "Amount in local currency of extended protection coverage."
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_between:
min_value: 0
max_value: 100000000
strictly: true
- name: start_at_utc
data_type: timestamp without time zone
description: "Timestamp when this Screen and Protect record becomes active."
tests:
- not_null
- name: start_date_utc
data_type: date
description: "Date when this Screen and Protect record becomes active."
tests:
- not_null
- name: end_at_utc
data_type: timestamp without time zone
description: "Timestamp when this Screen and Protect record is no
longer active. It can be null."
- name: end_date_utc
data_type: date
description: "Date when this Screen and Protect record is no
longer active. It can be null."
- name: dwh_extracted_at_utc
data_type: timestamp with time zone
description: |
Timestamp of when this record was extracted into DWH.
tests:
- not_null
- name: stg_core__standalone_protection
description: "Contains the standalone protection fees and protection amounts
for different protection options and currencies."
tests:
- dbt_expectations.expect_column_pair_values_A_to_be_greater_than_B:
column_A: end_at_utc
column_B: start_at_utc
or_equal: True
columns:
- name: id_standalone_protection
data_type: bigint
description: "Unique identifier for the standalone protection record.
Acts as the primary key for this table."
tests:
- not_null
- unique
- name: long_stay_fee_in_local_currency
data_type: numeric
description: "Nightly fee in local currency for long stay bookings.
Bookings of more than 30 days are considered long."
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_between:
min_value: 0
max_value: 1000
strictly: true
- name: short_stay_fee_in_local_currency
data_type: numeric
description: "Nightly fee in local currency for short stay bookings."
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_between:
min_value: 0
max_value: 1000
strictly: true
- name: id_currency
data_type: bigint
description: "Id of the currency, works as a foreign key
to the currency table."
tests:
- not_null
- relationships:
to: ref('stg_core__currency')
field: id_currency
- name: protection_starting_amount_in_local_currency
data_type: bigint
description: "Starting amount in local currency of protection coverage."
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_between:
min_value: 0
max_value: 1000000
strictly: true
- name: protection_extended_amount_in_local_currency
data_type: bigint
description: "Extended amount in local currency of protection coverage."
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_between:
min_value: 0
max_value: 100000000
strictly: true
- name: start_at_utc
data_type: timestamp without time zone
description: "Timestamp when this standalone protection record becomes active."
tests:
- not_null
- name: start_date_utc
data_type: date
description: "Date when this standalone protection record becomes active."
tests:
- not_null
- name: end_at_utc
data_type: timestamp without time zone
description: "Timestamp when this standalone protection record is no
longer active. It can be null."
- name: end_date_utc
data_type: date
description: "Date when this standalone protection record is no
longer active. It can be null."
- name: dwh_extracted_at_utc
data_type: timestamp with time zone
description: "Timestamp of when this record was extracted into DWH."
tests:
- not_null
- name: stg_core__account_type
description: |
A table containing the different types of accounts created in SH backend.
columns:
- name: id_account_type
data_type: bigint
description: |
Unique identifier of the account type.
Acts as the primary key for this table.
tests:
- not_null
- unique
- name: account_type_name
data_type: character varying
description: |
Name of the account, in upper case, without spacing.
Cannot be null. Must be unique.
tests:
- not_null
- unique
- name: account_type_display_name
data_type: character varying
description: |
A more nicer way to display the name of the account, better fit
for reporting purposes.
Cannot be null. Must be unique.
tests:
- not_null
- unique
- name: dwh_extracted_at_utc
data_type: timestamp with time zone
description: |
Timestamp of when this record was extracted into DWH.
tests:
- not_null

13
models/staging/core/stg_core__account_type.sql Normal file
View file

@ -0,0 +1,13 @@
with
raw_account_type as (select * from {{ source("core", "AccountType") }}),
stg_core__account_type as (
select
{{ adapter.quote("Id") }} as id_account_type,
upper({{ adapter.quote("Name") }}) as account_type_name,
{{ adapter.quote("FullName") }} as account_type_display_name,
{{ adapter.quote("_airbyte_extracted_at") }} as dwh_extracted_at_utc
from raw_account_type
)
select *
from stg_core__account_type

20
models/staging/core/stg_core__basic_protection.sql Normal file
View file

@ -0,0 +1,20 @@
with
raw_basic_protection as (select * from {{ source("core", "BasicProtection") }}),
stg_core__basic_protection as (
select
{{ adapter.quote("Id") }} as id_basic_protection,
{{ adapter.quote("LongStay") }} as long_stay_fee_in_local_currency,
{{ adapter.quote("ShortStay") }} as short_stay_fee_in_local_currency,
{{ adapter.quote("CurrencyId") }} as id_currency,
{{ adapter.quote("BasicAmount") }}
as protection_basic_amount_in_local_currency,
{{ adapter.quote("StartDate") }} as start_at_utc,
cast({{ adapter.quote("StartDate") }} as date) as start_date_utc,
{{ adapter.quote("EndDate") }} as end_at_utc,
cast({{ adapter.quote("EndDate") }} as date) as end_date_utc,
{{ adapter.quote("_airbyte_extracted_at") }} as dwh_extracted_at_utc
from raw_basic_protection
)
select *
from stg_core__basic_protection

19
models/staging/core/stg_core__damage_waiver.sql Normal file
View file

@ -0,0 +1,19 @@
with
raw_damage_waiver as (select * from {{ source("core", "DamageWaiver") }}),
stg_core__damage_waiver as (
select
{{ adapter.quote("Id") }} as id_damage_waiver,
{{ adapter.quote("BookingFee") }} as booking_fee_in_local_currency,
{{ adapter.quote("CurrencyId") }} as id_currency,
{{ adapter.quote("BasicAmount") }}
as protection_basic_amount_in_local_currency,
{{ adapter.quote("StartDate") }} as start_at_utc,
cast({{ adapter.quote("StartDate") }} as date) as start_date_utc,
{{ adapter.quote("EndDate") }} as end_at_utc,
cast({{ adapter.quote("EndDate") }} as date) as end_date_utc,
{{ adapter.quote("_airbyte_extracted_at") }} as dwh_extracted_at_utc
from raw_damage_waiver
)
select *
from stg_core__damage_waiver

22
models/staging/core/stg_core__screen_and_protect.sql Normal file
View file

@ -0,0 +1,22 @@
with
raw_screen_and_protect as (select * from {{ source("core", "ScreenAndProtect") }}),
stg_core__screen_and_protect as (
select
{{ adapter.quote("Id") }} as id_screen_and_protect,
{{ adapter.quote("LongStay") }} as long_stay_fee_in_local_currency,
{{ adapter.quote("ShortStay") }} as short_stay_fee_in_local_currency,
{{ adapter.quote("CurrencyId") }} as id_currency,
{{ adapter.quote("BasicAmount") }}
as protection_basic_amount_in_local_currency,
{{ adapter.quote("ExtendedAmount") }}
as protection_extended_amount_in_local_currency,
{{ adapter.quote("StartDate") }} as start_at_utc,
cast({{ adapter.quote("StartDate") }} as date) as start_date_utc,
{{ adapter.quote("EndDate") }} as end_at_utc,
cast({{ adapter.quote("EndDate") }} as date) as end_date_utc,
{{ adapter.quote("_airbyte_extracted_at") }} as dwh_extracted_at_utc
from raw_screen_and_protect
)
select *
from stg_core__screen_and_protect

24
models/staging/core/stg_core__standalone_protection.sql Normal file
View file

@ -0,0 +1,24 @@
with
raw_standalone_protection as (
select * from {{ source("core", "StandaloneProtection") }}
),
stg_core__standalone_protection as (
select
{{ adapter.quote("Id") }} as id_standalone_protection,
{{ adapter.quote("LongStay") }} as long_stay_fee_in_local_currency,
{{ adapter.quote("ShortStay") }} as short_stay_fee_in_local_currency,
{{ adapter.quote("CurrencyId") }} as id_currency,
{{ adapter.quote("StartingAmount") }}
as protection_starting_amount_in_local_currency,
{{ adapter.quote("ExtendedAmount") }}
as protection_extended_amount_in_local_currency,
{{ adapter.quote("StartDate") }} as start_at_utc,
cast({{ adapter.quote("StartDate") }} as date) as start_date_utc,
{{ adapter.quote("EndDate") }} as end_at_utc,
cast({{ adapter.quote("EndDate") }} as date) as end_date_utc,
{{ adapter.quote("_airbyte_extracted_at") }} as dwh_extracted_at_utc
from raw_standalone_protection
)
select *
from stg_core__standalone_protection

31
models/staging/screen_and_protect/schema.yml
View file

@ -43,30 +43,33 @@ models:
- "DAMAGE WAIVER"
- "PET PROTECTION"
- name: protection_starting_level
- name: protection_starting_amount_in_local_currency
data_type: numeric
description: Field used for protection type "STANDALONE PROTECTION" to
indicate the starting level of protection.
indicate the starting amount of protection.
In local currency.
tests:
- dbt_expectations.expect_column_values_to_be_between:
min_value: 0
max_value: 100000
strictly: true
- name: protection_basic_amount
- name: protection_basic_amount_in_local_currency
data_type: numeric
description: Field used for protection type "SCREEN & PROTECT",
"BASIC PROTECTION" or "DAMAGE WAIVER" to show the basic amount of protection.
In local currency.
tests:
- dbt_expectations.expect_column_values_to_be_between:
min_value: 0
max_value: 100000
strictly: true
- name: protection_extended_amount
- name: protection_extended_amount_in_local_currency
data_type: numeric
description: Field used for protection type "SCREEN & PROTECT",
or "STANDALONE PROTECTION" to show the basic amount of protection.
In local currency.
tests:
- dbt_expectations.expect_column_values_to_be_between:
min_value: 0
@ -214,27 +217,27 @@ models:
tests:
- not_null
- name: cosmos_creation_at_utc
- name: creation_at_utc
data_type: timestamp without time zone
description: |
Timestamp of when the verification request was created in Cosmos DB.
Timestamp of when the reservation was created.
tests:
- not_null
- name: cosmos_creation_date_utc
- name: creation_date_utc
data_type: date
description: |
Date of when the reservation was created.
tests:
- not_null
- name: cosmos_created_date_utc
data_type: date
description: |
Date of when the verification request was created in Cosmos DB.
tests:
- not_null
- name: created_date_utc
data_type: date
description: |
Date when the reservation was created.
tests:
- not_null
- name: cosmos_db_timestamp_utc
data_type: timestamp with time zone
description: Internal Cosmos DB timestamp of the last record update.

12
models/staging/screen_and_protect/stg_screen_and_protect__verification_requests.sql
View file

@ -22,7 +22,7 @@ with
then null
else {{ adapter.quote("documents") }} ->> 'ProtectionStartingLevel'
end as decimal(19, 4)
) as protection_starting_level,
) as protection_starting_amount_in_local_currency,
cast(
case
when
@ -31,7 +31,7 @@ with
then null
else {{ adapter.quote("documents") }} ->> 'ProtectionBasicAmount'
end as decimal(19, 4)
) as protection_basic_amount,
) as protection_basic_amount_in_local_currency,
cast(
case
when
@ -40,7 +40,7 @@ with
then null
else {{ adapter.quote("documents") }} ->> 'ProtectionExtendedAmount'
end as decimal(19, 4)
) as protection_extended_amount,
) as protection_extended_amount_in_local_currency,
({{ adapter.quote("documents") }} ->> 'PetProtection')::boolean
as pet_protection,
upper({{ adapter.quote("documents") }} ->> 'Status') as verification_status,
@ -86,11 +86,11 @@ with
({{ adapter.quote("documents") }} ->> 'UpdatedDate')::date
as updated_date_utc,
({{ adapter.quote("documents") }} ->> 'CreationDate')::timestamp
as cosmos_creation_at_utc,
as creation_at_utc,
({{ adapter.quote("documents") }} ->> 'CreationDate')::date
as cosmos_creation_date_utc,
as creation_date_utc,
({{ adapter.quote("documents") }} ->> 'CreatedDate')::date
as created_date_utc,
as cosmos_created_date_utc,
to_timestamp(
(({{ adapter.quote("documents") }} ->> '_ts'))::integer
) as cosmos_db_timestamp_utc