Merged PR 2619: edeposit verifications container to staging
# Description This PR turns the JSON documents of the verification container in sync into a table in staging. It also performs the necessary deduplication of records by ID and timestamp along the way. I'm intentionally skipping proper documentation to unblock a colleague while I fetch the necessary knowledge to populate the schema docs properly. I pinky promise I won't forget and I'll come back and fix it. # Checklist - [X] The edited models and dependants run properly with production data. - [ ] ~~The edited models are sufficiently documented.~~ __Nope, read above.__ - [X] The edited models contain PK tests, and I've ran and passed them. - [X] I have checked for DRY opportunities with other models and docs. - [X] I've picked the right materialization for the affected models. # Other - [ ] Check if a full-refresh is required after this PR is merged. Related work items: #20123
This commit is contained in:
commit
78838cb7f6
3 changed files with 115 additions and 0 deletions
8
models/staging/edeposit/_edeposit_sources.yml
Normal file
8
models/staging/edeposit/_edeposit_sources.yml
Normal file
|
|
@ -0,0 +1,8 @@
|
||||||
|
version: 2
|
||||||
|
|
||||||
|
sources:
|
||||||
|
- name: edeposit
|
||||||
|
schema: sync_cdb_edeposit
|
||||||
|
tables:
|
||||||
|
- name: verifications
|
||||||
|
identifier: verifications
|
||||||
15
models/staging/edeposit/schema.yml
Normal file
15
models/staging/edeposit/schema.yml
Normal file
|
|
@ -0,0 +1,15 @@
|
||||||
|
version: 2
|
||||||
|
|
||||||
|
models:
|
||||||
|
- name: stg_edeposit__verifications
|
||||||
|
description:
|
||||||
|
"Records of each transaction that happens in the edeposit API. Records are
|
||||||
|
mutable and can get updated."
|
||||||
|
columns:
|
||||||
|
- name: id
|
||||||
|
data_type: character varying
|
||||||
|
description: "Unique id for the specific transaction."
|
||||||
|
tests:
|
||||||
|
- unique
|
||||||
|
- not_null
|
||||||
|
# Plenty of stuff pending. You cheeky Pablo
|
||||||
92
models/staging/edeposit/stg_edeposit__verifications.sql
Normal file
92
models/staging/edeposit/stg_edeposit__verifications.sql
Normal file
|
|
@ -0,0 +1,92 @@
|
||||||
|
with
|
||||||
|
raw_verifications as (select * from {{ source("edeposit", "verifications") }}),
|
||||||
|
deduped_verifications as (
|
||||||
|
select *
|
||||||
|
from
|
||||||
|
-- Some thoughts for the future here:
|
||||||
|
-- ··· The query below is awful performance wise, I know. But data
|
||||||
|
-- size is tiny today. Let's tackle the problem as it comes.
|
||||||
|
--
|
||||||
|
-- ··· The deduping logic below will be the same for all the Cosmos
|
||||||
|
-- DB entities that get brought into the DWH. The only changing
|
||||||
|
-- parameters will be what's the source table and the PK. I'm
|
||||||
|
-- not gonna do the macro now, but it would probably be a good
|
||||||
|
-- idea when we have a second container from Cosmos hitting the
|
||||||
|
-- DWH.
|
||||||
|
(
|
||||||
|
select
|
||||||
|
*,
|
||||||
|
row_number() over (
|
||||||
|
partition by {{ adapter.quote("documents") }} ->> 'id'
|
||||||
|
order by
|
||||||
|
({{ adapter.quote("documents") }} ->> '_ts')::integer desc
|
||||||
|
) as rank
|
||||||
|
from {{ source("edeposit", "verifications") }}
|
||||||
|
)
|
||||||
|
where rank = 1
|
||||||
|
),
|
||||||
|
stg_edeposit__verifications as (
|
||||||
|
select
|
||||||
|
{{ adapter.quote("documents") }} ->> 'id' as id,
|
||||||
|
{{ adapter.quote("documents") }} ->> 'BookingId' as id_booking,
|
||||||
|
{{ adapter.quote("documents") }} ->> 'userId' as id_user,
|
||||||
|
|
||||||
|
{{ adapter.quote("documents") }} ->> 'Status' as verification_status,
|
||||||
|
{{ adapter.quote("documents") }}
|
||||||
|
->> 'StatusReason' as verification_status_reason,
|
||||||
|
{{ adapter.quote("documents") }} ->> 'EmailFlag' as email_flag,
|
||||||
|
{{ adapter.quote("documents") }} ->> 'PhoneFlag' as phone_flag,
|
||||||
|
{{ adapter.quote("documents") }} ->> 'WatchList' as watch_list,
|
||||||
|
|
||||||
|
{{ adapter.quote("documents") }} ->> 'Channel' as channel,
|
||||||
|
|
||||||
|
({{ adapter.quote("documents") }} ->> 'CheckIn')::timestamp
|
||||||
|
as checkin_at_utc,
|
||||||
|
({{ adapter.quote("documents") }} ->> 'CheckOut')::timestamp
|
||||||
|
as checkout_at_utc,
|
||||||
|
({{ adapter.quote("documents") }} ->> 'Cancelled')::boolean as is_cancelled,
|
||||||
|
({{ adapter.quote("documents") }} ->> 'CancellationDate')::timestamp
|
||||||
|
as cancelled_at_utc,
|
||||||
|
|
||||||
|
{{ adapter.quote("documents") }} ->> 'UserEmail' as user_email,
|
||||||
|
{{ adapter.quote("documents") }} ->> 'GuestEmail' as guest_email,
|
||||||
|
{{ adapter.quote("documents") }} ->> 'GuestLastName' as guest_last_name,
|
||||||
|
{{ adapter.quote("documents") }} ->> 'GuestFirstName' as guest_first_name,
|
||||||
|
{{ adapter.quote("documents") }} ->> 'GuestTelephone' as guest_telephone,
|
||||||
|
|
||||||
|
{{ adapter.quote("documents") }}
|
||||||
|
->> 'PropertyManagerName' as property_manager_name,
|
||||||
|
{{ adapter.quote("documents") }}
|
||||||
|
->> 'PropertyManagerEmail' as property_manager_email,
|
||||||
|
{{ adapter.quote("documents") }} ->> 'ListingName' as listing_name,
|
||||||
|
{{ adapter.quote("documents") }} ->> 'ListingTown' as listing_town,
|
||||||
|
|
||||||
|
{{ adapter.quote("documents") }} ->> 'ListingAddress' as listing_address,
|
||||||
|
{{ adapter.quote("documents") }} ->> 'ListingCountry' as listing_country,
|
||||||
|
{{ adapter.quote("documents") }} ->> 'ListingPostcode' as listing_postcode,
|
||||||
|
({{ adapter.quote("documents") }} ->> 'PetsAllowed')::boolean
|
||||||
|
as pets_allowed,
|
||||||
|
|
||||||
|
({{ adapter.quote("documents") }} ->> 'LevelOfProtectionAmount')::float
|
||||||
|
::integer as level_of_protection_amount,
|
||||||
|
(
|
||||||
|
{{ adapter.quote("documents") }} ->> 'LevelOfProtectionCurrency'
|
||||||
|
) as level_of_protection_currency,
|
||||||
|
|
||||||
|
{{ adapter.quote("documents") }} ->> '_attachments' as attachments,
|
||||||
|
|
||||||
|
({{ adapter.quote("documents") }} ->> 'StatusUpdatedDate')::timestamp
|
||||||
|
as status_updated_at_utc,
|
||||||
|
({{ adapter.quote("documents") }} ->> 'UpdatedDate')::timestamp
|
||||||
|
as updated_at_utc,
|
||||||
|
({{ adapter.quote("documents") }} ->> 'CreationDate')::timestamp
|
||||||
|
as creation_at_utc,
|
||||||
|
({{ adapter.quote("documents") }} ->> 'CreatedDate')::timestamp
|
||||||
|
as created_at_utc,
|
||||||
|
to_timestamp(
|
||||||
|
(({{ adapter.quote("documents") }} ->> '_ts'))::integer
|
||||||
|
) as cosmos_db_timestamp_utc
|
||||||
|
from deduped_verifications
|
||||||
|
)
|
||||||
|
select *
|
||||||
|
from stg_edeposit__verifications
|
||||||
Loading…
Add table
Add a link
Reference in a new issue