From 67dcc8b237e0f71b85b9bbb4c2724deba80cd91b Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Fri, 4 Oct 2024 17:53:58 +0200 Subject: [PATCH] split staging layer --- .../athena/int_athena__verifications.sql | 7 +- models/staging/athena/_athena_sources.yml | 10 +++ .../athena/stg_athena__verifications.sql | 88 +++++++++++++++++++ 3 files changed, 100 insertions(+), 5 deletions(-) create mode 100644 models/staging/athena/_athena_sources.yml create mode 100644 models/staging/athena/stg_athena__verifications.sql diff --git a/models/intermediate/athena/int_athena__verifications.sql b/models/intermediate/athena/int_athena__verifications.sql index 5bc57b9..9177310 100644 --- a/models/intermediate/athena/int_athena__verifications.sql +++ b/models/intermediate/athena/int_athena__verifications.sql @@ -1,7 +1,4 @@ -with - stg_edeposit__verifications as ( - select * from {{ ref("stg_edeposit__verifications") }} - ) +with stg_athena__verifications as (select * from {{ ref("stg_athena__verifications") }}) select -- note that these ids are not the same as the ones found in Core DWH -- they are completely unrelated @@ -45,5 +42,5 @@ select athena_creation_date_utc, created_at_utc, created_date_utc -from stg_edeposit__verifications +from stg_athena__verifications where version = 'V1' diff --git a/models/staging/athena/_athena_sources.yml b/models/staging/athena/_athena_sources.yml new file mode 100644 index 0000000..9f42604 --- /dev/null +++ b/models/staging/athena/_athena_sources.yml @@ -0,0 +1,10 @@ +version: 2 + +sources: + - name: athena + # This will have to change to the Athena sync schema once the + # Athena/Edeposit database split happens. + schema: sync_cdb_edeposit + tables: + - name: verifications + identifier: verifications diff --git a/models/staging/athena/stg_athena__verifications.sql b/models/staging/athena/stg_athena__verifications.sql new file mode 100644 index 0000000..7e9734a --- /dev/null +++ b/models/staging/athena/stg_athena__verifications.sql @@ -0,0 +1,88 @@ +with + raw_verifications as (select * from {{ source("athena", "verifications") }}), + deduped_verifications as ( + {{ cosmos_db_record_deduplication("raw_verifications", "id") }} + ), + stg_athena__verifications as ( + select + {{ adapter.quote("documents") }} ->> 'id' as id_verification, + {{ adapter.quote("documents") }} ->> 'BookingId' as id_booking, + {{ adapter.quote("documents") }} ->> 'userId' as id_user_partner, + {{ adapter.quote("documents") }} ->> 'ListingId' as id_accommodation, + + {{ adapter.quote("documents") }} ->> 'Version' as "version", + + cast( + {{ adapter.quote("documents") }} ->> 'NightlyFee' as decimal(19, 4) + ) as "nightly_fee_local", + + {{ adapter.quote("documents") }} ->> 'Status' as verification_status, + {{ adapter.quote("documents") }} + ->> 'StatusReason' as verification_status_reason, + {{ adapter.quote("documents") }} ->> 'EmailFlag' as email_flag, + {{ adapter.quote("documents") }} ->> 'PhoneFlag' as phone_flag, + {{ adapter.quote("documents") }} ->> 'WatchList' as watch_list, + + {{ adapter.quote("documents") }} ->> 'Channel' as channel, + + ({{ adapter.quote("documents") }} ->> 'CheckIn')::timestamp + as checkin_at_utc, + ({{ adapter.quote("documents") }} ->> 'CheckOut')::timestamp + as checkout_at_utc, + coalesce( + ({{ adapter.quote("documents") }} ->> 'Cancelled')::boolean, false + ) as is_cancelled, + ({{ adapter.quote("documents") }} ->> 'CancellationDate')::timestamp + as cancelled_at_utc, + + {{ adapter.quote("documents") }} ->> 'UserEmail' as user_email, + {{ adapter.quote("documents") }} ->> 'GuestEmail' as guest_email, + {{ adapter.quote("documents") }} ->> 'GuestLastName' as guest_last_name, + {{ adapter.quote("documents") }} ->> 'GuestFirstName' as guest_first_name, + {{ adapter.quote("documents") }} ->> 'GuestTelephone' as guest_telephone, + + {{ adapter.quote("documents") }} ->> 'CompanyName' as company_name, + {{ adapter.quote("documents") }} + ->> 'PropertyManagerName' as property_manager_name, + {{ adapter.quote("documents") }} + ->> 'PropertyManagerEmail' as property_manager_email, + {{ adapter.quote("documents") }} ->> 'ListingName' as listing_name, + {{ adapter.quote("documents") }} ->> 'ListingTown' as listing_town, + + {{ adapter.quote("documents") }} ->> 'ListingAddress' as listing_address, + {{ adapter.quote("documents") }} ->> 'ListingCountry' as listing_country, + {{ adapter.quote("documents") }} ->> 'ListingPostcode' as listing_postcode, + ({{ adapter.quote("documents") }} ->> 'PetsAllowed')::boolean + as pets_allowed, + + ({{ adapter.quote("documents") }} ->> 'LevelOfProtectionAmount')::float + ::integer as level_of_protection_amount, + ( + {{ adapter.quote("documents") }} ->> 'LevelOfProtectionCurrency' + ) as level_of_protection_currency, + + {{ adapter.quote("documents") }} ->> '_attachments' as attachments, + + ({{ adapter.quote("documents") }} ->> 'StatusUpdatedDate')::timestamp + as status_updated_at_utc, + ({{ adapter.quote("documents") }} ->> 'StatusUpdatedDate')::date + as status_updated_date_utc, + ({{ adapter.quote("documents") }} ->> 'UpdatedDate')::timestamp + as updated_at_utc, + ({{ adapter.quote("documents") }} ->> 'UpdatedDate')::date + as updated_date_utc, + ({{ adapter.quote("documents") }} ->> 'CreationDate')::timestamp + as athena_creation_at_utc, + ({{ adapter.quote("documents") }} ->> 'CreationDate')::date + as athena_creation_date_utc, + ({{ adapter.quote("documents") }} ->> 'CreatedDate')::timestamp + as created_at_utc, + ({{ adapter.quote("documents") }} ->> 'CreatedDate')::date + as created_date_utc, + to_timestamp( + (({{ adapter.quote("documents") }} ->> '_ts'))::integer + ) as cosmos_db_timestamp_utc + from deduped_verifications + ) +select * +from stg_athena__verifications