diff --git a/README.md b/README.md index b411b6d..e309062 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,7 @@ For other matters, use a `chores` branch (i.e. `chores/add-dbt-package`). ## Project organization -We organize models in four folders: +We organize models in three folders: - `staging` - Pretty much this: @@ -89,6 +89,8 @@ We organize models in four folders: - Split schema and domain with double underscode (ie `stg_core__booking`). - Always use sources to read into staging models. - SQL formatting should be done with `sqlfmt`. +- Other conventions + - In staging, enforce a `lower()` to user UUID fields to avoid nasty propagations in the DWH. When in doubt, do what dbt guys would do: Or Gitlab: diff --git a/models/intermediate/core/int_core__host_booking_fees.sql b/models/intermediate/core/int_core__host_booking_fees.sql index 246c904..5844528 100644 --- a/models/intermediate/core/int_core__host_booking_fees.sql +++ b/models/intermediate/core/int_core__host_booking_fees.sql @@ -25,9 +25,7 @@ from stg_core__booking b left join int_core__booking_charge_events bce on b.id_booking = bce.id_booking left join stg_core__booking_state bs on b.id_booking_state = bs.id_booking_state left join int_core__duplicate_bookings db on b.id_booking = db.id_booking -left join int_core__unified_user uu on lower(b.id_user_host) = lower(uu.id_user) --- We user 'lower' because the id_user_host can be found in capital letters or not --- depending on the table and Postgres is case sensitive +left join int_core__unified_user uu on b.id_user_host = uu.id_user left join int_simple_exchange_rates ser on ( diff --git a/models/staging/core/stg_core__accommodation.sql b/models/staging/core/stg_core__accommodation.sql index b28eedb..fa35a9a 100644 --- a/models/staging/core/stg_core__accommodation.sql +++ b/models/staging/core/stg_core__accommodation.sql @@ -3,7 +3,7 @@ with stg_core__accommodation as ( select {{ adapter.quote("AccommodationId") }} as id_accommodation, - {{ adapter.quote("CreatedByUserId") }} as id_creating_user, + lower({{ adapter.quote("CreatedByUserId") }}) as id_creating_user, {{ adapter.quote("PaymentValidationSetId") }} as id_payment_validation_set, {{ adapter.quote("FriendlyName") }} as friendly_name, {{ adapter.quote("CountryId") }} as id_country, diff --git a/models/staging/core/stg_core__accommodation_to_user.sql b/models/staging/core/stg_core__accommodation_to_user.sql index 7841d02..37ddf28 100644 --- a/models/staging/core/stg_core__accommodation_to_user.sql +++ b/models/staging/core/stg_core__accommodation_to_user.sql @@ -4,7 +4,7 @@ with ), stg_core__accommodation_to_user as ( select - {{ adapter.quote("SuperhogUserId") }} as id_user_owner, + lower({{ adapter.quote("SuperhogUserId") }}) as id_user_owner, {{ adapter.quote("AccommodationId") }} as id_accommodation, {{ adapter.quote("AccommodationToUserId") }} as id_accommodation_to_user, {{ adapter.quote("AccommodationUserType") }} as accommodation_user_type, diff --git a/models/staging/core/stg_core__booking.sql b/models/staging/core/stg_core__booking.sql index 604038b..cd3cd71 100644 --- a/models/staging/core/stg_core__booking.sql +++ b/models/staging/core/stg_core__booking.sql @@ -16,8 +16,8 @@ with stg_core__booking as ( select {{ adapter.quote("BookingId") }} as id_booking, - {{ adapter.quote("GuestUserId") }} as id_user_guest, - {{ adapter.quote("CreatedByUserId") }} as id_user_host, + lower({{ adapter.quote("GuestUserId") }}) as id_user_guest, + lower({{ adapter.quote("CreatedByUserId") }}) as id_user_host, {{ adapter.quote("IntegrationId") }} as id_integration, {{ adapter.quote("AccommodationId") }} as id_accommodation, {{ adapter.quote("BookingSourceId") }} as id_booking_source, diff --git a/models/staging/core/stg_core__claim.sql b/models/staging/core/stg_core__claim.sql index f76665f..a9c6de3 100644 --- a/models/staging/core/stg_core__claim.sql +++ b/models/staging/core/stg_core__claim.sql @@ -3,7 +3,7 @@ with stg_core__claim as ( select {{ adapter.quote("Id") }} as id_claim, - {{ adapter.quote("UserId") }} as id_user, + lower({{ adapter.quote("UserId") }}) as id_user, {{ adapter.quote("ClaimType") }} as claim_type, {{ adapter.quote("ClaimValue") }} as claim_value, {{ adapter.quote("_airbyte_extracted_at") }} as dwh_extracted_at_utc diff --git a/models/staging/core/stg_core__edeposit_user.sql b/models/staging/core/stg_core__edeposit_user.sql index 9962477..d0514fa 100644 --- a/models/staging/core/stg_core__edeposit_user.sql +++ b/models/staging/core/stg_core__edeposit_user.sql @@ -21,7 +21,7 @@ with stg_core__edeposit_user as ( select {{ adapter.quote("Id") }} as id, - {{ adapter.quote("UserId") }} as id_user_partner, + lower({{ adapter.quote("UserId") }}) as id_user_partner, upper(substring({{ adapter.quote("Currency") }} from 1 for 3)) as currency, -- this shitty thing right here is all because the currency is currently -- set up as a free text. diff --git a/models/staging/core/stg_core__integration.sql b/models/staging/core/stg_core__integration.sql index 17d5f3e..7097489 100644 --- a/models/staging/core/stg_core__integration.sql +++ b/models/staging/core/stg_core__integration.sql @@ -4,7 +4,7 @@ with select {{ adapter.quote("Id") }} as id_integration, {{ adapter.quote("Identifier") }} as uuid_integration, - {{ adapter.quote("SuperhogUserId") }} as id_superhog_user, + lower({{ adapter.quote("SuperhogUserId") }}) as id_superhog_user, {{ adapter.quote("IntegrationTypeId") }} id_integration_type, {{ adapter.quote("IsActive") }} as is_active, {{ adapter.quote("LoggingIsActive") }} as is_logging_active, diff --git a/models/staging/core/stg_core__payment_validation_set.sql b/models/staging/core/stg_core__payment_validation_set.sql index ec0befc..1ee3cac 100644 --- a/models/staging/core/stg_core__payment_validation_set.sql +++ b/models/staging/core/stg_core__payment_validation_set.sql @@ -8,7 +8,7 @@ with stg_core__payment_validation_set as ( select {{ adapter.quote("Id") }} as id_payment_validation_set, - {{ adapter.quote("SuperhogUserId") }} as id_guest_host, + lower({{ adapter.quote("SuperhogUserId") }}) as id_guest_host, {{ adapter.quote("Name") }} as payment_validation_set_name, {{ adapter.quote("Version") }} as payment_validation_set_version, {{ adapter.quote("IsActive") }} as is_active, diff --git a/models/staging/core/stg_core__price_plan_to_user.sql b/models/staging/core/stg_core__price_plan_to_user.sql index f7cfa29..b488f93 100644 --- a/models/staging/core/stg_core__price_plan_to_user.sql +++ b/models/staging/core/stg_core__price_plan_to_user.sql @@ -11,7 +11,7 @@ with stg_core__price_plan_to_user as ( select {{ adapter.quote("Id") }} as id_price_plan, - {{ adapter.quote("SuperhogUserId") }} as id_user_host, + lower({{ adapter.quote("SuperhogUserId") }}) as id_user_host, {{ adapter.quote("PricePlanChargedByTypeId") }} as id_price_plan_charged_by_type, {{ adapter.quote("StartDate") }} as start_at_utc, diff --git a/models/staging/core/stg_core__superhog_user.sql b/models/staging/core/stg_core__superhog_user.sql index 5f3468b..4a1d8c3 100644 --- a/models/staging/core/stg_core__superhog_user.sql +++ b/models/staging/core/stg_core__superhog_user.sql @@ -18,7 +18,7 @@ with {{ adapter.quote("VerifiedDate") }} as verified_at_utc, cast({{ adapter.quote("VerifiedDate") }} as date) as verified_date_utc, {{ adapter.quote("FlagAsProblem") }} as flag_as_problem, - {{ adapter.quote("SuperhogUserId") }} as id_superhoguser, + lower({{ adapter.quote("SuperhogUserId") }}) as id_superhoguser, {{ adapter.quote("NumberOfProperties") }} as number_of_properties, {{ adapter.quote("SuperhogVerifiedSetId") }} id_superhog_verified_set, {{ adapter.quote("PlatformCommsRecipient") }} as platform_comms_recipient, diff --git a/models/staging/core/stg_core__user.sql b/models/staging/core/stg_core__user.sql index 9e72d2d..56a06cf 100644 --- a/models/staging/core/stg_core__user.sql +++ b/models/staging/core/stg_core__user.sql @@ -6,7 +6,7 @@ with select {{ adapter.quote("Id") }} as id_user, {{ adapter.quote("AccountTypeId") }} as id_account_type, - {{ adapter.quote("CreatedUserId") }} as id_created_user, + lower({{ adapter.quote("CreatedUserId") }}) as id_created_user, {{ adapter.quote("BillingCountryId") }} as id_billing_country, {{ adapter.quote("AccountCurrencyId") }} as id_account_currency, {{ adapter.quote("Code") }} as user_code, diff --git a/models/staging/core/stg_core__user_product_bundle.sql b/models/staging/core/stg_core__user_product_bundle.sql index 42fed1e..4c949f7 100644 --- a/models/staging/core/stg_core__user_product_bundle.sql +++ b/models/staging/core/stg_core__user_product_bundle.sql @@ -5,7 +5,7 @@ with stg_core__user_product_bundle as ( select {{ adapter.quote("Id") }} as id_user_product_bundle, - {{ adapter.quote("SuperhogUserId") }} as id_user, + lower({{ adapter.quote("SuperhogUserId") }}) as id_user, {{ adapter.quote("ProductBundleId") }} as id_product_bundle, {{ adapter.quote("ProtectionPlanId") }} as id_protection_plan, diff --git a/models/staging/core/stg_core__user_role.sql b/models/staging/core/stg_core__user_role.sql index 97d9aec..0f8fbfd 100644 --- a/models/staging/core/stg_core__user_role.sql +++ b/models/staging/core/stg_core__user_role.sql @@ -3,7 +3,7 @@ with source as ( ), stg_core__user_role as ( select - {{ adapter.quote("UserId") }} as id_user, + lower({{ adapter.quote("UserId") }}) as id_user, {{ adapter.quote("RoleId") }} as id_role, {{ adapter.quote("_airbyte_extracted_at") }} as dwh_extracted_at_utc from source diff --git a/models/staging/core/stg_core__verification.sql b/models/staging/core/stg_core__verification.sql index 5fd8b9c..a8ffbca 100644 --- a/models/staging/core/stg_core__verification.sql +++ b/models/staging/core/stg_core__verification.sql @@ -5,7 +5,7 @@ with stg_core__verification as ( select {{ adapter.quote("Id") }} as id_verification, - {{ adapter.quote("SuperhogUserId") }} as id_user_guest, + lower({{ adapter.quote("SuperhogUserId") }}) as id_user_guest, {{ adapter.quote("VerificationStatusId") }} as id_verification_status, {{ adapter.quote("VerificationRequestId") }} as id_verification_request, {{ adapter.quote("Name") }} as verification, diff --git a/models/staging/core/stg_core__verification_request.sql b/models/staging/core/stg_core__verification_request.sql index d0101a8..f354cf8 100644 --- a/models/staging/core/stg_core__verification_request.sql +++ b/models/staging/core/stg_core__verification_request.sql @@ -22,8 +22,8 @@ with {{ adapter.quote("VerificationSetId") }} as id_verification_set, {{ adapter.quote("SuperhogVerifiedSetId") }} as id_superhog_verified_set, {{ adapter.quote("PaymentValidationSetId") }} as id_payment_validation_set, - {{ adapter.quote("SuperhogUserId") }} as id_user_guest, - {{ adapter.quote("CreatedByUserId") }} as id_user_host, + lower({{ adapter.quote("SuperhogUserId") }}) as id_user_guest, + lower({{ adapter.quote("CreatedByUserId") }}) as id_user_host, {{ adapter.quote("Url") }} as verification_url, {{ adapter.quote("CallbackUrl") }} as callback_url, {{ adapter.quote("RedirectUrl") }} as redirect_url, diff --git a/models/staging/core/stg_core__verification_set.sql b/models/staging/core/stg_core__verification_set.sql index 1f792d9..4e9159f 100644 --- a/models/staging/core/stg_core__verification_set.sql +++ b/models/staging/core/stg_core__verification_set.sql @@ -3,7 +3,7 @@ with stg_core__verification_set as ( select {{ adapter.quote("Id") }} as id_verification_set, - {{ adapter.quote("SuperhogUserId") }} as id_user_host, + lower({{ adapter.quote("SuperhogUserId") }}) as id_user_host, {{ adapter.quote("Version") }} as verification_set_version, {{ adapter.quote("AutoFill") }} as autofill, {{ adapter.quote("IsActive") }} as is_active, diff --git a/models/staging/core/stg_core__verification_to_payment.sql b/models/staging/core/stg_core__verification_to_payment.sql index fe16b30..be69c92 100644 --- a/models/staging/core/stg_core__verification_to_payment.sql +++ b/models/staging/core/stg_core__verification_to_payment.sql @@ -17,7 +17,7 @@ with cast({{ adapter.quote("RefundDueDate") }} as date) as refund_due_date_utc, {{ adapter.quote("PaymentDueDate") }} as payment_due_at_utc, cast({{ adapter.quote("PaymentDueDate") }} as date) as payment_due_date_utc, - {{ adapter.quote("SuperhogUserId") }} as id_guest_user, + lower({{ adapter.quote("SuperhogUserId") }}) as id_guest_user, {{ adapter.quote("VerificationId") }} as id_verification, {{ adapter.quote("VerificationPaymentTypeId") }} as id_verification_payment_type,