Merged PR 2725: Force id user field to lower in staging

# Description Forces lower case to all id_users in staging. Removes hardcoded lower case in intermediate. Adapts readme to contemplate the lowering of id users. I propose to merge, run in prod and run tests in prod as a proper evaluation method. BTW, I only find one id_user_host that was in capital letters, so that's why probably we didn't care that much about this. Still, I prefer have things clean from the start! ``` select * from staging.stg_core__booking scb left join intermediate.int_core__unified_user icuu on lower(scb.id_user_host) = lower(icuu.id_user) where scb.id_user_host <> icuu.id_user ``` # Checklist - [ ] The edited models and dependants run properly with production data. **All models run in stg, did not check all the dependants** - [ ] The edited models are sufficiently documented. **Have not checked** - [ ] The edited models contain PK tests, and I've ran and passed them. - [X] I have checked for DRY opportunities with other models and docs. - [ ] I've picked the right materialization for the affected models. # Other - [ ] Check if a full-refresh is required after this PR is merged. Related work items: #20776
2024-09-03 14:36:21 +00:00 · 2024-09-03 14:36:21 +00:00 · 6d59e21310
commit 6d59e21310
parent 1b30fbbca9
18 changed files with 22 additions and 22 deletions
--- a/models/intermediate/core/int_core__host_booking_fees.sql
+++ b/models/intermediate/core/int_core__host_booking_fees.sql
@ -25,9 +25,7 @@ from stg_core__booking b
 left join int_core__booking_charge_events bce on b.id_booking = bce.id_booking
 left join stg_core__booking_state bs on b.id_booking_state = bs.id_booking_state
 left join int_core__duplicate_bookings db on b.id_booking = db.id_booking
-left join int_core__unified_user uu on lower(b.id_user_host) = lower(uu.id_user)
-- We user 'lower' because the id_user_host can be found in capital letters or not
-- depending on the table and Postgres is case sensitive
+left join int_core__unified_user uu on b.id_user_host = uu.id_user
 left join
    int_simple_exchange_rates ser
    on (
--- a/models/staging/core/stg_core__accommodation.sql
+++ b/models/staging/core/stg_core__accommodation.sql
@ -3,7 +3,7 @@ with
    stg_core__accommodation as (
        select
            {{ adapter.quote("AccommodationId") }} as id_accommodation,
-            {{ adapter.quote("CreatedByUserId") }} as id_creating_user,
+            lower({{ adapter.quote("CreatedByUserId") }}) as id_creating_user,
            {{ adapter.quote("PaymentValidationSetId") }} as id_payment_validation_set,
            {{ adapter.quote("FriendlyName") }} as friendly_name,
            {{ adapter.quote("CountryId") }} as id_country,
--- a/models/staging/core/stg_core__accommodation_to_user.sql
+++ b/models/staging/core/stg_core__accommodation_to_user.sql
@ -4,7 +4,7 @@ with
    ),
    stg_core__accommodation_to_user as (
        select
-            {{ adapter.quote("SuperhogUserId") }} as id_user_owner,
+            lower({{ adapter.quote("SuperhogUserId") }}) as id_user_owner,
            {{ adapter.quote("AccommodationId") }} as id_accommodation,
            {{ adapter.quote("AccommodationToUserId") }} as id_accommodation_to_user,
            {{ adapter.quote("AccommodationUserType") }} as accommodation_user_type,
--- a/models/staging/core/stg_core__booking.sql
+++ b/models/staging/core/stg_core__booking.sql
@ -16,8 +16,8 @@ with
    stg_core__booking as (
        select
            {{ adapter.quote("BookingId") }} as id_booking,
-            {{ adapter.quote("GuestUserId") }} as id_user_guest,
-            {{ adapter.quote("CreatedByUserId") }} as id_user_host,
+            lower({{ adapter.quote("GuestUserId") }}) as id_user_guest,
+            lower({{ adapter.quote("CreatedByUserId") }}) as id_user_host,
            {{ adapter.quote("IntegrationId") }} as id_integration,
            {{ adapter.quote("AccommodationId") }} as id_accommodation,
            {{ adapter.quote("BookingSourceId") }} as id_booking_source,
--- a/models/staging/core/stg_core__claim.sql
+++ b/models/staging/core/stg_core__claim.sql
@ -3,7 +3,7 @@ with
    stg_core__claim as (
        select
            {{ adapter.quote("Id") }} as id_claim,
-            {{ adapter.quote("UserId") }} as id_user,
+            lower({{ adapter.quote("UserId") }}) as id_user,
            {{ adapter.quote("ClaimType") }} as claim_type,
            {{ adapter.quote("ClaimValue") }} as claim_value,
            {{ adapter.quote("_airbyte_extracted_at") }} as dwh_extracted_at_utc
--- a/models/staging/core/stg_core__edeposit_user.sql
+++ b/models/staging/core/stg_core__edeposit_user.sql
@ -21,7 +21,7 @@ with
    stg_core__edeposit_user as (
        select
            {{ adapter.quote("Id") }} as id,
-            {{ adapter.quote("UserId") }} as id_user_partner,
+            lower({{ adapter.quote("UserId") }}) as id_user_partner,
            upper(substring({{ adapter.quote("Currency") }} from 1 for 3)) as currency,
            -- this shitty thing right here is all because the currency is currently
            -- set up as a free text.
--- a/models/staging/core/stg_core__integration.sql
+++ b/models/staging/core/stg_core__integration.sql
@ -4,7 +4,7 @@ with
        select
            {{ adapter.quote("Id") }} as id_integration,
            {{ adapter.quote("Identifier") }} as uuid_integration,
-            {{ adapter.quote("SuperhogUserId") }} as id_superhog_user,
+            lower({{ adapter.quote("SuperhogUserId") }}) as id_superhog_user,
            {{ adapter.quote("IntegrationTypeId") }} id_integration_type,
            {{ adapter.quote("IsActive") }} as is_active,
            {{ adapter.quote("LoggingIsActive") }} as is_logging_active,
--- a/models/staging/core/stg_core__payment_validation_set.sql
+++ b/models/staging/core/stg_core__payment_validation_set.sql
@ -8,7 +8,7 @@ with
    stg_core__payment_validation_set as (
        select
            {{ adapter.quote("Id") }} as id_payment_validation_set,
-            {{ adapter.quote("SuperhogUserId") }} as id_guest_host,
+            lower({{ adapter.quote("SuperhogUserId") }}) as id_guest_host,
            {{ adapter.quote("Name") }} as payment_validation_set_name,
            {{ adapter.quote("Version") }} as payment_validation_set_version,
            {{ adapter.quote("IsActive") }} as is_active,
--- a/models/staging/core/stg_core__price_plan_to_user.sql
+++ b/models/staging/core/stg_core__price_plan_to_user.sql
@ -11,7 +11,7 @@ with
    stg_core__price_plan_to_user as (
        select
            {{ adapter.quote("Id") }} as id_price_plan,
-            {{ adapter.quote("SuperhogUserId") }} as id_user_host,
+            lower({{ adapter.quote("SuperhogUserId") }}) as id_user_host,
            {{ adapter.quote("PricePlanChargedByTypeId") }}
            as id_price_plan_charged_by_type,
            {{ adapter.quote("StartDate") }} as start_at_utc,
--- a/models/staging/core/stg_core__superhog_user.sql
+++ b/models/staging/core/stg_core__superhog_user.sql
@ -18,7 +18,7 @@ with
            {{ adapter.quote("VerifiedDate") }} as verified_at_utc,
            cast({{ adapter.quote("VerifiedDate") }} as date) as verified_date_utc,
            {{ adapter.quote("FlagAsProblem") }} as flag_as_problem,
-            {{ adapter.quote("SuperhogUserId") }} as id_superhoguser,
+            lower({{ adapter.quote("SuperhogUserId") }}) as id_superhoguser,
            {{ adapter.quote("NumberOfProperties") }} as number_of_properties,
            {{ adapter.quote("SuperhogVerifiedSetId") }} id_superhog_verified_set,
            {{ adapter.quote("PlatformCommsRecipient") }} as platform_comms_recipient,
--- a/models/staging/core/stg_core__user.sql
+++ b/models/staging/core/stg_core__user.sql
@ -6,7 +6,7 @@ with
        select
            {{ adapter.quote("Id") }} as id_user,
            {{ adapter.quote("AccountTypeId") }} as id_account_type,
-            {{ adapter.quote("CreatedUserId") }} as id_created_user,
+            lower({{ adapter.quote("CreatedUserId") }}) as id_created_user,
            {{ adapter.quote("BillingCountryId") }} as id_billing_country,
            {{ adapter.quote("AccountCurrencyId") }} as id_account_currency,
            {{ adapter.quote("Code") }} as user_code,
--- a/models/staging/core/stg_core__user_product_bundle.sql
+++ b/models/staging/core/stg_core__user_product_bundle.sql
@ -5,7 +5,7 @@ with
    stg_core__user_product_bundle as (
        select
            {{ adapter.quote("Id") }} as id_user_product_bundle,
-            {{ adapter.quote("SuperhogUserId") }} as id_user,
+            lower({{ adapter.quote("SuperhogUserId") }}) as id_user,
            {{ adapter.quote("ProductBundleId") }} as id_product_bundle,
            {{ adapter.quote("ProtectionPlanId") }} as id_protection_plan,

--- a/models/staging/core/stg_core__user_role.sql
+++ b/models/staging/core/stg_core__user_role.sql
@ -3,7 +3,7 @@ with source as (
 ),
 stg_core__user_role as (
    select
-        {{ adapter.quote("UserId") }} as id_user,
+        lower({{ adapter.quote("UserId") }}) as id_user,
        {{ adapter.quote("RoleId") }} as id_role,
        {{ adapter.quote("_airbyte_extracted_at") }} as dwh_extracted_at_utc
    from source
--- a/models/staging/core/stg_core__verification.sql
+++ b/models/staging/core/stg_core__verification.sql
@ -5,7 +5,7 @@ with
    stg_core__verification as (
        select
            {{ adapter.quote("Id") }} as id_verification,
-            {{ adapter.quote("SuperhogUserId") }} as id_user_guest,
+            lower({{ adapter.quote("SuperhogUserId") }}) as id_user_guest,
            {{ adapter.quote("VerificationStatusId") }} as id_verification_status,
            {{ adapter.quote("VerificationRequestId") }} as id_verification_request,
            {{ adapter.quote("Name") }} as verification,
--- a/models/staging/core/stg_core__verification_request.sql
+++ b/models/staging/core/stg_core__verification_request.sql
@ -22,8 +22,8 @@ with
            {{ adapter.quote("VerificationSetId") }} as id_verification_set,
            {{ adapter.quote("SuperhogVerifiedSetId") }} as id_superhog_verified_set,
            {{ adapter.quote("PaymentValidationSetId") }} as id_payment_validation_set,
-            {{ adapter.quote("SuperhogUserId") }} as id_user_guest,
-            {{ adapter.quote("CreatedByUserId") }} as id_user_host,
+            lower({{ adapter.quote("SuperhogUserId") }}) as id_user_guest,
+            lower({{ adapter.quote("CreatedByUserId") }}) as id_user_host,
            {{ adapter.quote("Url") }} as verification_url,
            {{ adapter.quote("CallbackUrl") }} as callback_url,
            {{ adapter.quote("RedirectUrl") }} as redirect_url,
--- a/models/staging/core/stg_core__verification_set.sql
+++ b/models/staging/core/stg_core__verification_set.sql
@ -3,7 +3,7 @@ with
    stg_core__verification_set as (
        select
            {{ adapter.quote("Id") }} as id_verification_set,
-            {{ adapter.quote("SuperhogUserId") }} as id_user_host,
+            lower({{ adapter.quote("SuperhogUserId") }}) as id_user_host,
            {{ adapter.quote("Version") }} as verification_set_version,
            {{ adapter.quote("AutoFill") }} as autofill,
            {{ adapter.quote("IsActive") }} as is_active,
--- a/models/staging/core/stg_core__verification_to_payment.sql
+++ b/models/staging/core/stg_core__verification_to_payment.sql
@ -17,7 +17,7 @@ with
            cast({{ adapter.quote("RefundDueDate") }} as date) as refund_due_date_utc,
            {{ adapter.quote("PaymentDueDate") }} as payment_due_at_utc,
            cast({{ adapter.quote("PaymentDueDate") }} as date) as payment_due_date_utc,
-            {{ adapter.quote("SuperhogUserId") }} as id_guest_user,
+            lower({{ adapter.quote("SuperhogUserId") }}) as id_guest_user,
            {{ adapter.quote("VerificationId") }} as id_verification,
            {{ adapter.quote("VerificationPaymentTypeId") }}
            as id_verification_payment_type,