Merged PR 1804: change core models to incremental where possible, set a 7 day safety window

change core models to incremental where possible, set a 7 day safety window

models/staging/core/stg_core__booking.sql

@@ -1,11 +1,13 @@
 {{
     config(
+        materialized="incremental",
+        unique_key="id_booking",
         indexes=[
             {"columns": ["id_booking"]},
             {"columns": ["id_user_host"]},
             {"columns": ["id_user_guest"]},
             {"columns": ["id_verification_request"]},
-        ]
+        ],
     )
 }}
 
@@ -41,6 +43,12 @@ with
             {{ adapter.quote("_airbyte_extracted_at") }} as dwh_extracted_at_utc
 
         from raw_booking
+
     )
 select *
 from stg_core__booking
+{% if is_incremental() %}
+    where
+        updated_at_utc
+        >= ((select max(updated_at_utc) from {{ this }}) - interval '7 days')
+{% endif %}

models/staging/core/stg_core__payment.sql

@@ -1,3 +1,5 @@
+{{ config(materialized="incremental", unique_key="id_payment") }}
+
 with
     raw_payment as (select * from {{ source("core", "Payment") }}),
     sgt_core__payment as (
@@ -23,3 +25,9 @@ with
     )
 select *
 from sgt_core__payment
+
+{% if is_incremental() %}
+    where
+        updated_at_utc
+        >= ((select max(updated_at_utc) from {{ this }}) - interval '7 days')
+{% endif %}

models/staging/core/stg_core__payment_validation_set.sql

@@ -1,3 +1,6 @@
+{{ config(materialized="incremental", unique_key="id_payment_validation_set") }}
+
+
 with
     raw_payment_validation_set as (
         select * from {{ source("core", "PaymentValidationSet") }}
@@ -20,3 +23,9 @@ with
     )
 select *
 from stg_core__payment_validation_set
+
+{% if is_incremental() %}
+    where
+        updated_at_utc
+        >= ((select max(updated_at_utc) from {{ this }}) - interval '7 days')
+{% endif %}

models/staging/core/stg_core__payment_validation_set_to_currency.sql

@@ -1,3 +1,9 @@
+{{
+    config(
+        materialized="incremental", unique_key="id_payment_validation_set_to_currency"
+    )
+}}
+
 with
     raw_payment_validation_set_to_currency as (
         select * from {{ source("core", "PaymentValidationSetToCurrency") }}
@@ -29,3 +35,9 @@ with
     )
 select *
 from stg_core__payment_validation_set_to_currency
+
+{% if is_incremental() %}
+    where
+        updated_at_utc
+        >= ((select max(updated_at_utc) from {{ this }}) - interval '7 days')
+{% endif %}

models/staging/core/stg_core__price_plan_to_user.sql

@@ -1,4 +1,10 @@
-{{ config(indexes=[{"columns": ["id_price_plan"]}, {"columns": ["id_user_host"]}]) }}
+{{
+    config(
+        materialized="incremental",
+        unique_key="id_price_plan",
+        indexes=[{"columns": ["id_price_plan"]}, {"columns": ["id_user_host"]}],
+    )
+}}
 
 with
     raw_price_plan_to_user as (select * from {{ source("core", "PricePlanToUser") }}),
@@ -29,3 +35,8 @@ with
     )
 select *
 from stg_core__price_plan_to_user
+{% if is_incremental() %}
+    where
+        updated_at_utc
+        >= ((select max(updated_at_utc) from {{ this }}) - interval '7 days')
+{% endif %}

models/staging/core/stg_core__superhog_user.sql

@@ -1,4 +1,10 @@
-{{ config(indexes=[{"columns": ["id_superhoguser"]}]) }}
+{{
+    config(
+        materialized="incremental",
+        unique_key="id_superhoguser",
+        indexes=[{"columns": ["id_superhoguser"]}],
+    )
+}}
 
 with
     raw_superhog_user as (select * from {{ source("core", "superhog_user") }}),
@@ -26,3 +32,8 @@ with
     )
 select *
 from stg_core__superhog_user
+{% if is_incremental() %}
+    where
+        updated_date_utc
+        >= ((select max(updated_date_utc) from {{ this }}) - interval '7 days')
+{% endif %}

models/staging/core/stg_core__verification.sql

@@ -1,3 +1,5 @@
+{{ config(materialized="incremental", unique_key="id_verification") }}
+
 with
     raw_verification as (select * from {{ source("core", "Verification") }}),
     stg_core__verification as (
@@ -22,3 +24,8 @@ with
     )
 select *
 from stg_core__verification
+{% if is_incremental() %}
+    where
+        updated_at_utc
+        >= ((select max(updated_at_utc) from {{ this }}) - interval '7 days')
+{% endif %}

models/staging/core/stg_core__verification_request.sql

@@ -1,10 +1,12 @@
 {{
     config(
+        materialized="incremental",
+        unique_key="id_verification_request",
         indexes=[
             {"columns": ["id_verification_request"]},
             {"columns": ["id_user_guest"]},
             {"columns": ["id_user_host"]},
-        ]
+        ],
     )
 }}
 
@@ -55,3 +57,8 @@ with
     )
 select *
 from stg_core__verification_request
+{% if is_incremental() %}
+    where
+        updated_at_utc
+        >= ((select max(updated_at_utc) from {{ this }}) - interval '7 days')
+{% endif %}

models/staging/core/stg_core__verification_set_to_verification_type.sql

@@ -1,3 +1,10 @@
+{{
+    config(
+        materialized="incremental",
+        unique_key="id_verification_set_to_verification_type",
+    )
+}}
+
 with
     raw_verificationsettoverificationtype as (
         select * from {{ source("core", "VerificationSetToVerificationType") }}
@@ -21,3 +28,8 @@ with
     )
 select *
 from stg_core__verification_set_to_verification_type
+{% if is_incremental() %}
+    where
+        updated_at_utc
+        >= ((select max(updated_at_utc) from {{ this }}) - interval '7 days')
+{% endif %}

models/staging/core/stg_core__verification_to_payment.sql

@@ -1,3 +1,5 @@
+{{ config(materialized="incremental", unique_key="id_verification_to_payment") }}
+
 with
     raw_verificationtopayment as (
         select * from {{ source("core", "VerificationToPayment") }}
@@ -25,3 +27,8 @@ with
     )
 select *
 from stg_core__verification_to_payment
+{% if is_incremental() %}
+    where
+        updated_at_utc
+        >= ((select max(updated_at_utc) from {{ this }}) - interval '7 days')
+{% endif %}

models/staging/core/stg_core__verification_type.sql

@@ -1,3 +1,6 @@
+{{ config(materialized="incremental", unique_key="id_verification_type") }}
+
+
 with
     raw_verification_type as (select * from {{ source("core", "VerificationType") }}),
     stg_core__verification_type as (
@@ -16,3 +19,8 @@ with
     )
 select *
 from stg_core__verification_type
+{% if is_incremental() %}
+    where
+        updated_at_utc
+        >= ((select max(updated_at_utc) from {{ this }}) - interval '7 days')
+{% endif %}