a few quick and dirty improvements

This commit is contained in:
Pablo Martin 2024-05-10 00:31:27 +02:00
parent a902abfa24
commit 824ff8da9e
7 changed files with 17 additions and 3 deletions

View file

@ -29,6 +29,7 @@ clean-targets: # directories to be removed by `dbt clean`
# directory as views. These settings can be overridden in the individual model
# files using the `{{ config(...) }}` macro.
models:
+unlogged: true
dwh_dbt:
staging:
+materialized: table

View file

@ -1,3 +1,4 @@
{{ config(materialized="table", unique_key="id_booking") }}
with
stg_core__booking as (select * from {{ ref("stg_core__booking") }}),
int_core__price_plans as (select * from {{ ref("int_core__price_plans") }}),

View file

@ -1,3 +1,4 @@
{{ config(materialized="table", unique_key="id_booking") }}
with
stg_core__booking as (select * from {{ ref("stg_core__booking") }}),
stg_core__booking_state as (select * from {{ ref("stg_core__booking_state") }}),

View file

@ -1,3 +1,4 @@
{{ config(materialized="table", unique_key="id_booking") }}
with
stg_core__booking_state as (select * from {{ ref("stg_core__booking_state") }}),
stg_core__booking_with_dup_flag as (

View file

@ -37,6 +37,7 @@ As for the general logic, so that the monster below makes some slight sense:
- Finally, whatever doesn't fall into Not Started or Action Required,
is considered Complete.
*/
{{ config(materialized="table", unique_key="id_verification_request") }}
with
stg_core__verification_request as (
select * from {{ ref("stg_core__verification_request") }}

View file

@ -1,3 +1,4 @@
{{ config(materialized="table", unique_key="id_verification_request") }}
with
stg_core__verification_request as (
select * from {{ ref("stg_core__verification_request") }}

View file

@ -14,6 +14,14 @@ source venv/bin/activate
echo "Triggering dbt seed" | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1
dbt seed | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1
# Run dbt
echo "Triggering dbt run" | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1
dbt run | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1
# Run staging layer
echo "Triggering dbt run: Staging" | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1
dbt run -s models/staging | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1
# Run intermediate layer
echo "Triggering dbt run: Intermediate" | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1
dbt run -s models/intermediate | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1
# Run reporting layer
echo "Triggering dbt run: Reporting" | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1
dbt run -s models/reporting | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1