From 8666b88953b350ff565c218be1a103b2ebfed16b Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Fri, 21 Mar 2025 16:53:22 +0100 Subject: [PATCH 01/14] add run step --- ci/.azure-pipelines.pr.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ci/.azure-pipelines.pr.yml b/ci/.azure-pipelines.pr.yml index 3502fb4..0b16057 100644 --- a/ci/.azure-pipelines.pr.yml +++ b/ci/.azure-pipelines.pr.yml @@ -78,6 +78,14 @@ steps: displayName: 'Validate models compiled SQL' +- script: | + source venv/bin/activate + set -a && source .env && set +a + + dbt run --select "state:modified" --state ~/dbt-ci/master-artifacts/ --defer + + displayName: 'Run modified models' + - script: | set -a && source .env && set +a From 8938545d3a0b3650e50555abd99dd93e7d50c387 Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Fri, 21 Mar 2025 16:58:46 +0100 Subject: [PATCH 02/14] add silly change to test ci --- .../intermediate/athena/int_athena__verifications_with_fees.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/models/intermediate/athena/int_athena__verifications_with_fees.sql b/models/intermediate/athena/int_athena__verifications_with_fees.sql index e770aaf..bcf06f3 100644 --- a/models/intermediate/athena/int_athena__verifications_with_fees.sql +++ b/models/intermediate/athena/int_athena__verifications_with_fees.sql @@ -15,6 +15,7 @@ with where v.version = 'V1' and v.id_booking is not null ) select + 1 as lol, v.id_verification, v.id_booking, v.verification_status, From c59600d9f25c622d3d83c222129766b35bd73a92 Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Mon, 24 Mar 2025 11:34:13 +0100 Subject: [PATCH 03/14] another silly change to trigger build --- .../intermediate/athena/int_athena__verifications_with_fees.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/models/intermediate/athena/int_athena__verifications_with_fees.sql b/models/intermediate/athena/int_athena__verifications_with_fees.sql index bcf06f3..cf85e05 100644 --- a/models/intermediate/athena/int_athena__verifications_with_fees.sql +++ b/models/intermediate/athena/int_athena__verifications_with_fees.sql @@ -16,6 +16,7 @@ with ) select 1 as lol, + 2 as lolol, v.id_verification, v.id_booking, v.verification_status, From f2e447ba60e4ed4d60eaa7e9d5160485c1bd2bb7 Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Mon, 24 Mar 2025 11:55:53 +0100 Subject: [PATCH 04/14] running on pointer database --- ci/.azure-pipelines.pr.yml | 25 +++++++------------------ macros/generate_schema_name.sql | 14 ++++++++++---- 2 files changed, 17 insertions(+), 22 deletions(-) diff --git a/ci/.azure-pipelines.pr.yml b/ci/.azure-pipelines.pr.yml index 0b16057..5e10878 100644 --- a/ci/.azure-pipelines.pr.yml +++ b/ci/.azure-pipelines.pr.yml @@ -12,15 +12,15 @@ steps: cp ~/dbt-ci/.env . # Check if it matches the refs/pull//merge pattern - if [[ $BUILD_SOURCEBRANCH =~ ^refs/pull/([0-9]+)/merge$ ]]; then - POSTGRES_DATABASE="ci_pr_${BASH_REMATCH[1]}" - else - POSTGRES_DATABASE="ci_${BUILD_SOURCEBRANCH#refs/heads/}" - fi + # if [[ $BUILD_SOURCEBRANCH =~ ^refs/pull/([0-9]+)/merge$ ]]; then + # POSTGRES_DATABASE="ci_pr_${BASH_REMATCH[1]}" + # else + # POSTGRES_DATABASE="ci_${BUILD_SOURCEBRANCH#refs/heads/}" + # fi echo "POSTGRES_DATABASE: $POSTGRES_DATABASE" - echo "POSTGRES_DATABASE=${POSTGRES_DATABASE}" >> .env + echo "POSTGRES_DATABASE=prd-pointer}" >> .env set -a && source .env && set +a displayName: 'Load env' @@ -30,12 +30,6 @@ steps: psql -h $POSTGRES_HOST -U $POSTGRES_USER -d postgres -c "SELECT 1 FROM pg_database WHERE datname = 'postgres';" displayName: 'Smoke test Postgres connection' -- script: | - set -a && source .env && set +a - - psql -h $POSTGRES_HOST -U $POSTGRES_USER -d postgres -c "CREATE DATABASE ${POSTGRES_DATABASE}" - displayName: 'Create branch dedicated database' - - script: | cd ~/dbt-ci /bin/bash build-master-artifacts.sh @@ -80,16 +74,11 @@ steps: - script: | source venv/bin/activate + echo "CI_SCHEMA_NAME=HARDCODED_SCHEMA" >> .env set -a && source .env && set +a dbt run --select "state:modified" --state ~/dbt-ci/master-artifacts/ --defer displayName: 'Run modified models' -- script: | - set -a && source .env && set +a - - psql -h $POSTGRES_HOST -U $POSTGRES_USER -d postgres -c "DROP DATABASE ${POSTGRES_DATABASE}" - condition: always() - displayName: 'Delete branch dedicated database' \ No newline at end of file diff --git a/macros/generate_schema_name.sql b/macros/generate_schema_name.sql index c32da69..26de2c8 100644 --- a/macros/generate_schema_name.sql +++ b/macros/generate_schema_name.sql @@ -8,11 +8,17 @@ To understand better, check: */ {% macro generate_schema_name(custom_schema_name, node) -%} - {%- set default_schema = target.schema -%} - {%- if custom_schema_name is none -%} {{ default_schema }} + {% macro generate_schema_name(custom_schema_name, node) -%} - {%- else -%} {{ custom_schema_name | trim }} + {%- set default_schema = target.schema -%} + {%- set is_ci_execution = env_var("IS_CI_EXECUTION", "0") | int -%} + {%- set ci_schema_name = env_var("CI_SCHEMA_NAME", "ci_default") -%} - {%- endif -%} + {%- if is_ci_execution == 1 -%} {{ ci_schema_name }} + {%- elif custom_schema_name is none -%} {{ default_schema }} + {%- else -%} {{ custom_schema_name | trim }} + {%- endif -%} + + {%- endmacro %} {%- endmacro %} From 75b85073ae768fa7c4c6023f2fa996f1b566a3cc Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Mon, 24 Mar 2025 11:58:15 +0100 Subject: [PATCH 05/14] typo in macro --- macros/generate_schema_name.sql | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/macros/generate_schema_name.sql b/macros/generate_schema_name.sql index 26de2c8..fb86fed 100644 --- a/macros/generate_schema_name.sql +++ b/macros/generate_schema_name.sql @@ -8,17 +8,13 @@ To understand better, check: */ {% macro generate_schema_name(custom_schema_name, node) -%} - {% macro generate_schema_name(custom_schema_name, node) -%} + {%- set default_schema = target.schema -%} + {%- set is_ci_execution = env_var("IS_CI_EXECUTION", "0") | int -%} + {%- set ci_schema_name = env_var("CI_SCHEMA_NAME", "ci_default") -%} - {%- set default_schema = target.schema -%} - {%- set is_ci_execution = env_var("IS_CI_EXECUTION", "0") | int -%} - {%- set ci_schema_name = env_var("CI_SCHEMA_NAME", "ci_default") -%} - - {%- if is_ci_execution == 1 -%} {{ ci_schema_name }} - {%- elif custom_schema_name is none -%} {{ default_schema }} - {%- else -%} {{ custom_schema_name | trim }} - {%- endif -%} - - {%- endmacro %} + {%- if is_ci_execution == 1 -%} {{ ci_schema_name }} + {%- elif custom_schema_name is none -%} {{ default_schema }} + {%- else -%} {{ custom_schema_name | trim }} + {%- endif -%} {%- endmacro %} From 8a4ec6c25768f80bb2d44418b1181ead7bb2dae6 Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Mon, 24 Mar 2025 12:01:53 +0100 Subject: [PATCH 06/14] typo --- ci/.azure-pipelines.pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/.azure-pipelines.pr.yml b/ci/.azure-pipelines.pr.yml index 5e10878..fe281d6 100644 --- a/ci/.azure-pipelines.pr.yml +++ b/ci/.azure-pipelines.pr.yml @@ -20,7 +20,7 @@ steps: echo "POSTGRES_DATABASE: $POSTGRES_DATABASE" - echo "POSTGRES_DATABASE=prd-pointer}" >> .env + echo "POSTGRES_DATABASE=prd-pointer" >> .env set -a && source .env && set +a displayName: 'Load env' From 73f40ff1a50d9020fe94d09eebe7ff45b97df7c3 Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Mon, 24 Mar 2025 12:11:23 +0100 Subject: [PATCH 07/14] continue --- ci/.azure-pipelines.pr.yml | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/ci/.azure-pipelines.pr.yml b/ci/.azure-pipelines.pr.yml index fe281d6..e3c1578 100644 --- a/ci/.azure-pipelines.pr.yml +++ b/ci/.azure-pipelines.pr.yml @@ -72,6 +72,12 @@ steps: displayName: 'Validate models compiled SQL' +- script: | + set -a && source .env && set +a + + psql -h $POSTGRES_HOST -U $POSTGRES_USER -d prd-pointer -c "DROP SCHEMA IF EXISTS HARDCODED_SCHEMA CASCADE;" + displayName: "Preemptive DROP SCHEMA" + - script: | source venv/bin/activate echo "CI_SCHEMA_NAME=HARDCODED_SCHEMA" >> .env @@ -81,4 +87,18 @@ steps: displayName: 'Run modified models' - \ No newline at end of file +- script: | + source venv/bin/activate + echo "CI_SCHEMA_NAME=HARDCODED_SCHEMA" >> .env + set -a && source .env && set +a + + dbt test --select "state:modified" --state ~/dbt-ci/master-artifacts/ --defer + + displayName: 'Test modified models' + +- script: | + set -a && source .env && set +a + psql -h $POSTGRES_HOST -U $POSTGRES_USER -d prd-pointer -c "DROP SCHEMA IF EXISTS HARDCODED_SCHEMA CASCADE;" + + condition: always() + displayName: 'Delete PR schema' \ No newline at end of file From f6c10430388d7abe6224e01f6553ffea7f3f308f Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Mon, 24 Mar 2025 12:16:59 +0100 Subject: [PATCH 08/14] fix identation --- ci/.azure-pipelines.pr.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/.azure-pipelines.pr.yml b/ci/.azure-pipelines.pr.yml index e3c1578..3283b40 100644 --- a/ci/.azure-pipelines.pr.yml +++ b/ci/.azure-pipelines.pr.yml @@ -76,7 +76,8 @@ steps: set -a && source .env && set +a psql -h $POSTGRES_HOST -U $POSTGRES_USER -d prd-pointer -c "DROP SCHEMA IF EXISTS HARDCODED_SCHEMA CASCADE;" - displayName: "Preemptive DROP SCHEMA" + + displayName: "Preemptive DROP SCHEMA" - script: | source venv/bin/activate From e311431fbb7624b9a9aa0a8772e5ef1ed198e691 Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Mon, 24 Mar 2025 12:22:43 +0100 Subject: [PATCH 09/14] also run downstream --- ci/.azure-pipelines.pr.yml | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/ci/.azure-pipelines.pr.yml b/ci/.azure-pipelines.pr.yml index 3283b40..a382628 100644 --- a/ci/.azure-pipelines.pr.yml +++ b/ci/.azure-pipelines.pr.yml @@ -75,13 +75,13 @@ steps: - script: | set -a && source .env && set +a - psql -h $POSTGRES_HOST -U $POSTGRES_USER -d prd-pointer -c "DROP SCHEMA IF EXISTS HARDCODED_SCHEMA CASCADE;" + psql -h $POSTGRES_HOST -U $POSTGRES_USER -d prd-pointer -c "DROP SCHEMA IF EXISTS hardcoded_schema CASCADE;" displayName: "Preemptive DROP SCHEMA" - script: | source venv/bin/activate - echo "CI_SCHEMA_NAME=HARDCODED_SCHEMA" >> .env + echo "CI_SCHEMA_NAME=hardcoded_schema" >> .env set -a && source .env && set +a dbt run --select "state:modified" --state ~/dbt-ci/master-artifacts/ --defer @@ -90,7 +90,7 @@ steps: - script: | source venv/bin/activate - echo "CI_SCHEMA_NAME=HARDCODED_SCHEMA" >> .env + echo "CI_SCHEMA_NAME=hardcoded_schema" >> .env set -a && source .env && set +a dbt test --select "state:modified" --state ~/dbt-ci/master-artifacts/ --defer @@ -98,8 +98,26 @@ steps: displayName: 'Test modified models' - script: | + source venv/bin/activate + echo "CI_SCHEMA_NAME=hardcoded_schema" >> .env set -a && source .env && set +a - psql -h $POSTGRES_HOST -U $POSTGRES_USER -d prd-pointer -c "DROP SCHEMA IF EXISTS HARDCODED_SCHEMA CASCADE;" + + dbt run --select "state:modified+" --exclude "state:modified" --state ~/dbt-ci/master-artifacts/ --defer + + displayName: 'Run downstream models' + +- script: | + source venv/bin/activate + echo "CI_SCHEMA_NAME=hardcoded_schema" >> .env + set -a && source .env && set +a + + dbt test --select "state:modified+" --exclude "state:modified" --state ~/dbt-ci/master-artifacts/ --defer + + displayName: 'Test downstream models' + +- script: | + set -a && source .env && set +a + psql -h $POSTGRES_HOST -U $POSTGRES_USER -d prd-pointer -c "DROP SCHEMA IF EXISTS hardcoded_schema CASCADE;" condition: always() displayName: 'Delete PR schema' \ No newline at end of file From db66bb6d97122ece10e8445972f32b8a2d8c3780 Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Mon, 24 Mar 2025 14:17:02 +0100 Subject: [PATCH 10/14] replace hardcodes --- ci/.azure-pipelines.pr.yml | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/ci/.azure-pipelines.pr.yml b/ci/.azure-pipelines.pr.yml index a382628..2aea215 100644 --- a/ci/.azure-pipelines.pr.yml +++ b/ci/.azure-pipelines.pr.yml @@ -11,18 +11,18 @@ steps: - script: | cp ~/dbt-ci/.env . - # Check if it matches the refs/pull//merge pattern - # if [[ $BUILD_SOURCEBRANCH =~ ^refs/pull/([0-9]+)/merge$ ]]; then - # POSTGRES_DATABASE="ci_pr_${BASH_REMATCH[1]}" - # else - # POSTGRES_DATABASE="ci_${BUILD_SOURCEBRANCH#refs/heads/}" - # fi + if [[ $BUILD_SOURCEBRANCH =~ ^refs/pull/([0-9]+)/merge$ ]]; then + "CI_SCHEMA_NAME="ci_pr_${BASH_REMATCH[1]}" + else + echo "Not running in a PR. Exiting" + exit 1 + fi - echo "POSTGRES_DATABASE: $POSTGRES_DATABASE" + echo "CI_SCHEMA_NAME: $CI_SCHEMA_NAME" - echo "POSTGRES_DATABASE=prd-pointer" >> .env + echo "CI_SCHEMA_NAME=$CI_SCHEMA_NAME" >> .env set -a && source .env && set +a - displayName: 'Load env' + displayName: 'Prepare env vars' - script: | set -a && source .env && set +a @@ -52,6 +52,7 @@ steps: - script: | source venv/bin/activate set -a && source .env && set +a + export IS_CI_EXECUTION=1 dbt docs generate --no-compile --empty-catalog --select "state:modified" --state ~/dbt-ci/master-artifacts/ @@ -60,6 +61,7 @@ steps: - script: | source venv/bin/activate set -a && source .env && set +a + export IS_CI_EXECUTION=1 dbt compile --debug --select "state:modified" --state ~/dbt-ci/master-artifacts/ displayName: 'Run dbt compile' @@ -75,14 +77,14 @@ steps: - script: | set -a && source .env && set +a - psql -h $POSTGRES_HOST -U $POSTGRES_USER -d prd-pointer -c "DROP SCHEMA IF EXISTS hardcoded_schema CASCADE;" + psql -h $POSTGRES_HOST -U $POSTGRES_USER -d prd-pointer -c "DROP SCHEMA IF EXISTS $CI_SCHEMA_NAME CASCADE;" displayName: "Preemptive DROP SCHEMA" - script: | source venv/bin/activate - echo "CI_SCHEMA_NAME=hardcoded_schema" >> .env set -a && source .env && set +a + export IS_CI_EXECUTION=1 dbt run --select "state:modified" --state ~/dbt-ci/master-artifacts/ --defer @@ -90,8 +92,8 @@ steps: - script: | source venv/bin/activate - echo "CI_SCHEMA_NAME=hardcoded_schema" >> .env set -a && source .env && set +a + export IS_CI_EXECUTION=1 dbt test --select "state:modified" --state ~/dbt-ci/master-artifacts/ --defer @@ -99,8 +101,8 @@ steps: - script: | source venv/bin/activate - echo "CI_SCHEMA_NAME=hardcoded_schema" >> .env set -a && source .env && set +a + export IS_CI_EXECUTION=1 dbt run --select "state:modified+" --exclude "state:modified" --state ~/dbt-ci/master-artifacts/ --defer @@ -108,8 +110,8 @@ steps: - script: | source venv/bin/activate - echo "CI_SCHEMA_NAME=hardcoded_schema" >> .env set -a && source .env && set +a + export IS_CI_EXECUTION=1 dbt test --select "state:modified+" --exclude "state:modified" --state ~/dbt-ci/master-artifacts/ --defer @@ -117,7 +119,7 @@ steps: - script: | set -a && source .env && set +a - psql -h $POSTGRES_HOST -U $POSTGRES_USER -d prd-pointer -c "DROP SCHEMA IF EXISTS hardcoded_schema CASCADE;" + psql -h $POSTGRES_HOST -U $POSTGRES_USER -d prd-pointer -c "DROP SCHEMA IF EXISTS $CI_SCHEMA_NAME CASCADE;" condition: always() displayName: 'Delete PR schema' \ No newline at end of file From 18173f7ffda4a80267e2aa6a11fc2e03fdcf9425 Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Mon, 24 Mar 2025 14:21:05 +0100 Subject: [PATCH 11/14] typo --- ci/.azure-pipelines.pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/.azure-pipelines.pr.yml b/ci/.azure-pipelines.pr.yml index 2aea215..27450b6 100644 --- a/ci/.azure-pipelines.pr.yml +++ b/ci/.azure-pipelines.pr.yml @@ -12,7 +12,7 @@ steps: cp ~/dbt-ci/.env . if [[ $BUILD_SOURCEBRANCH =~ ^refs/pull/([0-9]+)/merge$ ]]; then - "CI_SCHEMA_NAME="ci_pr_${BASH_REMATCH[1]}" + CI_SCHEMA_NAME="ci_pr_${BASH_REMATCH[1]}" else echo "Not running in a PR. Exiting" exit 1 From 5b6f0a6bc4a4a35f111f6e6f3861737cf2a7d913 Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Mon, 24 Mar 2025 14:59:06 +0100 Subject: [PATCH 12/14] trigger a lot of runs --- models/staging/core/stg_core__user.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/models/staging/core/stg_core__user.sql b/models/staging/core/stg_core__user.sql index 56a06cf..32a46b5 100644 --- a/models/staging/core/stg_core__user.sql +++ b/models/staging/core/stg_core__user.sql @@ -4,6 +4,7 @@ with raw_user as (select * from {{ source("core", "User") }}), stg_core__user as ( select + 1 as lol, {{ adapter.quote("Id") }} as id_user, {{ adapter.quote("AccountTypeId") }} as id_account_type, lower({{ adapter.quote("CreatedUserId") }}) as id_created_user, From ddc38ef513e047d0368ebce5f2adb90196783ca1 Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Mon, 24 Mar 2025 16:27:15 +0100 Subject: [PATCH 13/14] only direct downstream, not all downstream --- ci/.azure-pipelines.pr.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ci/.azure-pipelines.pr.yml b/ci/.azure-pipelines.pr.yml index 27450b6..f188e04 100644 --- a/ci/.azure-pipelines.pr.yml +++ b/ci/.azure-pipelines.pr.yml @@ -104,18 +104,18 @@ steps: set -a && source .env && set +a export IS_CI_EXECUTION=1 - dbt run --select "state:modified+" --exclude "state:modified" --state ~/dbt-ci/master-artifacts/ --defer + dbt run --select "state:modified+1" --exclude "state:modified" --state ~/dbt-ci/master-artifacts/ --defer - displayName: 'Run downstream models' + displayName: 'Run direct dependants of modified models' - script: | source venv/bin/activate set -a && source .env && set +a export IS_CI_EXECUTION=1 - dbt test --select "state:modified+" --exclude "state:modified" --state ~/dbt-ci/master-artifacts/ --defer + dbt test --select "state:modified+1" --exclude "state:modified" --state ~/dbt-ci/master-artifacts/ --defer - displayName: 'Test downstream models' + displayName: 'Test direct dependants of modified models' - script: | set -a && source .env && set +a From 5008cb10826dc3f915774bf642c409e139b8ad64 Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Mon, 24 Mar 2025 16:32:38 +0100 Subject: [PATCH 14/14] remove silly test changes --- .../intermediate/athena/int_athena__verifications_with_fees.sql | 2 -- models/staging/core/stg_core__user.sql | 1 - 2 files changed, 3 deletions(-) diff --git a/models/intermediate/athena/int_athena__verifications_with_fees.sql b/models/intermediate/athena/int_athena__verifications_with_fees.sql index cf85e05..e770aaf 100644 --- a/models/intermediate/athena/int_athena__verifications_with_fees.sql +++ b/models/intermediate/athena/int_athena__verifications_with_fees.sql @@ -15,8 +15,6 @@ with where v.version = 'V1' and v.id_booking is not null ) select - 1 as lol, - 2 as lolol, v.id_verification, v.id_booking, v.verification_status, diff --git a/models/staging/core/stg_core__user.sql b/models/staging/core/stg_core__user.sql index 32a46b5..56a06cf 100644 --- a/models/staging/core/stg_core__user.sql +++ b/models/staging/core/stg_core__user.sql @@ -4,7 +4,6 @@ with raw_user as (select * from {{ source("core", "User") }}), stg_core__user as ( select - 1 as lol, {{ adapter.quote("Id") }} as id_user, {{ adapter.quote("AccountTypeId") }} as id_account_type, lower({{ adapter.quote("CreatedUserId") }}) as id_created_user,