From d26c70bba47813d358a5810412e736e85d4fdfe7 Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Fri, 21 Mar 2025 12:15:50 +0100 Subject: [PATCH 01/13] add steps in ci --- ci/.azure-pipelines.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ci/.azure-pipelines.yml b/ci/.azure-pipelines.yml index 211af25..f4cf9d4 100644 --- a/ci/.azure-pipelines.yml +++ b/ci/.azure-pipelines.yml @@ -39,6 +39,11 @@ steps: psql -h $POSTGRES_HOST -U $POSTGRES_USER -d postgres -c "CREATE DATABASE ${POSTGRES_DATABASE}" displayName: 'Create branch dedicated database' +- script: | + cd ~/dbt-ci + /bin/bash build-master-artifacts.sh + displayName: 'Build master artifacts' + - script: | python3 -m venv venv source venv/bin/activate @@ -65,7 +70,7 @@ steps: source venv/bin/activate set -a && source .env && set +a - dbt compile + dbt compile --select "state:modified"' --state ~/dbt-ci/master-artifacts/ displayName: 'Run dbt compile' - script: | From 14b927b8d41e4a0ab2f3f8a071f41e27ee294765 Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Fri, 21 Mar 2025 12:23:09 +0100 Subject: [PATCH 02/13] typo --- ci/.azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/.azure-pipelines.yml b/ci/.azure-pipelines.yml index f4cf9d4..cb13b03 100644 --- a/ci/.azure-pipelines.yml +++ b/ci/.azure-pipelines.yml @@ -70,7 +70,7 @@ steps: source venv/bin/activate set -a && source .env && set +a - dbt compile --select "state:modified"' --state ~/dbt-ci/master-artifacts/ + dbt compile --select "state:modified" --state ~/dbt-ci/master-artifacts/ displayName: 'Run dbt compile' - script: | From bc7ea8d001d14108ab3265cd0913812925a62095 Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Fri, 21 Mar 2025 12:28:18 +0100 Subject: [PATCH 03/13] make dbt docs a bit more efficient, hopefully --- ci/.azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/.azure-pipelines.yml b/ci/.azure-pipelines.yml index cb13b03..ec9ba2f 100644 --- a/ci/.azure-pipelines.yml +++ b/ci/.azure-pipelines.yml @@ -62,7 +62,7 @@ steps: source venv/bin/activate set -a && source .env && set +a - dbt docs generate + dbt docs generate --no-compile --empty-catalog --select "state:modified" --state ~/dbt-ci/master-artifacts/ displayName: 'Run dbt docs' From 395c0643659a3f91c9e2b4c9493616c9121d660a Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Fri, 21 Mar 2025 12:30:25 +0100 Subject: [PATCH 04/13] edit one model to test state detection --- .../intermediate/kpis/int_kpis__agg_daily_created_bookings.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/models/intermediate/kpis/int_kpis__agg_daily_created_bookings.sql b/models/intermediate/kpis/int_kpis__agg_daily_created_bookings.sql index c312a74..c1608e5 100644 --- a/models/intermediate/kpis/int_kpis__agg_daily_created_bookings.sql +++ b/models/intermediate/kpis/int_kpis__agg_daily_created_bookings.sql @@ -6,6 +6,7 @@ {% for dimension in dimensions %} select -- Unique Key -- + 1 as lol, date, {{ dimension.dimension }} as dimension, {{ dimension.dimension_value }} as dimension_value, From e0fbf093b8c3725c221080096a32a9a42679941c Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Fri, 21 Mar 2025 12:32:31 +0100 Subject: [PATCH 05/13] undo stupid change --- .../intermediate/kpis/int_kpis__agg_daily_created_bookings.sql | 1 - 1 file changed, 1 deletion(-) diff --git a/models/intermediate/kpis/int_kpis__agg_daily_created_bookings.sql b/models/intermediate/kpis/int_kpis__agg_daily_created_bookings.sql index c1608e5..c312a74 100644 --- a/models/intermediate/kpis/int_kpis__agg_daily_created_bookings.sql +++ b/models/intermediate/kpis/int_kpis__agg_daily_created_bookings.sql @@ -6,7 +6,6 @@ {% for dimension in dimensions %} select -- Unique Key -- - 1 as lol, date, {{ dimension.dimension }} as dimension, {{ dimension.dimension_value }} as dimension_value, From 40f28a941188aa926fa860c43930fe3e1d8060c3 Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Fri, 21 Mar 2025 14:25:36 +0100 Subject: [PATCH 06/13] remove master trigger --- ci/.azure-pipelines.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/ci/.azure-pipelines.yml b/ci/.azure-pipelines.yml index ec9ba2f..dee5e86 100644 --- a/ci/.azure-pipelines.yml +++ b/ci/.azure-pipelines.yml @@ -1,7 +1,3 @@ -trigger: - branches: - include: - - master pool: name: ci-pool-prd From 5d92c4321570fd61e0becf72fab66d2353777c5d Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Fri, 21 Mar 2025 14:38:48 +0100 Subject: [PATCH 07/13] add ls step --- ci/.azure-pipelines.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ci/.azure-pipelines.yml b/ci/.azure-pipelines.yml index dee5e86..8a98a8b 100644 --- a/ci/.azure-pipelines.yml +++ b/ci/.azure-pipelines.yml @@ -62,6 +62,14 @@ steps: displayName: 'Run dbt docs' +- script: | + source venv/bin/activate + set -a && source .env && set +a + + echo "Running dbt ls so you see what models are selected." + dbt ls --select "state:modified" --state ~/dbt-ci/master-artifacts/ + displayName: 'Run dbt list' + - script: | source venv/bin/activate set -a && source .env && set +a From 229ab901d1eb2455304b365aaeadcf99e97a8776 Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Fri, 21 Mar 2025 15:02:31 +0100 Subject: [PATCH 08/13] restrict list to models, tests and exposures --- ci/.azure-pipelines.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ci/.azure-pipelines.yml b/ci/.azure-pipelines.yml index 8a98a8b..af8de88 100644 --- a/ci/.azure-pipelines.yml +++ b/ci/.azure-pipelines.yml @@ -67,7 +67,10 @@ steps: set -a && source .env && set +a echo "Running dbt ls so you see what models are selected." - dbt ls --select "state:modified" --state ~/dbt-ci/master-artifacts/ + dbt ls \ + --select "state:modified" \ + --state ~/dbt-ci/master-artifacts/ \ + --resource-types model,test,exposure displayName: 'Run dbt list' - script: | From eedc5931c7018659250751ae16ddd517362d688f Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Fri, 21 Mar 2025 15:02:47 +0100 Subject: [PATCH 09/13] restrict list --- ci/.azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/.azure-pipelines.yml b/ci/.azure-pipelines.yml index af8de88..382810a 100644 --- a/ci/.azure-pipelines.yml +++ b/ci/.azure-pipelines.yml @@ -86,7 +86,7 @@ steps: cd ci /bin/bash sqlfluff-check.sh ../target/compiled - displayName: 'Validate compiled SQL' + displayName: 'Validate models compiled SQL' - script: | set -a && source .env && set +a From 682cf592a6105dfd95d90c2dfadbcaa8ef31d1c7 Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Fri, 21 Mar 2025 15:09:47 +0100 Subject: [PATCH 10/13] typo --- ci/.azure-pipelines.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ci/.azure-pipelines.yml b/ci/.azure-pipelines.yml index 382810a..d95cede 100644 --- a/ci/.azure-pipelines.yml +++ b/ci/.azure-pipelines.yml @@ -66,11 +66,13 @@ steps: source venv/bin/activate set -a && source .env && set +a - echo "Running dbt ls so you see what models are selected." + echo "Running dbt ls so you see what models, tests and exposures are selected." dbt ls \ --select "state:modified" \ --state ~/dbt-ci/master-artifacts/ \ - --resource-types model,test,exposure + --resource-type model + --resource-type test + --resource-type exposure displayName: 'Run dbt list' - script: | From 30ab85bd24283030e595c93da289bc5f33b5918b Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Fri, 21 Mar 2025 15:14:53 +0100 Subject: [PATCH 11/13] typo, disable run on every commit --- ci/.azure-pipelines.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ci/.azure-pipelines.yml b/ci/.azure-pipelines.yml index d95cede..9516acd 100644 --- a/ci/.azure-pipelines.yml +++ b/ci/.azure-pipelines.yml @@ -1,3 +1,4 @@ +trigger: none # Because we trigger this only in PRs, no need to have triggers here pool: name: ci-pool-prd @@ -70,8 +71,8 @@ steps: dbt ls \ --select "state:modified" \ --state ~/dbt-ci/master-artifacts/ \ - --resource-type model - --resource-type test + --resource-type model \ + --resource-type test \ --resource-type exposure displayName: 'Run dbt list' From dc51c503d0fcfb8676c985ddc146d422c8a1e9bc Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Fri, 21 Mar 2025 15:20:37 +0100 Subject: [PATCH 12/13] a couple of small changes --- ci/.azure-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/.azure-pipelines.yml b/ci/.azure-pipelines.yml index 9516acd..5bd98fb 100644 --- a/ci/.azure-pipelines.yml +++ b/ci/.azure-pipelines.yml @@ -6,7 +6,7 @@ pool: steps: - script: | echo "Running on branch: ${BUILD_SOURCEBRANCH#refs/heads/}" - displayName: 'Smoke test filesystem access' + displayName: 'Print branch' - script: | cp ~/dbt-ci/.env . @@ -80,7 +80,7 @@ steps: source venv/bin/activate set -a && source .env && set +a - dbt compile --select "state:modified" --state ~/dbt-ci/master-artifacts/ + dbt compile --debug --select "state:modified" --state ~/dbt-ci/master-artifacts/ displayName: 'Run dbt compile' - script: | From 2779a73d1064d4bcf43d632f68be6cfdf7033437 Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Fri, 21 Mar 2025 15:24:32 +0100 Subject: [PATCH 13/13] remove list step --- ci/.azure-pipelines.yml | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/ci/.azure-pipelines.yml b/ci/.azure-pipelines.yml index 5bd98fb..3502fb4 100644 --- a/ci/.azure-pipelines.yml +++ b/ci/.azure-pipelines.yml @@ -63,19 +63,6 @@ steps: displayName: 'Run dbt docs' -- script: | - source venv/bin/activate - set -a && source .env && set +a - - echo "Running dbt ls so you see what models, tests and exposures are selected." - dbt ls \ - --select "state:modified" \ - --state ~/dbt-ci/master-artifacts/ \ - --resource-type model \ - --resource-type test \ - --resource-type exposure - displayName: 'Run dbt list' - - script: | source venv/bin/activate set -a && source .env && set +a