From 7cc893cad497aa63c8d470625cbf63e9af15e982 Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Wed, 19 Mar 2025 09:14:26 +0100 Subject: [PATCH 1/7] silly change just to trigger PR --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 6d95825..408ca5b 100644 --- a/README.md +++ b/README.md @@ -134,6 +134,10 @@ Once you build the docs with `run_docs.sh`, you will have a bunch of files. To o This goes beyond the scope of this project: to understand how you can serve these, refer to our [infra script repo](https://guardhog.visualstudio.com/Data/_git/data-infra-script). Specifically, the bits around the web gateway set up. +## CI + +TBD. + ## Stuff that we haven't done but we would like to - Automate formatting with git pre-commit. From 4b33114d6753b9fb5251b789dbeee930e7436dd1 Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Wed, 19 Mar 2025 09:19:30 +0100 Subject: [PATCH 2/7] modify pipeline --- .azure-pipelines.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml index f86e72c..94e7675 100644 --- a/.azure-pipelines.yml +++ b/.azure-pipelines.yml @@ -1,8 +1,12 @@ trigger: -- master # Runs on each commit to master + branches: + include: + - master pr: -- master # Runs on PRs targeting master + branches: + include: + - master pool: name: ci-pool-prd From 821c7d285037ad103b981618e590b9360f881002 Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Wed, 19 Mar 2025 09:26:03 +0100 Subject: [PATCH 3/7] remove pr stuff because it's fake news --- .azure-pipelines.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml index 94e7675..f482dd0 100644 --- a/.azure-pipelines.yml +++ b/.azure-pipelines.yml @@ -3,11 +3,6 @@ trigger: include: - master -pr: - branches: - include: - - master - pool: name: ci-pool-prd From 1e19efc89f1c3415c75aa3e921f47012206e27f3 Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Wed, 19 Mar 2025 15:17:33 +0100 Subject: [PATCH 4/7] make temporal db name selection a bit smarter --- .azure-pipelines.yml | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml index f482dd0..06505fc 100644 --- a/.azure-pipelines.yml +++ b/.azure-pipelines.yml @@ -13,24 +13,30 @@ steps: - script: | cp ~/dbt-ci/.env . - echo "POSTGRES_DATABASE=${BUILD_SOURCEBRANCH#refs/heads/}" >> .env + + # Check if it matches the refs/pull//merge pattern + if [[ $BUILD_SOURCEBRANCH =~ ^refs/pull/([0-9]+)/merge$ ]]; then + TEMPORAL_DATABASE_NAME="ci_pr_${BASH_REMATCH[1]}" + else + TEMPORAL_DATABASE_NAME="ci_${BUILD_SOURCEBRANCH#refs/heads/}" + fi + + echo "TEMPORAL_DATABASE_NAME: $TEMPORAL_DATABASE_NAME" + + echo "POSTGRES_DATABASE=${TEMPORAL_DATABASE_NAME}" >> .env set -a && source .env && set +a displayName: 'Load env' - script: | - cp ~/dbt-ci/.env . - echo "POSTGRES_DATABASE=${BUILD_SOURCEBRANCH#refs/heads/}" >> .env set -a && source .env && set +a psql -h $POSTGRES_HOST -U $POSTGRES_USER -d postgres -c "SELECT 1 FROM pg_database WHERE datname = 'postgres';" displayName: 'Smoke test Postgres connection' - script: | - cp ~/dbt-ci/.env . - echo "POSTGRES_DATABASE=${BUILD_SOURCEBRANCH#refs/heads/}" >> .env set -a && source .env && set +a - psql -h $POSTGRES_HOST -U $POSTGRES_USER -d postgres -c "CREATE DATABASE ci_${BUILD_SOURCEBRANCH#refs/heads/}" + psql -h $POSTGRES_HOST -U $POSTGRES_USER -d postgres -c "CREATE DATABASE ${TEMPORAL_DATABASE_NAME}" displayName: 'Create branch dedicated database' - script: | @@ -64,7 +70,7 @@ steps: - script: | set -a && source .env && set +a - psql -h $POSTGRES_HOST -U $POSTGRES_USER -d postgres -c "DROP DATABASE ci_${BUILD_SOURCEBRANCH#refs/heads/}" + psql -h $POSTGRES_HOST -U $POSTGRES_USER -d postgres -c "DROP DATABASE ${TEMPORAL_DATABASE_NAME}" condition: always() displayName: 'Delete branch dedicated database' \ No newline at end of file From 841b64dc852de6a4f78ff72e5991102409b8eb59 Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Wed, 19 Mar 2025 15:19:27 +0100 Subject: [PATCH 5/7] unfuck variables --- .azure-pipelines.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml index 06505fc..930e606 100644 --- a/.azure-pipelines.yml +++ b/.azure-pipelines.yml @@ -16,14 +16,14 @@ steps: # Check if it matches the refs/pull//merge pattern if [[ $BUILD_SOURCEBRANCH =~ ^refs/pull/([0-9]+)/merge$ ]]; then - TEMPORAL_DATABASE_NAME="ci_pr_${BASH_REMATCH[1]}" + POSTGRES_DATABASE="ci_pr_${BASH_REMATCH[1]}" else - TEMPORAL_DATABASE_NAME="ci_${BUILD_SOURCEBRANCH#refs/heads/}" + POSTGRES_DATABASE="ci_${BUILD_SOURCEBRANCH#refs/heads/}" fi - echo "TEMPORAL_DATABASE_NAME: $TEMPORAL_DATABASE_NAME" + echo "POSTGRES_DATABASE: $POSTGRES_DATABASE" - echo "POSTGRES_DATABASE=${TEMPORAL_DATABASE_NAME}" >> .env + echo "POSTGRES_DATABASE=${POSTGRES_DATABASE}" >> .env set -a && source .env && set +a displayName: 'Load env' From d334e95738a4b6ea5ac37f987154b1982f326349 Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Wed, 19 Mar 2025 15:23:59 +0100 Subject: [PATCH 6/7] typo --- .azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml index 930e606..0395989 100644 --- a/.azure-pipelines.yml +++ b/.azure-pipelines.yml @@ -36,7 +36,7 @@ steps: - script: | set -a && source .env && set +a - psql -h $POSTGRES_HOST -U $POSTGRES_USER -d postgres -c "CREATE DATABASE ${TEMPORAL_DATABASE_NAME}" + psql -h $POSTGRES_HOST -U $POSTGRES_USER -d postgres -c "CREATE DATABASE ${POSTGRES_DATABASE}" displayName: 'Create branch dedicated database' - script: | From 918d618f75f7316d6e93f93a9b0d6e4f19948704 Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Wed, 19 Mar 2025 15:26:50 +0100 Subject: [PATCH 7/7] adjust naming in deletion step --- .azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml index 0395989..61585f9 100644 --- a/.azure-pipelines.yml +++ b/.azure-pipelines.yml @@ -70,7 +70,7 @@ steps: - script: | set -a && source .env && set +a - psql -h $POSTGRES_HOST -U $POSTGRES_USER -d postgres -c "DROP DATABASE ${TEMPORAL_DATABASE_NAME}" + psql -h $POSTGRES_HOST -U $POSTGRES_USER -d postgres -c "DROP DATABASE ${POSTGRES_DATABASE}" condition: always() displayName: 'Delete branch dedicated database' \ No newline at end of file