diff --git a/ci/.azure-pipelines.pr.yml b/ci/.azure-pipelines.pr.yml index 3502fb4..f188e04 100644 --- a/ci/.azure-pipelines.pr.yml +++ b/ci/.azure-pipelines.pr.yml @@ -11,18 +11,18 @@ steps: - script: | cp ~/dbt-ci/.env . - # Check if it matches the refs/pull//merge pattern if [[ $BUILD_SOURCEBRANCH =~ ^refs/pull/([0-9]+)/merge$ ]]; then - POSTGRES_DATABASE="ci_pr_${BASH_REMATCH[1]}" + CI_SCHEMA_NAME="ci_pr_${BASH_REMATCH[1]}" else - POSTGRES_DATABASE="ci_${BUILD_SOURCEBRANCH#refs/heads/}" + echo "Not running in a PR. Exiting" + exit 1 fi - echo "POSTGRES_DATABASE: $POSTGRES_DATABASE" + echo "CI_SCHEMA_NAME: $CI_SCHEMA_NAME" - echo "POSTGRES_DATABASE=${POSTGRES_DATABASE}" >> .env + echo "CI_SCHEMA_NAME=$CI_SCHEMA_NAME" >> .env set -a && source .env && set +a - displayName: 'Load env' + displayName: 'Prepare env vars' - script: | set -a && source .env && set +a @@ -30,12 +30,6 @@ steps: psql -h $POSTGRES_HOST -U $POSTGRES_USER -d postgres -c "SELECT 1 FROM pg_database WHERE datname = 'postgres';" displayName: 'Smoke test Postgres connection' -- script: | - set -a && source .env && set +a - - psql -h $POSTGRES_HOST -U $POSTGRES_USER -d postgres -c "CREATE DATABASE ${POSTGRES_DATABASE}" - displayName: 'Create branch dedicated database' - - script: | cd ~/dbt-ci /bin/bash build-master-artifacts.sh @@ -58,6 +52,7 @@ steps: - script: | source venv/bin/activate set -a && source .env && set +a + export IS_CI_EXECUTION=1 dbt docs generate --no-compile --empty-catalog --select "state:modified" --state ~/dbt-ci/master-artifacts/ @@ -66,6 +61,7 @@ steps: - script: | source venv/bin/activate set -a && source .env && set +a + export IS_CI_EXECUTION=1 dbt compile --debug --select "state:modified" --state ~/dbt-ci/master-artifacts/ displayName: 'Run dbt compile' @@ -81,7 +77,49 @@ steps: - script: | set -a && source .env && set +a - psql -h $POSTGRES_HOST -U $POSTGRES_USER -d postgres -c "DROP DATABASE ${POSTGRES_DATABASE}" + psql -h $POSTGRES_HOST -U $POSTGRES_USER -d prd-pointer -c "DROP SCHEMA IF EXISTS $CI_SCHEMA_NAME CASCADE;" + + displayName: "Preemptive DROP SCHEMA" + +- script: | + source venv/bin/activate + set -a && source .env && set +a + export IS_CI_EXECUTION=1 + + dbt run --select "state:modified" --state ~/dbt-ci/master-artifacts/ --defer + + displayName: 'Run modified models' + +- script: | + source venv/bin/activate + set -a && source .env && set +a + export IS_CI_EXECUTION=1 + + dbt test --select "state:modified" --state ~/dbt-ci/master-artifacts/ --defer + + displayName: 'Test modified models' + +- script: | + source venv/bin/activate + set -a && source .env && set +a + export IS_CI_EXECUTION=1 + + dbt run --select "state:modified+1" --exclude "state:modified" --state ~/dbt-ci/master-artifacts/ --defer + + displayName: 'Run direct dependants of modified models' + +- script: | + source venv/bin/activate + set -a && source .env && set +a + export IS_CI_EXECUTION=1 + + dbt test --select "state:modified+1" --exclude "state:modified" --state ~/dbt-ci/master-artifacts/ --defer + + displayName: 'Test direct dependants of modified models' + +- script: | + set -a && source .env && set +a + psql -h $POSTGRES_HOST -U $POSTGRES_USER -d prd-pointer -c "DROP SCHEMA IF EXISTS $CI_SCHEMA_NAME CASCADE;" + condition: always() - displayName: 'Delete branch dedicated database' - \ No newline at end of file + displayName: 'Delete PR schema' \ No newline at end of file diff --git a/macros/generate_schema_name.sql b/macros/generate_schema_name.sql index c32da69..fb86fed 100644 --- a/macros/generate_schema_name.sql +++ b/macros/generate_schema_name.sql @@ -9,10 +9,12 @@ To understand better, check: {% macro generate_schema_name(custom_schema_name, node) -%} {%- set default_schema = target.schema -%} - {%- if custom_schema_name is none -%} {{ default_schema }} + {%- set is_ci_execution = env_var("IS_CI_EXECUTION", "0") | int -%} + {%- set ci_schema_name = env_var("CI_SCHEMA_NAME", "ci_default") -%} + {%- if is_ci_execution == 1 -%} {{ ci_schema_name }} + {%- elif custom_schema_name is none -%} {{ default_schema }} {%- else -%} {{ custom_schema_name | trim }} - {%- endif -%} {%- endmacro %}