trigger: none # Because we trigger this only in PRs, no need to have triggers here pool: name: ci-pool-prd steps: - script: | echo "Running on branch: ${BUILD_SOURCEBRANCH#refs/heads/}" displayName: 'Print branch' - script: | cp ~/dbt-ci/.env . if [[ $BUILD_SOURCEBRANCH =~ ^refs/pull/([0-9]+)/merge$ ]]; then CI_SCHEMA_NAME="ci_pr_${BASH_REMATCH[1]}" else echo "Not running in a PR. Exiting" exit 1 fi echo "CI_SCHEMA_NAME: $CI_SCHEMA_NAME" echo "CI_SCHEMA_NAME=$CI_SCHEMA_NAME" >> .env set -a && source .env && set +a displayName: 'Prepare env vars' - script: | set -a && source .env && set +a psql -h $POSTGRES_HOST -U $POSTGRES_USER -d postgres -c "SELECT 1 FROM pg_database WHERE datname = 'postgres';" displayName: 'Smoke test Postgres connection' - script: | set -a && source .env && set +a psql -h $POSTGRES_HOST -U $POSTGRES_USER -d prd-pointer -c "select refresh_foreign_schemas(ARRAY[$PRD_SCHEMAS_TO_SYNC]);" displayName: 'Sync Foreign Data Wrappers schemas' - script: | cd ~/dbt-ci /bin/bash build-master-artifacts.sh displayName: 'Build master artifacts' - script: | python3 -m venv venv source venv/bin/activate pip install -r requirements.txt pip install -r ci/ci-requirements.txt displayName: 'Create python virtual environment' - script: | source venv/bin/activate dbt deps displayName: 'Install dbt deps' - script: | source venv/bin/activate set -a && source .env && set +a export IS_CI_EXECUTION=1 dbt docs generate --no-compile --empty-catalog --select "state:modified" --state ~/dbt-ci/master-artifacts/ displayName: 'Run dbt docs' - script: | source venv/bin/activate set -a && source .env && set +a export IS_CI_EXECUTION=1 dbt compile --debug --select "state:modified" --state ~/dbt-ci/master-artifacts/ displayName: 'Run dbt compile' - script: | source venv/bin/activate cd ci /bin/bash sqlfluff-check.sh ../target/compiled displayName: 'Validate models compiled SQL' - script: | set -a && source .env && set +a psql -h $POSTGRES_HOST -U $POSTGRES_USER -d prd-pointer -c "DROP SCHEMA IF EXISTS $CI_SCHEMA_NAME CASCADE;" displayName: "Preemptive DROP SCHEMA" - script: | source venv/bin/activate set -a && source .env && set +a export IS_CI_EXECUTION=1 dbt run --select "state:modified" --state ~/dbt-ci/master-artifacts/ --defer displayName: 'Run modified models' - script: | source venv/bin/activate set -a && source .env && set +a export IS_CI_EXECUTION=1 dbt test --select "state:modified" --state ~/dbt-ci/master-artifacts/ --defer displayName: 'Test modified models' - script: | source venv/bin/activate set -a && source .env && set +a export IS_CI_EXECUTION=1 dbt run --select "state:modified+1" --exclude "state:modified" --state ~/dbt-ci/master-artifacts/ --defer displayName: 'Run direct dependants of modified models' - script: | source venv/bin/activate set -a && source .env && set +a export IS_CI_EXECUTION=1 dbt test --select "state:modified+1" --exclude "state:modified" --state ~/dbt-ci/master-artifacts/ --defer displayName: 'Test direct dependants of modified models' - script: | set -a && source .env && set +a psql -h $POSTGRES_HOST -U $POSTGRES_USER -d prd-pointer -c "DROP SCHEMA IF EXISTS $CI_SCHEMA_NAME CASCADE;" condition: always() displayName: 'Delete PR schema'