131 lines
No EOL
3.4 KiB
YAML
131 lines
No EOL
3.4 KiB
YAML
trigger: none # Because we trigger this only in PRs, no need to have triggers here
|
|
|
|
pool:
|
|
name: ci-pool-prd
|
|
|
|
steps:
|
|
- script: |
|
|
echo "Running on branch: ${BUILD_SOURCEBRANCH#refs/heads/}"
|
|
displayName: 'Print branch'
|
|
|
|
- script: |
|
|
cp ~/dbt-ci/.env .
|
|
|
|
if [[ $BUILD_SOURCEBRANCH =~ ^refs/pull/([0-9]+)/merge$ ]]; then
|
|
CI_SCHEMA_NAME="ci_pr_${BASH_REMATCH[1]}"
|
|
else
|
|
echo "Not running in a PR. Exiting"
|
|
exit 1
|
|
fi
|
|
|
|
echo "CI_SCHEMA_NAME: $CI_SCHEMA_NAME"
|
|
|
|
echo "CI_SCHEMA_NAME=$CI_SCHEMA_NAME" >> .env
|
|
set -a && source .env && set +a
|
|
displayName: 'Prepare env vars'
|
|
|
|
- script: |
|
|
set -a && source .env && set +a
|
|
|
|
psql -h $POSTGRES_HOST -U $POSTGRES_USER -d postgres -c "SELECT 1 FROM pg_database WHERE datname = 'postgres';"
|
|
displayName: 'Smoke test Postgres connection'
|
|
|
|
- script: |
|
|
set -a && source .env && set +a
|
|
|
|
psql -h $POSTGRES_HOST -U $POSTGRES_USER -d prd-pointer -c "select refresh_foreign_schemas(ARRAY[$PRD_SCHEMAS_TO_SYNC]);"
|
|
displayName: 'Sync Foreign Data Wrappers schemas'
|
|
|
|
- script: |
|
|
cd ci
|
|
/bin/bash build-master-artifacts.sh
|
|
displayName: 'Build master artifacts'
|
|
|
|
- script: |
|
|
python3 -m venv venv
|
|
source venv/bin/activate
|
|
pip install -r requirements.txt
|
|
pip install -r ci/ci-requirements.txt
|
|
|
|
displayName: 'Create python virtual environment'
|
|
|
|
- script: |
|
|
source venv/bin/activate
|
|
dbt deps
|
|
|
|
displayName: 'Install dbt deps'
|
|
|
|
- script: |
|
|
source venv/bin/activate
|
|
set -a && source .env && set +a
|
|
export IS_CI_EXECUTION=1
|
|
|
|
dbt docs generate --no-compile --empty-catalog --select "state:modified" --state ~/dbt-ci/master-artifacts/
|
|
|
|
displayName: 'Run dbt docs'
|
|
|
|
- script: |
|
|
source venv/bin/activate
|
|
set -a && source .env && set +a
|
|
export IS_CI_EXECUTION=1
|
|
|
|
dbt compile --debug --select "state:modified" --state ~/dbt-ci/master-artifacts/
|
|
displayName: 'Run dbt compile'
|
|
|
|
- script: |
|
|
source venv/bin/activate
|
|
|
|
cd ci
|
|
/bin/bash sqlfluff-check.sh ../target/compiled
|
|
|
|
displayName: 'Validate models compiled SQL'
|
|
|
|
- script: |
|
|
set -a && source .env && set +a
|
|
|
|
psql -h $POSTGRES_HOST -U $POSTGRES_USER -d prd-pointer -c "DROP SCHEMA IF EXISTS $CI_SCHEMA_NAME CASCADE;"
|
|
|
|
displayName: "Preemptive DROP SCHEMA"
|
|
|
|
- script: |
|
|
source venv/bin/activate
|
|
set -a && source .env && set +a
|
|
export IS_CI_EXECUTION=1
|
|
|
|
dbt run --select "state:modified" --state ~/dbt-ci/master-artifacts/ --defer
|
|
|
|
displayName: 'Run modified models'
|
|
|
|
- script: |
|
|
source venv/bin/activate
|
|
set -a && source .env && set +a
|
|
export IS_CI_EXECUTION=1
|
|
|
|
dbt test --select "state:modified" --state ~/dbt-ci/master-artifacts/ --defer
|
|
|
|
displayName: 'Test modified models'
|
|
|
|
- script: |
|
|
source venv/bin/activate
|
|
set -a && source .env && set +a
|
|
export IS_CI_EXECUTION=1
|
|
|
|
dbt run --select "state:modified+1" --exclude "state:modified" --state ~/dbt-ci/master-artifacts/ --defer
|
|
|
|
displayName: 'Run direct dependants of modified models'
|
|
|
|
- script: |
|
|
source venv/bin/activate
|
|
set -a && source .env && set +a
|
|
export IS_CI_EXECUTION=1
|
|
|
|
dbt test --select "state:modified+1" --exclude "state:modified" --state ~/dbt-ci/master-artifacts/ --defer
|
|
|
|
displayName: 'Test direct dependants of modified models'
|
|
|
|
- script: |
|
|
set -a && source .env && set +a
|
|
psql -h $POSTGRES_HOST -U $POSTGRES_USER -d prd-pointer -c "DROP SCHEMA IF EXISTS $CI_SCHEMA_NAME CASCADE;"
|
|
|
|
condition: always()
|
|
displayName: 'Delete PR schema' |