data-dwh-dbt-project/ci/.azure-pipelines.pr.yml

trigger: none # Because we trigger this only in PRs, no need to have triggers here

pool:
  name: ci-pool-prd

steps:
- script: |
    echo "Running on branch: ${BUILD_SOURCEBRANCH#refs/heads/}"
  displayName: 'Print branch'

- script: |
    cp ~/dbt-ci/.env .

    if [[ $BUILD_SOURCEBRANCH =~ ^refs/pull/([0-9]+)/merge$ ]]; then
      CI_SCHEMA_NAME="ci_pr_${BASH_REMATCH[1]}"
    else
      echo "Not running in a PR. Exiting"
      exit 1
    fi

    echo "CI_SCHEMA_NAME: $CI_SCHEMA_NAME"

    echo "CI_SCHEMA_NAME=$CI_SCHEMA_NAME" >> .env
    set -a && source .env && set +a
  displayName: 'Prepare env vars'

- script: |
    set -a && source .env && set +a

    psql -h $POSTGRES_HOST -U $POSTGRES_USER -d postgres -c "SELECT 1 FROM pg_database WHERE datname = 'postgres';"
  displayName: 'Smoke test Postgres connection'

- script: |
    set -a && source .env && set +a

    psql -h $POSTGRES_HOST -U $POSTGRES_USER -d prd-pointer -c "select refresh_foreign_schemas(ARRAY[$PRD_SCHEMAS_TO_SYNC]);"
  displayName: 'Sync Foreign Data Wrappers schemas'

- script: |
    cd ci
    /bin/bash build-master-artifacts.sh
  displayName: 'Build master artifacts'

- script: |
    python3 -m venv venv
    source venv/bin/activate
    pip install -r requirements.txt
    pip install -r ci/ci-requirements.txt

  displayName: 'Create python virtual environment'

- script: |
    source venv/bin/activate
    dbt deps

  displayName: 'Install dbt deps'

- script: |
    source venv/bin/activate
    set -a && source .env && set +a
    export IS_CI_EXECUTION=1

    dbt docs generate --no-compile --empty-catalog --select "state:modified" --state ~/dbt-ci/master-artifacts/

  displayName: 'Run dbt docs'

- script: |
    source venv/bin/activate
    set -a && source .env && set +a
    export IS_CI_EXECUTION=1

    dbt compile --debug --select "state:modified" --state ~/dbt-ci/master-artifacts/
  displayName: 'Run dbt compile'

- script: |
    source venv/bin/activate

    cd ci
    /bin/bash sqlfluff-check.sh ../target/compiled

  displayName: 'Validate models compiled SQL'

- script: |
    set -a && source .env && set +a

    psql -h $POSTGRES_HOST -U $POSTGRES_USER -d prd-pointer -c "DROP SCHEMA IF EXISTS $CI_SCHEMA_NAME CASCADE;"
  
  displayName: "Preemptive DROP SCHEMA"

- script: |
    source venv/bin/activate
    set -a && source .env && set +a
    export IS_CI_EXECUTION=1

    dbt run --select "state:modified" --state ~/dbt-ci/master-artifacts/ --defer

  displayName: 'Run modified models'

- script: |
    source venv/bin/activate
    set -a && source .env && set +a
    export IS_CI_EXECUTION=1

    dbt test --select "state:modified" --state ~/dbt-ci/master-artifacts/ --defer

  displayName: 'Test modified models'

- script: |
    source venv/bin/activate
    set -a && source .env && set +a
    export IS_CI_EXECUTION=1

    dbt run --select "state:modified+1" --exclude "state:modified"  --state ~/dbt-ci/master-artifacts/ --defer

  displayName: 'Run direct dependants of modified models'

- script: |
    source venv/bin/activate
    set -a && source .env && set +a
    export IS_CI_EXECUTION=1

    dbt test --select "state:modified+1" --exclude "state:modified" --state ~/dbt-ci/master-artifacts/ --defer

  displayName: 'Test direct dependants of modified models'

- script: |
    set -a && source .env && set +a
    psql -h $POSTGRES_HOST -U $POSTGRES_USER -d prd-pointer -c "DROP SCHEMA IF EXISTS $CI_SCHEMA_NAME CASCADE;"

  condition: always()
  displayName: 'Delete PR schema'