data-dwh-dbt-project/run_tests.sh

#!/bin/bash

# === Logging setup ===
TIMESTAMP=$(date +"%Y%m%d_%H%M%S")
LOG_FILE="/home/azureuser/dbt_test_logs/dbt_tests_${TIMESTAMP}.log"
exec >> "$LOG_FILE" 2>&1

echo "=== dbt test run started at $TIMESTAMP ==="

# === Slack webhook setup ===
script_dir=$(dirname "$0")
webhooks_file="slack_webhook_urls.txt"
env_file="$script_dir/$webhooks_file"

if [ -f "$env_file" ]; then
  export $(grep -v '^#' "$env_file" | xargs)
else
  echo "Error: $webhooks_file file not found in the script directory."
  exit 1
fi

slack_failure_message=":rotating_light::rotating_light::rotating_light: One or more failures in dbt tests in production. :rotating_light::rotating_light::rotating_light:"
slack_success_message=":white_check_mark::white_check_mark::white_check_mark: dbt tests executed successfully in production. :white_check_mark::white_check_mark::white_check_mark:"

has_any_step_failed=0

# === Navigate to project ===
cd /home/azureuser/data-dwh-dbt-project || exit 1

# === Update from Git ===
echo "Updating dbt project from git."
git checkout master
git pull

# === Activate virtual environment ===
source venv/bin/activate

# === Run dbt tests ===
echo "Triggering dbt test"
dbt test
if [ $? -ne 0 ]; then
  has_any_step_failed=1
fi

# === Handle success ===
if [ $has_any_step_failed -eq 0 ]; then
  curl -X POST -H 'Content-type: application/json' \
    --data "{\"text\":\"$slack_success_message\"}" \
    "$SLACK_RECEIPT_WEBHOOK_URL"
  exit 0
fi

# === Handle failures: parse log and send individual Slack messages ===
echo "Parsing log file for test failures..."

grep -E "Failure in test|Got [0-9]+ result|compiled code at" "$LOG_FILE" | while read -r line; do
  if [[ "$line" =~ Failure\ in\ test\ ([^[:space:]]+)\ \((.*)\) ]]; then
    TEST_NAME="${BASH_REMATCH[1]}"
    echo "==> Detected failure: $TEST_NAME"
  fi

  if [[ "$line" =~ Got\ ([0-9]+)\ result ]]; then
    FAILED_ROWS="${BASH_REMATCH[1]}"

  fi

  if [[ "$line" =~ compiled\ code\ at\ (.*) ]]; then
    RELATIVE_PATH="${BASH_REMATCH[1]}"
    COMPILED_SQL_FILE="/home/azureuser/data-dwh-dbt-project/${RELATIVE_PATH}"
    # Check sqlfluff availability
    if ! command -v sqlfluff >/dev/null 2>&1; then
      echo "ERROR: sqlfluff is not installed or not in PATH"
      SQL_QUERY="sqlfluff not found on system"
    elif [ -f "$COMPILED_SQL_FILE" ]; then
      echo "File exists, attempting to format with sqlfluff..."
      FORMATTED_SQL=$(sqlfluff render "$COMPILED_SQL_FILE" --dialect postgres 2>&1)
      if [ -n "$FORMATTED_SQL" ]; then
        echo "We have formatted SQL"
        SQL_QUERY=$(echo "$FORMATTED_SQL" | sed 's/"/\\"/g')
      else
        echo "sqlfluff returned empty result, falling back to raw file content"
        SQL_QUERY=$(<"$COMPILED_SQL_FILE" sed 's/"/\\"/g')
      fi
    else
      echo "ERROR: File not found: $COMPILED_SQL_FILE"
      SQL_QUERY="Could not find compiled SQL file: $COMPILED_SQL_FILE"
    fi

    # === Send Slack message for this failed test ===
    echo "Sending message for failed test $TEST_NAME"
    SLACK_MESSAGE=":rotating_light: *Test Failure Detected!* :rotating_light:\n\n*Test:* \`$TEST_NAME\`\n*Failed Rows:* $FAILED_ROWS\n*Query:*\n\`\`\`\n$SQL_QUERY\n\`\`\`"

    curl -X POST -H 'Content-type: application/json' \
      --data "{\"text\":\"$SLACK_MESSAGE\"}" \
      "$SLACK_ALERT_WEBHOOK_URL"
  fi
done