97 lines
No EOL
3.3 KiB
Bash
97 lines
No EOL
3.3 KiB
Bash
#!/bin/bash
|
|
|
|
# === Logging setup ===
|
|
TIMESTAMP=$(date +"%Y%m%d_%H%M%S")
|
|
LOG_FILE="/home/azureuser/dbt_test_logs/dbt_tests_${TIMESTAMP}.log"
|
|
exec >> "$LOG_FILE" 2>&1
|
|
|
|
echo "=== dbt test run started at $TIMESTAMP ==="
|
|
|
|
# === Slack webhook setup ===
|
|
script_dir=$(dirname "$0")
|
|
webhooks_file="slack_webhook_urls.txt"
|
|
env_file="$script_dir/$webhooks_file"
|
|
|
|
if [ -f "$env_file" ]; then
|
|
export $(grep -v '^#' "$env_file" | xargs)
|
|
else
|
|
echo "Error: $webhooks_file file not found in the script directory."
|
|
exit 1
|
|
fi
|
|
|
|
slack_failure_message=":rotating_light::rotating_light::rotating_light: One or more failures in dbt tests in production. :rotating_light::rotating_light::rotating_light:"
|
|
slack_success_message=":white_check_mark::white_check_mark::white_check_mark: dbt tests executed successfully in production. :white_check_mark::white_check_mark::white_check_mark:"
|
|
|
|
has_any_step_failed=0
|
|
|
|
# === Navigate to project ===
|
|
cd /home/azureuser/data-dwh-dbt-project || exit 1
|
|
|
|
# === Update from Git ===
|
|
echo "Updating dbt project from git."
|
|
git checkout master
|
|
git pull
|
|
|
|
# === Activate virtual environment ===
|
|
source venv/bin/activate
|
|
|
|
# === Run dbt tests ===
|
|
echo "Triggering dbt test"
|
|
dbt test
|
|
if [ $? -ne 0 ]; then
|
|
has_any_step_failed=1
|
|
fi
|
|
|
|
# === Handle success ===
|
|
if [ $has_any_step_failed -eq 0 ]; then
|
|
curl -X POST -H 'Content-type: application/json' \
|
|
--data "{\"text\":\"$slack_success_message\"}" \
|
|
"$SLACK_RECEIPT_WEBHOOK_URL"
|
|
exit 0
|
|
fi
|
|
|
|
# === Handle failures: parse log and send individual Slack messages ===
|
|
echo "Parsing log file for test failures..."
|
|
|
|
grep -E "Failure in test|Got [0-9]+ result|compiled code at" "$LOG_FILE" | while read -r line; do
|
|
if [[ "$line" =~ Failure\ in\ test\ ([^[:space:]]+)\ \((.*)\) ]]; then
|
|
TEST_NAME="${BASH_REMATCH[1]}"
|
|
echo "==> Detected failure: $TEST_NAME"
|
|
fi
|
|
|
|
if [[ "$line" =~ Got\ ([0-9]+)\ result ]]; then
|
|
FAILED_ROWS="${BASH_REMATCH[1]}"
|
|
|
|
fi
|
|
|
|
if [[ "$line" =~ compiled\ code\ at\ (.*) ]]; then
|
|
RELATIVE_PATH="${BASH_REMATCH[1]}"
|
|
COMPILED_SQL_FILE="/home/azureuser/data-dwh-dbt-project/${RELATIVE_PATH}"
|
|
# Check sqlfluff availability
|
|
if ! command -v sqlfluff >/dev/null 2>&1; then
|
|
echo "ERROR: sqlfluff is not installed or not in PATH"
|
|
SQL_QUERY="sqlfluff not found on system"
|
|
elif [ -f "$COMPILED_SQL_FILE" ]; then
|
|
echo "File exists, attempting to format with sqlfluff..."
|
|
FORMATTED_SQL=$(sqlfluff render "$COMPILED_SQL_FILE" --dialect postgres 2>&1)
|
|
if [ -n "$FORMATTED_SQL" ]; then
|
|
echo "We have formatted SQL"
|
|
SQL_QUERY=$(echo "$FORMATTED_SQL" | sed 's/"/\\"/g')
|
|
else
|
|
echo "sqlfluff returned empty result, falling back to raw file content"
|
|
SQL_QUERY=$(<"$COMPILED_SQL_FILE" sed 's/"/\\"/g')
|
|
fi
|
|
else
|
|
echo "ERROR: File not found: $COMPILED_SQL_FILE"
|
|
SQL_QUERY="Could not find compiled SQL file: $COMPILED_SQL_FILE"
|
|
fi
|
|
|
|
# === Send Slack message for this failed test ===
|
|
echo "Sending message for failed test $TEST_NAME"
|
|
SLACK_MESSAGE=":rotating_light: *Test Failure Detected!* :rotating_light:\n\n*Test:* \`$TEST_NAME\`\n*Failed Rows:* $FAILED_ROWS\n*Query:*\n\`\`\`\n$SQL_QUERY\n\`\`\`"
|
|
|
|
curl -X POST -H 'Content-type: application/json' \
|
|
--data "{\"text\":\"$SLACK_MESSAGE\"}" \
|
|
"$SLACK_ALERT_WEBHOOK_URL"
|
|
fi
|
|
done |