Merged PR 5678: Revert 'Prettify alerts in test script'

# Description

We revert this script to its previous step due to it being too fragile in the current implementation and us not having capacity to make it robust enough right now.

Reverts !5551

Related work items: #31476
This commit is contained in:
Pablo Martin 2025-07-11 09:20:01 +00:00 committed by Oriol Roqué Paniagua
parent 2f14b3305c
commit ddc0a6a3f4

View file

@ -1,17 +1,11 @@
#!/bin/bash
# === Logging setup ===
TIMESTAMP=$(date +"%Y%m%d_%H%M%S")
LOG_FILE="/home/azureuser/dbt_test_logs/dbt_tests_${TIMESTAMP}.log"
exec >> "$LOG_FILE" 2>&1
exec >> /home/azureuser/dbt_tests.log 2>&1
echo "=== dbt test run started at $TIMESTAMP ==="
# === Slack webhook setup ===
# Define the Slack webhook URL
script_dir=$(dirname "$0")
webhooks_file="slack_webhook_urls.txt"
env_file="$script_dir/$webhooks_file"
if [ -f "$env_file" ]; then
export $(grep -v '^#' "$env_file" | xargs)
else
@ -19,79 +13,34 @@ else
exit 1
fi
# Messages to be sent to Slack
slack_failure_message=":rotating_light::rotating_light::rotating_light: One or more failures in dbt tests in production. :rotating_light::rotating_light::rotating_light:"
slack_success_message=":white_check_mark::white_check_mark::white_check_mark: dbt tests executed successfully in production. :white_check_mark::white_check_mark::white_check_mark:"
# Initialize the failure flag
has_any_step_failed=0
# === Navigate to project ===
cd /home/azureuser/data-dwh-dbt-project || exit 1
cd /home/azureuser/data-dwh-dbt-project
# === Update from Git ===
# Update from git
echo "Updating dbt project from git."
git checkout master
git pull
# === Activate virtual environment ===
# Activate venv
source venv/bin/activate
# === Run dbt tests ===
# Run tests
echo "Triggering dbt test"
dbt test
if [ $? -ne 0 ]; then
has_any_step_failed=1
fi
# === Handle success ===
if [ $has_any_step_failed -eq 0 ]; then
curl -X POST -H 'Content-type: application/json' \
--data "{\"text\":\"$slack_success_message\"}" \
"$SLACK_RECEIPT_WEBHOOK_URL"
exit 0
# Check if any step failed and send a Slack message
if [ $has_any_step_failed -eq 1 ]; then
curl -X POST -H 'Content-type: application/json' --data "{\"text\":\"$slack_failure_message\"}" $SLACK_ALERT_WEBHOOK_URL
fi
if [ $has_any_step_failed -eq 0 ]; then
curl -X POST -H 'Content-type: application/json' --data "{\"text\":\"$slack_success_message\"}" $SLACK_RECEIPT_WEBHOOK_URL
fi
# === Handle failures: parse log and send individual Slack messages ===
echo "Parsing log file for test failures..."
grep -E "Failure in test|Got [0-9]+ result|compiled code at" "$LOG_FILE" | while read -r line; do
if [[ "$line" =~ Failure\ in\ test\ ([^[:space:]]+)\ \((.*)\) ]]; then
TEST_NAME="${BASH_REMATCH[1]}"
echo "==> Detected failure: $TEST_NAME"
fi
if [[ "$line" =~ Got\ ([0-9]+)\ result ]]; then
FAILED_ROWS="${BASH_REMATCH[1]}"
fi
if [[ "$line" =~ compiled\ code\ at\ (.*) ]]; then
RELATIVE_PATH="${BASH_REMATCH[1]}"
COMPILED_SQL_FILE="/home/azureuser/data-dwh-dbt-project/${RELATIVE_PATH}"
# Check sqlfluff availability
if ! command -v sqlfluff >/dev/null 2>&1; then
echo "ERROR: sqlfluff is not installed or not in PATH"
SQL_QUERY="sqlfluff not found on system"
elif [ -f "$COMPILED_SQL_FILE" ]; then
echo "File exists, attempting to format with sqlfluff..."
FORMATTED_SQL=$(sqlfluff render "$COMPILED_SQL_FILE" --dialect postgres 2>&1)
if [ -n "$FORMATTED_SQL" ]; then
echo "We have formatted SQL"
SQL_QUERY=$(echo "$FORMATTED_SQL" | sed 's/"/\\"/g')
else
echo "sqlfluff returned empty result, falling back to raw file content"
SQL_QUERY=$(<"$COMPILED_SQL_FILE" sed 's/"/\\"/g')
fi
else
echo "ERROR: File not found: $COMPILED_SQL_FILE"
SQL_QUERY="Could not find compiled SQL file: $COMPILED_SQL_FILE"
fi
# === Send Slack message for this failed test ===
echo "Sending message for failed test $TEST_NAME"
SLACK_MESSAGE=":rotating_light: *Test Failure Detected!* :rotating_light:\n\n*Test:* \`$TEST_NAME\`\n*Failed Rows:* $FAILED_ROWS\n*Query:*\n\`\`\`\n$SQL_QUERY\n\`\`\`"
curl -X POST -H 'Content-type: application/json' \
--data "{\"text\":\"$SLACK_MESSAGE\"}" \
"$SLACK_ALERT_WEBHOOK_URL"
fi
done