Merged PR 5551: Prettify alerts in test script

# Description

This PR modifies the test script messaging towards slack. Until now, when tests failed, we simply sent a general message to slack saying something failed. With this PR, we will send one message per test, detailing the table and test, number of bad records, and the compiled test's SQL query that can help us debug.

On testing success, the output will still be a simple success message.

# Checklist

NA

# Other

NA

Related work items: #31476

run_tests.sh

@@ -1,11 +1,17 @@
 #!/bin/bash
 
-exec >> /home/azureuser/dbt_tests.log 2>&1
+# === Logging setup ===
+TIMESTAMP=$(date +"%Y%m%d_%H%M%S")
+LOG_FILE="/home/azureuser/dbt_test_logs/dbt_tests_${TIMESTAMP}.log"
+exec >> "$LOG_FILE" 2>&1
 
-# Define the Slack webhook URL
+echo "=== dbt test run started at $TIMESTAMP ==="
+
+# === Slack webhook setup ===
 script_dir=$(dirname "$0")
 webhooks_file="slack_webhook_urls.txt"
 env_file="$script_dir/$webhooks_file"
+
 if [ -f "$env_file" ]; then
   export $(grep -v '^#' "$env_file" | xargs)
 else
@@ -13,34 +19,79 @@ else
   exit 1
 fi
 
-# Messages to be sent to Slack
 slack_failure_message=":rotating_light::rotating_light::rotating_light: One or more failures in dbt tests in production. :rotating_light::rotating_light::rotating_light:"
 slack_success_message=":white_check_mark::white_check_mark::white_check_mark: dbt tests executed successfully in production. :white_check_mark::white_check_mark::white_check_mark:"
 
-# Initialize the failure flag
 has_any_step_failed=0
 
-cd /home/azureuser/data-dwh-dbt-project
+# === Navigate to project ===
+cd /home/azureuser/data-dwh-dbt-project || exit 1
 
-# Update from git
+# === Update from Git ===
 echo "Updating dbt project from git."
 git checkout master
 git pull
 
-# Activate venv
+# === Activate virtual environment ===
 source venv/bin/activate
 
-# Run tests
+# === Run dbt tests ===
 echo "Triggering dbt test"
 dbt test
 if [ $? -ne 0 ]; then
   has_any_step_failed=1
 fi
 
-# Check if any step failed and send a Slack message
-if [ $has_any_step_failed -eq 1 ]; then
-  curl -X POST -H 'Content-type: application/json' --data "{\"text\":\"$slack_failure_message\"}" $SLACK_ALERT_WEBHOOK_URL
-fi
+# === Handle success ===
 if [ $has_any_step_failed -eq 0 ]; then
-  curl -X POST -H 'Content-type: application/json' --data "{\"text\":\"$slack_success_message\"}" $SLACK_RECEIPT_WEBHOOK_URL
-fi
+  curl -X POST -H 'Content-type: application/json' \
+    --data "{\"text\":\"$slack_success_message\"}" \
+    "$SLACK_RECEIPT_WEBHOOK_URL"
+  exit 0
+fi
+
+# === Handle failures: parse log and send individual Slack messages ===
+echo "Parsing log file for test failures..."
+
+grep -E "Failure in test|Got [0-9]+ result|compiled code at" "$LOG_FILE" | while read -r line; do
+  if [[ "$line" =~ Failure\ in\ test\ ([^[:space:]]+)\ \((.*)\) ]]; then
+    TEST_NAME="${BASH_REMATCH[1]}"
+    echo "==> Detected failure: $TEST_NAME"
+  fi
+
+  if [[ "$line" =~ Got\ ([0-9]+)\ result ]]; then
+    FAILED_ROWS="${BASH_REMATCH[1]}"
+
+  fi
+
+  if [[ "$line" =~ compiled\ code\ at\ (.*) ]]; then
+    RELATIVE_PATH="${BASH_REMATCH[1]}"
+    COMPILED_SQL_FILE="/home/azureuser/data-dwh-dbt-project/${RELATIVE_PATH}"
+    # Check sqlfluff availability
+    if ! command -v sqlfluff >/dev/null 2>&1; then
+      echo "ERROR: sqlfluff is not installed or not in PATH"
+      SQL_QUERY="sqlfluff not found on system"
+    elif [ -f "$COMPILED_SQL_FILE" ]; then
+      echo "File exists, attempting to format with sqlfluff..."
+      FORMATTED_SQL=$(sqlfluff render "$COMPILED_SQL_FILE" --dialect postgres 2>&1)
+      if [ -n "$FORMATTED_SQL" ]; then
+        echo "We have formatted SQL"
+        SQL_QUERY=$(echo "$FORMATTED_SQL" | sed 's/"/\\"/g')
+      else
+        echo "sqlfluff returned empty result, falling back to raw file content"
+        SQL_QUERY=$(<"$COMPILED_SQL_FILE" sed 's/"/\\"/g')
+      fi
+    else
+      echo "ERROR: File not found: $COMPILED_SQL_FILE"
+      SQL_QUERY="Could not find compiled SQL file: $COMPILED_SQL_FILE"
+    fi
+
+    # === Send Slack message for this failed test ===
+    echo "Sending message for failed test $TEST_NAME"
+    SLACK_MESSAGE=":rotating_light: *Test Failure Detected!* :rotating_light:\n\n*Test:* \`$TEST_NAME\`\n*Failed Rows:* $FAILED_ROWS\n*Query:*\n\`\`\`\n$SQL_QUERY\n\`\`\`"
+
+    curl -X POST -H 'Content-type: application/json' \
+      --data "{\"text\":\"$SLACK_MESSAGE\"}" \
+      "$SLACK_ALERT_WEBHOOK_URL"
+  fi
+done