modify test script

run_tests.sh

@@ -1,11 +1,17 @@
 #!/bin/bash
 
-exec >> /home/azureuser/dbt_tests.log 2>&1
+# === Logging setup ===
+TIMESTAMP=$(date +"%Y%m%d_%H%M%S")
+LOG_FILE="/home/azureuser/dbt_test_logs/dbt_tests_${TIMESTAMP}.log"
+exec >> "$LOG_FILE" 2>&1
 
-# Define the Slack webhook URL
+echo "=== dbt test run started at $TIMESTAMP ==="
+
+# === Slack webhook setup ===
 script_dir=$(dirname "$0")
 webhooks_file="slack_webhook_urls.txt"
 env_file="$script_dir/$webhooks_file"
+
 if [ -f "$env_file" ]; then
   export $(grep -v '^#' "$env_file" | xargs)
 else
@@ -13,34 +19,79 @@ else
   exit 1
 fi
 
-# Messages to be sent to Slack
 slack_failure_message=":rotating_light::rotating_light::rotating_light: One or more failures in dbt tests in production. :rotating_light::rotating_light::rotating_light:"
 slack_success_message=":white_check_mark::white_check_mark::white_check_mark: dbt tests executed successfully in production. :white_check_mark::white_check_mark::white_check_mark:"
 
-# Initialize the failure flag
 has_any_step_failed=0
 
-cd /home/azureuser/data-dwh-dbt-project
+# === Navigate to project ===
+cd /home/azureuser/data-dwh-dbt-project || exit 1
 
-# Update from git
+# === Update from Git ===
 echo "Updating dbt project from git."
 git checkout master
 git pull
 
-# Activate venv
+# === Activate virtual environment ===
 source venv/bin/activate
 
-# Run tests
+# === Run dbt tests ===
 echo "Triggering dbt test"
 dbt test
 if [ $? -ne 0 ]; then
   has_any_step_failed=1
 fi
 
-# Check if any step failed and send a Slack message
-if [ $has_any_step_failed -eq 1 ]; then
-  curl -X POST -H 'Content-type: application/json' --data "{\"text\":\"$slack_failure_message\"}" $SLACK_ALERT_WEBHOOK_URL
-fi
+# === Handle success ===
 if [ $has_any_step_failed -eq 0 ]; then
-  curl -X POST -H 'Content-type: application/json' --data "{\"text\":\"$slack_success_message\"}" $SLACK_RECEIPT_WEBHOOK_URL
-fi
+  curl -X POST -H 'Content-type: application/json' \
+    --data "{\"text\":\"$slack_success_message\"}" \
+    "$SLACK_RECEIPT_WEBHOOK_URL"
+  exit 0
+fi
+
+# === Handle failures: parse log and send individual Slack messages ===
+echo "Parsing log file for test failures..."
+
+grep -E "Failure in test|Got [0-9]+ result|compiled code at" "$LOG_FILE" | while read -r line; do
+  if [[ "$line" =~ Failure\ in\ test\ ([^[:space:]]+)\ \((.*)\) ]]; then
+    TEST_NAME="${BASH_REMATCH[1]}"
+    echo "==> Detected failure: $TEST_NAME"
+  fi
+
+  if [[ "$line" =~ Got\ ([0-9]+)\ result ]]; then
+    FAILED_ROWS="${BASH_REMATCH[1]}"
+
+  fi
+
+  if [[ "$line" =~ compiled\ code\ at\ (.*) ]]; then
+    RELATIVE_PATH="${BASH_REMATCH[1]}"
+    COMPILED_SQL_FILE="/home/azureuser/data-dwh-dbt-project/${RELATIVE_PATH}"
+    # Check sqlfluff availability
+    if ! command -v sqlfluff >/dev/null 2>&1; then
+      echo "ERROR: sqlfluff is not installed or not in PATH"
+      SQL_QUERY="sqlfluff not found on system"
+    elif [ -f "$COMPILED_SQL_FILE" ]; then
+      echo "File exists, attempting to format with sqlfluff..."
+      FORMATTED_SQL=$(sqlfluff render "$COMPILED_SQL_FILE" --dialect postgres 2>&1)
+      if [ -n "$FORMATTED_SQL" ]; then
+        echo "We have formatted SQL"
+        SQL_QUERY=$(echo "$FORMATTED_SQL" | sed 's/"/\\"/g')
+      else
+        echo "sqlfluff returned empty result, falling back to raw file content"
+        SQL_QUERY=$(<"$COMPILED_SQL_FILE" sed 's/"/\\"/g')
+      fi
+    else
+      echo "ERROR: File not found: $COMPILED_SQL_FILE"
+      SQL_QUERY="Could not find compiled SQL file: $COMPILED_SQL_FILE"
+    fi
+
+    # === Send Slack message for this failed test ===
+    echo "Sending message for failed test $TEST_NAME"
+    SLACK_MESSAGE=":rotating_light: *Test Failure Detected!* :rotating_light:\n\n*Test:* \`$TEST_NAME\`\n*Failed Rows:* $FAILED_ROWS\n*Query:*\n\`\`\`\n$SQL_QUERY\n\`\`\`"
+
+    curl -X POST -H 'Content-type: application/json' \
+      --data "{\"text\":\"$SLACK_MESSAGE\"}" \
+      "$SLACK_ALERT_WEBHOOK_URL"
+  fi
+done