diff --git a/run_tests.sh b/run_tests.sh index 7060001..a64ff45 100644 --- a/run_tests.sh +++ b/run_tests.sh @@ -1,17 +1,11 @@ #!/bin/bash -# === Logging setup === -TIMESTAMP=$(date +"%Y%m%d_%H%M%S") -LOG_FILE="/home/azureuser/dbt_test_logs/dbt_tests_${TIMESTAMP}.log" -exec >> "$LOG_FILE" 2>&1 +exec >> /home/azureuser/dbt_tests.log 2>&1 -echo "=== dbt test run started at $TIMESTAMP ===" - -# === Slack webhook setup === +# Define the Slack webhook URL script_dir=$(dirname "$0") webhooks_file="slack_webhook_urls.txt" env_file="$script_dir/$webhooks_file" - if [ -f "$env_file" ]; then export $(grep -v '^#' "$env_file" | xargs) else @@ -19,79 +13,34 @@ else exit 1 fi +# Messages to be sent to Slack slack_failure_message=":rotating_light::rotating_light::rotating_light: One or more failures in dbt tests in production. :rotating_light::rotating_light::rotating_light:" slack_success_message=":white_check_mark::white_check_mark::white_check_mark: dbt tests executed successfully in production. :white_check_mark::white_check_mark::white_check_mark:" +# Initialize the failure flag has_any_step_failed=0 -# === Navigate to project === -cd /home/azureuser/data-dwh-dbt-project || exit 1 +cd /home/azureuser/data-dwh-dbt-project -# === Update from Git === +# Update from git echo "Updating dbt project from git." git checkout master git pull -# === Activate virtual environment === +# Activate venv source venv/bin/activate -# === Run dbt tests === +# Run tests echo "Triggering dbt test" dbt test if [ $? -ne 0 ]; then has_any_step_failed=1 fi -# === Handle success === -if [ $has_any_step_failed -eq 0 ]; then - curl -X POST -H 'Content-type: application/json' \ - --data "{\"text\":\"$slack_success_message\"}" \ - "$SLACK_RECEIPT_WEBHOOK_URL" - exit 0 +# Check if any step failed and send a Slack message +if [ $has_any_step_failed -eq 1 ]; then + curl -X POST -H 'Content-type: application/json' --data "{\"text\":\"$slack_failure_message\"}" $SLACK_ALERT_WEBHOOK_URL fi - -# === Handle failures: parse log and send individual Slack messages === -echo "Parsing log file for test failures..." - -grep -E "Failure in test|Got [0-9]+ result|compiled code at" "$LOG_FILE" | while read -r line; do - if [[ "$line" =~ Failure\ in\ test\ ([^[:space:]]+)\ \((.*)\) ]]; then - TEST_NAME="${BASH_REMATCH[1]}" - echo "==> Detected failure: $TEST_NAME" - fi - - if [[ "$line" =~ Got\ ([0-9]+)\ result ]]; then - FAILED_ROWS="${BASH_REMATCH[1]}" - - fi - - if [[ "$line" =~ compiled\ code\ at\ (.*) ]]; then - RELATIVE_PATH="${BASH_REMATCH[1]}" - COMPILED_SQL_FILE="/home/azureuser/data-dwh-dbt-project/${RELATIVE_PATH}" - # Check sqlfluff availability - if ! command -v sqlfluff >/dev/null 2>&1; then - echo "ERROR: sqlfluff is not installed or not in PATH" - SQL_QUERY="sqlfluff not found on system" - elif [ -f "$COMPILED_SQL_FILE" ]; then - echo "File exists, attempting to format with sqlfluff..." - FORMATTED_SQL=$(sqlfluff render "$COMPILED_SQL_FILE" --dialect postgres 2>&1) - if [ -n "$FORMATTED_SQL" ]; then - echo "We have formatted SQL" - SQL_QUERY=$(echo "$FORMATTED_SQL" | sed 's/"/\\"/g') - else - echo "sqlfluff returned empty result, falling back to raw file content" - SQL_QUERY=$(<"$COMPILED_SQL_FILE" sed 's/"/\\"/g') - fi - else - echo "ERROR: File not found: $COMPILED_SQL_FILE" - SQL_QUERY="Could not find compiled SQL file: $COMPILED_SQL_FILE" - fi - - # === Send Slack message for this failed test === - echo "Sending message for failed test $TEST_NAME" - SLACK_MESSAGE=":rotating_light: *Test Failure Detected!* :rotating_light:\n\n*Test:* \`$TEST_NAME\`\n*Failed Rows:* $FAILED_ROWS\n*Query:*\n\`\`\`\n$SQL_QUERY\n\`\`\`" - - curl -X POST -H 'Content-type: application/json' \ - --data "{\"text\":\"$SLACK_MESSAGE\"}" \ - "$SLACK_ALERT_WEBHOOK_URL" - fi -done \ No newline at end of file +if [ $has_any_step_failed -eq 0 ]; then + curl -X POST -H 'Content-type: application/json' --data "{\"text\":\"$slack_success_message\"}" $SLACK_RECEIPT_WEBHOOK_URL +fi \ No newline at end of file