counterweight/data-dwh-dbt-project
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

add failure states, .env file and message sending

Browse source
This commit is contained in:
Pablo Martin 2024-05-30 16:12:51 +02:00
parent 307887126c
commit b1aa29b228
1 changed files with 36 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

36
run_dbt.sh
View file

@ -1,5 +1,22 @@
#!/bin/bash
# Define the Slack webhook URL and message text
script_dir=$(dirname "$0")
env_file="$script_dir/slack_webhook_urls.txt"
if [ -f "$env_file" ]; then
export $(grep -v '^#' "$env_file" | xargs)
else
echo "Error: slack_webhook_urls.txt file not found in the script directory."
exit 1
fi
# Messages to be sent to Slack
slack_failure_message=":siren::siren::siren: One or more failures during scheduled dbt run in production. :siren::siren::siren:"
slack_success_message=":white_check_mark::white_check_mark::white_check_mark: dbt run executed successfully in production. :white_check_mark::white_check_mark::white_check_mark:"
# Initialize the failure flag
has_any_step_failed=0
cd /home/azureuser/data-dwh-dbt-project
# Update from git
@ -13,15 +30,34 @@ source venv/bin/activate
# Run seeds
echo "Triggering dbt seed" | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1
dbt seed | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1
if [ $? -ne 0 ]; then
has_any_step_failed=1
fi
# Run staging layer
echo "Triggering dbt run: Staging" | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1
dbt run -s models/staging | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1
if [ $? -ne 0 ]; then
has_any_step_failed=1
fi
# Run intermediate layer
echo "Triggering dbt run: Intermediate" | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1
dbt run -s models/intermediate | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1
if [ $? -ne 0 ]; then
has_any_step_failed=1
fi
# Run reporting layer
echo "Triggering dbt run: Reporting" | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1
dbt run -s models/reporting | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1
if [ $? -ne 0 ]; then
has_any_step_failed=1
fi
# Check if any step failed and send a Slack message
if [ $has_any_step_failed -eq 1 ]; then
curl -X POST -H 'Content-type: application/json' --data "{\"text\":\"$slack_failure_message\"}" $SLACK_WEBHOOK_URL
else
curl -X POST -H 'Content-type: application/json' --data "{\"text\":\"$slack_success_message\"}" $SLACK_WEBHOOK_URL
fi