65 lines
No EOL
2.9 KiB
Bash
65 lines
No EOL
2.9 KiB
Bash
#!/bin/bash
|
|
|
|
# Define the Slack webhook URL
|
|
script_dir=$(dirname "$0")
|
|
webhooks_file="slack_webhook_urls.txt"
|
|
env_file="$script_dir/$webhooks_file"
|
|
if [ -f "$env_file" ]; then
|
|
export $(grep -v '^#' "$env_file" | xargs)
|
|
else
|
|
echo "Error: $webhooks_file file not found in the script directory."
|
|
exit 1
|
|
fi
|
|
|
|
# Messages to be sent to Slack
|
|
slack_failure_message=":rotating_light::rotating_light::rotating_light: One or more failures during scheduled dbt run in production. :rotating_light::rotating_light::rotating_light:"
|
|
slack_success_message=":white_check_mark::white_check_mark::white_check_mark: dbt run executed successfully in production. :white_check_mark::white_check_mark::white_check_mark:"
|
|
|
|
# Initialize the failure flag
|
|
has_any_step_failed=0
|
|
|
|
cd /home/azureuser/data-dwh-dbt-project
|
|
|
|
# Update from git
|
|
echo "Updating dbt project from git." | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1
|
|
git checkout master | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1
|
|
git pull | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1
|
|
|
|
# Activate venv
|
|
source venv/bin/activate
|
|
|
|
# Run seeds
|
|
echo "Triggering dbt seed" | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1
|
|
dbt seed | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1
|
|
if [ $? -ne 0 ]; then
|
|
has_any_step_failed=1
|
|
fi
|
|
|
|
# Run staging layer
|
|
echo "Triggering dbt run: Staging" | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1
|
|
dbt run -s models/staging | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1
|
|
if [ $? -ne 0 ]; then
|
|
has_any_step_failed=1
|
|
fi
|
|
|
|
# Run intermediate layer
|
|
echo "Triggering dbt run: Intermediate" | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1
|
|
dbt run -s models/intermediate | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1
|
|
if [ $? -ne 0 ]; then
|
|
has_any_step_failed=1
|
|
fi
|
|
|
|
# Run reporting layer
|
|
echo "Triggering dbt run: Reporting" | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1
|
|
dbt run -s models/reporting | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1
|
|
if [ $? -ne 0 ]; then
|
|
has_any_step_failed=1
|
|
fi
|
|
|
|
# Check if any step failed and send a Slack message
|
|
if [ $has_any_step_failed -eq 1 ]; then
|
|
curl -X POST -H 'Content-type: application/json' --data "{\"text\":\"$slack_failure_message\"}" $SLACK_ALERT_WEBHOOK_URL
|
|
fi
|
|
if [ $has_any_step_failed -eq 0 ]; then
|
|
curl -X POST -H 'Content-type: application/json' --data "{\"text\":\"$slack_success_message\"}" $SLACK_RECEIPT_WEBHOOK_URL
|
|
fi |