data-dwh-dbt-project/run_dbt.sh

#!/bin/bash

# Define the Slack webhook URL and message text
script_dir=$(dirname "$0")
env_file="$script_dir/slack_webhook_urls.txt"
if [ -f "$env_file" ]; then
  export $(grep -v '^#' "$env_file" | xargs)
else
  echo "Error: slack_webhook_urls.txt file not found in the script directory."
  exit 1
fi

# Messages to be sent to Slack
slack_failure_message=":siren::siren::siren: One or more failures during scheduled dbt run in production. :siren::siren::siren:"
slack_success_message=":white_check_mark::white_check_mark::white_check_mark: dbt run executed successfully in production. :white_check_mark::white_check_mark::white_check_mark:"

# Initialize the failure flag
has_any_step_failed=0

cd /home/azureuser/data-dwh-dbt-project

# Update from git
echo "Updating dbt project from git." | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1
git checkout master | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1
git pull | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1

# Activate venv
source venv/bin/activate

# Run seeds
echo "Triggering dbt seed" | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1
dbt seed | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1
if [ $? -ne 0 ]; then
  has_any_step_failed=1
fi

# Run staging layer
echo "Triggering dbt run: Staging" | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1
dbt run -s models/staging | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1
if [ $? -ne 0 ]; then
  has_any_step_failed=1
fi

# Run intermediate layer
echo "Triggering dbt run: Intermediate" | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1
dbt run -s models/intermediate | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1
if [ $? -ne 0 ]; then
  has_any_step_failed=1
fi

# Run reporting layer
echo "Triggering dbt run: Reporting" | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1
dbt run -s models/reporting | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1
if [ $? -ne 0 ]; then
  has_any_step_failed=1
fi

# Check if any step failed and send a Slack message
if [ $has_any_step_failed -eq 1 ]; then
  curl -X POST -H 'Content-type: application/json' --data "{\"text\":\"$slack_failure_message\"}" $SLACK_WEBHOOK_URL
else
  curl -X POST -H 'Content-type: application/json' --data "{\"text\":\"$slack_success_message\"}" $SLACK_WEBHOOK_URL
fi