diff --git a/run_dbt.sh b/run_dbt.sh index b29255c..95b3c11 100644 --- a/run_dbt.sh +++ b/run_dbt.sh @@ -1,5 +1,7 @@ #!/bin/bash +exec >> /home/azureuser/dbt_run.log 2>&1 + # Define the Slack webhook URL script_dir=$(dirname "$0") webhooks_file="slack_webhook_urls.txt" @@ -21,37 +23,37 @@ has_any_step_failed=0 cd /home/azureuser/data-dwh-dbt-project # Update from git -echo "Updating dbt project from git." | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1 -git checkout master | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1 -git pull | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1 +echo "Updating dbt project from git." +git checkout master +git pull # Activate venv source venv/bin/activate # Run seeds -echo "Triggering dbt seed" | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1 -dbt seed | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1 +echo "Triggering dbt seed" +dbt seed if [ $? -ne 0 ]; then has_any_step_failed=1 fi # Run staging layer -echo "Triggering dbt run: Staging" | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1 -dbt run -s models/staging | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1 +echo "Triggering dbt run: Staging" +dbt run -s models/staging if [ $? -ne 0 ]; then has_any_step_failed=1 fi # Run intermediate layer -echo "Triggering dbt run: Intermediate" | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1 -dbt run -s models/intermediate | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1 +echo "Triggering dbt run: Intermediate" +dbt run -s models/intermediate if [ $? -ne 0 ]; then has_any_step_failed=1 fi # Run reporting layer -echo "Triggering dbt run: Reporting" | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1 -dbt run -s models/reporting | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1 +echo "Triggering dbt run: Reporting" +dbt run -s models/reporting if [ $? -ne 0 ]; then has_any_step_failed=1 fi