counterweight/data-dwh-dbt-project
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merged PR 2644: Fix deployment alerts

Browse source 
# Description

This PR fixes a bug in the deployment script. The bug makes the alerts not work as expected, because the bash piping will trigger alerts if the log writing of commands fail, instead of if commands fail, which is what is expected.

This PR modifies the way logging works. Logging remains the same, but alerts now will work as expected.

Related work items: #18183
This commit is contained in:
Pablo Martín 2024-08-26 14:19:47 +00:00
commit 46ff9514a6
1 changed files with 13 additions and 11 deletions
Download patch file Download diff file Expand all files Collapse all files

24
run_dbt.sh
View file

@ -1,5 +1,7 @@
#!/bin/bash
exec >> /home/azureuser/dbt_run.log 2>&1
# Define the Slack webhook URL
script_dir=$(dirname "$0")
webhooks_file="slack_webhook_urls.txt"
@ -21,37 +23,37 @@ has_any_step_failed=0
cd /home/azureuser/data-dwh-dbt-project
# Update from git
echo "Updating dbt project from git." | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1
git checkout master | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1
git pull | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1
echo "Updating dbt project from git."
git checkout master
git pull
# Activate venv
source venv/bin/activate
# Run seeds
echo "Triggering dbt seed" | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1
dbt seed | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1
echo "Triggering dbt seed"
dbt seed
if [ $? -ne 0 ]; then
has_any_step_failed=1
fi
# Run staging layer
echo "Triggering dbt run: Staging" | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1
dbt run -s models/staging | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1
echo "Triggering dbt run: Staging"
dbt run -s models/staging
if [ $? -ne 0 ]; then
has_any_step_failed=1
fi
# Run intermediate layer
echo "Triggering dbt run: Intermediate" | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1
dbt run -s models/intermediate | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1
echo "Triggering dbt run: Intermediate"
dbt run -s models/intermediate
if [ $? -ne 0 ]; then
has_any_step_failed=1
fi
# Run reporting layer
echo "Triggering dbt run: Reporting" | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1
dbt run -s models/reporting | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1
echo "Triggering dbt run: Reporting"
dbt run -s models/reporting
if [ $? -ne 0 ]; then
has_any_step_failed=1
fi