counterweight/data-dwh-dbt-project
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merged PR 2644: Fix deployment alerts

Browse source 
# Description

This PR fixes a bug in the deployment script. The bug makes the alerts not work as expected, because the bash piping will trigger alerts if the log writing of commands fail, instead of if commands fail, which is what is expected.

This PR modifies the way logging works. Logging remains the same, but alerts now will work as expected.

Related work items: #18183
This commit is contained in:
Pablo Martín 2024-08-26 14:19:47 +00:00
commit 46ff9514a6
1 changed files with 13 additions and 11 deletions
Download patch file Download diff file Expand all files Collapse all files

24
run_dbt.sh
View file

@ -1,5 +1,7 @@
#!/bin/bash #!/bin/bash
exec >> /home/azureuser/dbt_run.log 2>&1
# Define the Slack webhook URL # Define the Slack webhook URL
script_dir=$(dirname "$0") script_dir=$(dirname "$0")
webhooks_file="slack_webhook_urls.txt" webhooks_file="slack_webhook_urls.txt"
@ -21,37 +23,37 @@ has_any_step_failed=0
cd /home/azureuser/data-dwh-dbt-project cd /home/azureuser/data-dwh-dbt-project
# Update from git # Update from git
echo "Updating dbt project from git." | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1 echo "Updating dbt project from git."
git checkout master | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1 git checkout master
git pull | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1 git pull
# Activate venv # Activate venv
source venv/bin/activate source venv/bin/activate
# Run seeds # Run seeds
echo "Triggering dbt seed" | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1 echo "Triggering dbt seed"
dbt seed | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1 dbt seed
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
has_any_step_failed=1 has_any_step_failed=1
fi fi
# Run staging layer # Run staging layer
echo "Triggering dbt run: Staging" | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1 echo "Triggering dbt run: Staging"
dbt run -s models/staging | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1 dbt run -s models/staging
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
has_any_step_failed=1 has_any_step_failed=1
fi fi
# Run intermediate layer # Run intermediate layer
echo "Triggering dbt run: Intermediate" | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1 echo "Triggering dbt run: Intermediate"
dbt run -s models/intermediate | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1 dbt run -s models/intermediate
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
has_any_step_failed=1 has_any_step_failed=1
fi fi
# Run reporting layer # Run reporting layer
echo "Triggering dbt run: Reporting" | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1 echo "Triggering dbt run: Reporting"
dbt run -s models/reporting | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1 dbt run -s models/reporting
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
has_any_step_failed=1 has_any_step_failed=1
fi fi