counterweight/data-dwh-dbt-project
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

add exec, remove pipes

Browse source
This commit is contained in:
Pablo Martin 2024-08-26 12:55:51 +02:00
parent 5e41e88b59
commit 5c55aa8094
1 changed files with 13 additions and 11 deletions
Download patch file Download diff file Expand all files Collapse all files

24
run_dbt.sh
View file

@ -1,5 +1,7 @@
#!/bin/bash
exec >> /home/azureuser/dbt_run.log 2>&1
# Define the Slack webhook URL
script_dir=$(dirname "$0")
webhooks_file="slack_webhook_urls.txt"
@ -21,37 +23,37 @@ has_any_step_failed=0
cd /home/azureuser/data-dwh-dbt-project
# Update from git
echo "Updating dbt project from git." | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1
git checkout master | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1
git pull | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1
echo "Updating dbt project from git."
git checkout master
git pull
# Activate venv
source venv/bin/activate
# Run seeds
echo "Triggering dbt seed" | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1
dbt seed | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1
echo "Triggering dbt seed"
dbt seed
if [ $? -ne 0 ]; then
has_any_step_failed=1
fi
# Run staging layer
echo "Triggering dbt run: Staging" | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1
dbt run -s models/staging | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1
echo "Triggering dbt run: Staging"
dbt run -s models/staging
if [ $? -ne 0 ]; then
has_any_step_failed=1
fi
# Run intermediate layer
echo "Triggering dbt run: Intermediate" | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1
dbt run -s models/intermediate | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1
echo "Triggering dbt run: Intermediate"
dbt run -s models/intermediate
if [ $? -ne 0 ]; then
has_any_step_failed=1
fi
# Run reporting layer
echo "Triggering dbt run: Reporting" | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1
dbt run -s models/reporting | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1
echo "Triggering dbt run: Reporting"
dbt run -s models/reporting
if [ $? -ne 0 ]; then
has_any_step_failed=1
fi