counterweight/data-dwh-dbt-project
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

add exec, remove pipes

Browse source
This commit is contained in:
Pablo Martin 2024-08-26 12:55:51 +02:00
parent 5e41e88b59
commit 5c55aa8094
1 changed files with 13 additions and 11 deletions
Download patch file Download diff file Expand all files Collapse all files

24
run_dbt.sh
View file

@ -1,5 +1,7 @@
#!/bin/bash #!/bin/bash
exec >> /home/azureuser/dbt_run.log 2>&1
# Define the Slack webhook URL # Define the Slack webhook URL
script_dir=$(dirname "$0") script_dir=$(dirname "$0")
webhooks_file="slack_webhook_urls.txt" webhooks_file="slack_webhook_urls.txt"
@ -21,37 +23,37 @@ has_any_step_failed=0
cd /home/azureuser/data-dwh-dbt-project cd /home/azureuser/data-dwh-dbt-project
# Update from git # Update from git
echo "Updating dbt project from git." | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1 echo "Updating dbt project from git."
git checkout master | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1 git checkout master
git pull | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1 git pull
# Activate venv # Activate venv
source venv/bin/activate source venv/bin/activate
# Run seeds # Run seeds
echo "Triggering dbt seed" | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1 echo "Triggering dbt seed"
dbt seed | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1 dbt seed
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
has_any_step_failed=1 has_any_step_failed=1
fi fi
# Run staging layer # Run staging layer
echo "Triggering dbt run: Staging" | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1 echo "Triggering dbt run: Staging"
dbt run -s models/staging | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1 dbt run -s models/staging
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
has_any_step_failed=1 has_any_step_failed=1
fi fi
# Run intermediate layer # Run intermediate layer
echo "Triggering dbt run: Intermediate" | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1 echo "Triggering dbt run: Intermediate"
dbt run -s models/intermediate | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1 dbt run -s models/intermediate
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
has_any_step_failed=1 has_any_step_failed=1
fi fi
# Run reporting layer # Run reporting layer
echo "Triggering dbt run: Reporting" | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1 echo "Triggering dbt run: Reporting"
dbt run -s models/reporting | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/dbt_run.log 2>&1 dbt run -s models/reporting
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
has_any_step_failed=1 has_any_step_failed=1
fi fi