counterweight/data-anaxi
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

production script

Browse source
This commit is contained in:
Pablo Martin 2024-08-16 11:29:20 +02:00
parent 64977b8fe7
commit d8dc1c917a
2 changed files with 72 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

2
README.md
View file

@ -83,6 +83,8 @@ anaxi sync-stream --stream-id <your-stream-name>
highest_synced_timestamp: '2024-08-16T9:02:23+00:00' highest_synced_timestamp: '2024-08-16T9:02:23+00:00'
``` ```
The internal logs of `anaxi` will be found in `~/.anaxi/anaxi.log`. The logs of the script will be at `~/anaxi_run.log`.
## Relevant internals and implementation details ## Relevant internals and implementation details
### Tracking checkpoints ### Tracking checkpoints

70
run_anaxi.sh Normal file
View file

@ -0,0 +1,70 @@
#!/bin/bash
exec >> /home/azureuser/anaxi_run.log 2>&1
echo "Starting script at $(date)"
# Define the Slack webhook URL
script_dir=$(dirname "$0")
webhooks_file="slack_webhook_urls.txt"
env_file="$script_dir/$webhooks_file"
if [ -f "$env_file" ]; then
export $(grep -v '^#' "$env_file" | xargs)
else
echo "Error: $webhooks_file file not found in the script directory."
exit 1
fi
# Messages to be sent to Slack
slack_failure_message=":rotating_light::rotating_light::rotating_light: One or more failures during scheduled anaxi run in production. :rotating_light::rotating_light::rotating_light:"
slack_success_message=":white_check_mark::white_check_mark::white_check_mark: anaxi run executed successfully in production. :white_check_mark::white_check_mark::white_check_mark:"
# Initialize the failure flag
has_any_step_failed=0
cd /home/azureuser/data-anaxi
# Update from git
echo "Updating anaxi project from git."
git checkout master
git pull
# Make poetry available
echo "Including Poetry in PATH"
export PATH="$HOME/.local/bin:$PATH"
# DWH-healthcheck
echo "Running DWH healthcheck..."
poetry run anaxi postgres-healthcheck --postgres-database dwh
if [ $? -ne 0 ]; then
has_any_step_failed=1
echo "Something went wrong when healthchecking DWH."
fi
# Define some options here. You might want to change this some day.
currencies="CAD,EUR,NZD,ZAR,AUD,USD,PLN,GBP,CHF"
start_date=$(date -d "yesterday" +"%Y-%m-%d")
end_date=$(date -d "yesterday" +"%Y-%m-%d")
# Actually run syncs
streams=(
"some-stream"
"another-stream"
)
for stream in "${streams[@]}"; do
echo "Starting sync for stream $stream"
poetry run anaxi sync-stream --stream-id $stream
if [ $? -ne 0 ]; then
has_any_step_failed=1
echo "Something went wrong when syncing stream: $stream."
fi
done
# Check if any step failed and send a Slack message
if [ $has_any_step_failed -eq 1 ]; then
curl -X POST -H 'Content-type: application/json' --data "{\"text\":\"$slack_failure_message\"}" $SLACK_ALERT_WEBHOOK_URL
fi
if [ $has_any_step_failed -eq 0 ]; then
curl -X POST -H 'Content-type: application/json' --data "{\"text\":\"$slack_success_message\"}" $SLACK_RECEIPT_WEBHOOK_URL
fi