counterweight/data-xexe
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

more deployment stuff

Browse source
This commit is contained in:
Pablo Martin 2024-06-13 19:18:40 +02:00
parent 4626152379
commit f8fb8e6b76
2 changed files with 12 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

4
README.md
View file

@ -30,8 +30,8 @@ Once you have done this, you can run:
To be able to write rates into the DWH, take these points into consideration:
- `xexe` expects to find the following: a database called `dwh`, schema called `sync_xedotcom_currencies`. These should already exist before `xexe` runs.
- `xexe` should run with a user that has permission to write into `dwh/sync_xedotcom_currencies` and to create tables. It will create the right tables if it can't find them.
- `xexe` expects to find the following: a database called `dwh`, schema called `sync_xedotcom_currency_rates`. These should already exist before `xexe` runs. You should probably create this schema with the same user that you will use to run `xexe` regularly.
- `xexe` should run with a user that has permission to write into `dwh/sync_xedotcom_currency_rates` and to create tables. It will create the right tables if it can't find them.
- These details are hardcoded in the `constants` module. You might want to refactor them into run-time configuration options if you find yourself having to change them often.
### General Usage

15
run_xexe.sh
View file

@ -21,12 +21,12 @@ has_any_step_failed=0
cd /home/azureuser/data-xexe
# Update from git
echo "Updating dbt project from git." | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/xexe_run.log 2>&1
git checkout master | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/xexe_run.log 2>&1
git pull | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/xexe_run.log 2>&1
echo "Updating xexe project from git." | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/xexe_run.log 2>&1
#git checkout master | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/xexe_run.log 2>&1
#git pull | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/xexe_run.log 2>&1
# Activate venv
source venv/bin/activate
poetry shell
# DWH-healthcheck
echo "Running DWH healthcheck..." | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/xexe_run.log 2>&1
@ -44,9 +44,14 @@ if [ $? -ne 0 ]; then
echo "Something went wrong when healthchecking xe.com." | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/xexe_run.log 2>&1
fi
# Define some options here. You might want to change this some day.
currencies="CAD,EUR,NZD,ZAR,AUD,USD,PLN,GBP"
start_date=$(date -d "yesterday" +"%Y-%m-%d")
end_date=$(date -d "yesterday" +"%Y-%m-%d")
# run the actual thing
echo "Getting rates from xe.com..." | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/xexe_run.log 2>&1
xexe get-rates --currencies CAD,EUR,NZD,ZAR,AUD,USD,PLN,GBP --output dwh --ignore-warnings | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/xexe_run.log 2>&1
xexe get-rates --currencies "$currencies" --start-date "$start_date" --end-date "$end_date" --rates-source xe --output dwh --ignore-warnings | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/xexe_run.log 2>&1
if [ $? -ne 0 ]; then
has_any_step_failed=1
echo "Something went wrong when getting rates." | while IFS= read -r line; do printf '%s %s\n' "$(date)" "$line"; done >> /home/azureuser/xexe_run.log 2>&1