counterweight/data-dwh-dbt-project
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

testing adjustmets

Browse source
This commit is contained in:
Pablo Martin 2025-04-04 14:46:31 +02:00
parent 901d39a045
commit 8ea3f0b6a2
4 changed files with 10 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

5
ci/README.md
View file

@ -23,8 +23,9 @@ If you don't have this, it probably means you need to review our Infrastructure
- SSH into the CI VM. - SSH into the CI VM.
- Create a folder in the user home directory named `dbt-ci`. - Create a folder in the user home directory named `dbt-ci`.
- Create a copy of the `ci.env` file there naming it `.env` (`cp ci.env ~/dbt-ci/.env`) and fill it with values of your choice. - Create a copy of the `ci/ci.env` file there naming it `.env` (assuming you're in the repo root dir, `cp ci/ci.env ~/dbt-ci/.env`) and fill it with values of your choice.
- Execute the script named `ci-vm-setup.sh` in this folder. This script will take care of most of the setup that need to be executed, including: - Modify the `docker-compose.yml` file with values for the Postgres server parameters. Which values to set depend on your hardware. If you don't want or can't decide values for these parameters, you can just comment the lines.
- Enter the `ci` folder and execute the script named `ci-vm-setup.sh` in with `.env` file you just filled in sourced (you can run this: `(set -a && source ~/dbt-ci/.env && set +a && bash ci-vm-setup.sh)`). This script will take care of most of the setup that need to be executed, including:
- Preparing the postgres database. - Preparing the postgres database.
- Setting up the dockerized postgres with the right database, FDW, etc. - Setting up the dockerized postgres with the right database, FDW, etc.
- Prepare the `profiles.yml` file. - Prepare the `profiles.yml` file.

4
ci/ci-vm-setup.sh
View file

@ -1,8 +1,10 @@
# Start container # Start container
docker compose build -d docker compose --env-file ~/dbt-ci/.env up -d
# Run script to set things up in Postgres (DB, FDWs, etc) # Run script to set things up in Postgres (DB, FDWs, etc)
envsubst < postgres-initial-setup.sql | psql -h $POSTGRES_HOST -U $POSTGRES_USER -d postgres envsubst < postgres-initial-setup.sql | psql -h $POSTGRES_HOST -U $POSTGRES_USER -d postgres
# Copy profiles file # Copy profiles file
mkdir -p ~/.dbt
cp ci.profiles.yml ~/.dbt/profiles.yml cp ci.profiles.yml ~/.dbt/profiles.yml

4
ci/docker-compose.yml
View file

@ -4,7 +4,7 @@ services:
container_name: postgres_db container_name: postgres_db
environment: environment:
POSTGRES_USER: ${POSTGRES_USER} POSTGRES_USER: ${POSTGRES_USER}
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} POSTGRES_PASSWORD: ${PGPASSWORD}
POSTGRES_DB: postgres POSTGRES_DB: postgres
ports: ports:
- "5432:5432" - "5432:5432"
@ -12,7 +12,7 @@ services:
- postgres_data:/var/lib/postgresql/data - postgres_data:/var/lib/postgresql/data
# Note that some of the values below are very HW specific. You should # Note that some of the values below are very HW specific. You should
# absolutely adjust them to the available hardware where this will be # absolutely adjust them to the available hardware where this will be
# running. This might help if you feel lost: # running. This might help if you feel lost: https://pgtune.leopard.in.ua/
command: [ command: [
"-c", "max_connections=XX", "-c", "max_connections=XX",
"-c", "shared_buffers=XGB", "-c", "shared_buffers=XGB",

4
ci/postgres-initial-setup.sql
View file

@ -1,5 +1,5 @@
CREATE DATABASE prd-pointer; CREATE DATABASE prd_pointer;
\c prd-pointer \c prd_pointer
CREATE EXTENSION postgres_fdw; CREATE EXTENSION postgres_fdw;