From 8ea3f0b6a27b12ce914276cfc20e33400e6dab07 Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Fri, 4 Apr 2025 14:46:31 +0200 Subject: [PATCH] testing adjustmets --- ci/README.md | 5 +++-- ci/ci-vm-setup.sh | 4 +++- ci/docker-compose.yml | 4 ++-- ci/postgres-initial-setup.sql | 4 ++-- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/ci/README.md b/ci/README.md index 534c42e..9094747 100644 --- a/ci/README.md +++ b/ci/README.md @@ -23,8 +23,9 @@ If you don't have this, it probably means you need to review our Infrastructure - SSH into the CI VM. - Create a folder in the user home directory named `dbt-ci`. -- Create a copy of the `ci.env` file there naming it `.env` (`cp ci.env ~/dbt-ci/.env`) and fill it with values of your choice. -- Execute the script named `ci-vm-setup.sh` in this folder. This script will take care of most of the setup that need to be executed, including: +- Create a copy of the `ci/ci.env` file there naming it `.env` (assuming you're in the repo root dir, `cp ci/ci.env ~/dbt-ci/.env`) and fill it with values of your choice. +- Modify the `docker-compose.yml` file with values for the Postgres server parameters. Which values to set depend on your hardware. If you don't want or can't decide values for these parameters, you can just comment the lines. +- Enter the `ci` folder and execute the script named `ci-vm-setup.sh` in with `.env` file you just filled in sourced (you can run this: `(set -a && source ~/dbt-ci/.env && set +a && bash ci-vm-setup.sh)`). This script will take care of most of the setup that need to be executed, including: - Preparing the postgres database. - Setting up the dockerized postgres with the right database, FDW, etc. - Prepare the `profiles.yml` file. diff --git a/ci/ci-vm-setup.sh b/ci/ci-vm-setup.sh index 997ce86..a2b8ca3 100644 --- a/ci/ci-vm-setup.sh +++ b/ci/ci-vm-setup.sh @@ -1,8 +1,10 @@ # Start container -docker compose build -d +docker compose --env-file ~/dbt-ci/.env up -d # Run script to set things up in Postgres (DB, FDWs, etc) + envsubst < postgres-initial-setup.sql | psql -h $POSTGRES_HOST -U $POSTGRES_USER -d postgres # Copy profiles file +mkdir -p ~/.dbt cp ci.profiles.yml ~/.dbt/profiles.yml diff --git a/ci/docker-compose.yml b/ci/docker-compose.yml index c1c14ec..31f0d6d 100644 --- a/ci/docker-compose.yml +++ b/ci/docker-compose.yml @@ -4,7 +4,7 @@ services: container_name: postgres_db environment: POSTGRES_USER: ${POSTGRES_USER} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_PASSWORD: ${PGPASSWORD} POSTGRES_DB: postgres ports: - "5432:5432" @@ -12,7 +12,7 @@ services: - postgres_data:/var/lib/postgresql/data # Note that some of the values below are very HW specific. You should # absolutely adjust them to the available hardware where this will be - # running. This might help if you feel lost: + # running. This might help if you feel lost: https://pgtune.leopard.in.ua/ command: [ "-c", "max_connections=XX", "-c", "shared_buffers=XGB", diff --git a/ci/postgres-initial-setup.sql b/ci/postgres-initial-setup.sql index e0c28ff..805c065 100644 --- a/ci/postgres-initial-setup.sql +++ b/ci/postgres-initial-setup.sql @@ -1,5 +1,5 @@ -CREATE DATABASE prd-pointer; -\c prd-pointer +CREATE DATABASE prd_pointer; +\c prd_pointer CREATE EXTENSION postgres_fdw;