diff --git a/human-script.md b/human-script.md index dc739c6..b295df5 100644 --- a/human-script.md +++ b/human-script.md @@ -408,11 +408,11 @@ Follow this to deploy the entire data infra. ### 4.2 Create database and schemas -- Run the following script to create a new database and the needed schemas +- Run the following commands to create a new database and the needed schemas ```sql CREATE DATABASE dwh; - \connect dwh; + -- Change active DB to dwh CREATE SCHEMA staging; CREATE SCHEMA intermediate; CREATE SCHEMA reporting; @@ -428,7 +428,7 @@ Follow this to deploy the entire data infra. - A Power BI user, with `consumer` role. - *Note: replace the password fields with serious passwords and note them down.* - ```bash + ```sql GRANT pg_read_all_data TO dwh_admin_infratest; CREATE ROLE airbyte_user LOGIN PASSWORD 'password' VALID UNTIL 'infinity'; @@ -492,13 +492,71 @@ Follow this to deploy the entire data infra. ### 5.2 Deploying Airbyte -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -CONTINUE HERE -Script out these instructions https://docs.airbyte.com/deploying-airbyte/on-azure-vm-cloud-shell +- SSH into the VM. +- Run the following script to install docker and deploy Airbyte +- *Note: replace the variables at the beginning with proper values* -### 5.3 Test connection to DWH +```bash -- This step is optional. If you are deploying production, you should probably skip it to avoid making the DWH dirty. + AIRBYTE_ADMIN_USER=your-user-here + AIRBYTE_ADMIN_PASSWORD=your-password-here + YOUR_ENV= + PRIVATE_DNS_ZONE_NAME=${YOUR_ENV}.data.superhog.com + + echo "Installing docker." + apt-get update -y + apt-get install apt-transport-https ca-certificates curl gnupg lsb-release -y + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg + echo "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + apt-get update + apt-get install docker-ce docker-ce-cli -y + usermod -a -G docker $USER + newgrp docker + apt-get install docker-compose-plugin -y + + echo "Deploying Airbyte" + mkdir airbyte + cd airbyte + wget https://raw.githubusercontent.com/airbytehq/airbyte/master/run-ab-platform.sh + chmod +x run-ab-platform.sh + ./run-ab-platform.sh -b + + + echo "Setting credentials." + sed -i -e "s/BASIC_AUTH_USERNAME=airbyte/BASIC_AUTH_USERNAME=${AIRBYTE_ADMIN_USER}/g" .env + sed -i -e "s/BASIC_AUTH_PASSWORD=password/BASIC_AUTH_PASSWORD=${AIRBYTE_ADMIN_PASSWORD}/g" .env + + echo "Restarting Airbyte." + docker compose down; docker compose up -d + + echo "Deploying Caddy Webserver" + apt install -y debian-keyring debian-archive-keyring apt-transport-https curl + curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/gpg.key' | sudo gpg --dearmor -o /usr/share/keyrings/caddy-stable-archive-keyring.gpg + curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/debian.deb.txt' | sudo tee /etc/apt/sources.list.d/caddy-stable.list + apt update + apt install caddy + + echo "Write caddyfile" + + touch /etc/caddy/Caddyfile + cat > /etc/caddy/Caddyfile << EOL + + # Airbyte web UI + http://airbyte-${YOUR_ENV}.${PRIVATE_DNS_ZONE_NAME} { + reverse_proxy localhost:8000 + } + + EOL + + echo "Restart caddy" + systemctl restart caddy + + echo "You can now access at http://airbyte-${YOUR_ENV}.${PRIVATE_DNS_ZONE_NAME}" + + echo "Finished." + ``` + +- Visit http://airbyte-..data.superhog.com. If you are prompted for user and password, it means Airbyte is running properly and is reachable. ## 6. Power BI