more meltano stuff
This commit is contained in:
parent
7e3a88faf3
commit
c0990bf87c
GPG key ID:
883EDBAA726BD96C
7 changed files with 286 additions and 25 deletions
0
pizza_dw/log.txt
Normal file
0
pizza_dw/log.txt
Normal file
|
|
@ -14,10 +14,17 @@ environments:
|
|||
password: app123
|
||||
database: app_db
|
||||
ssl: false
|
||||
select:
|
||||
- public-customers.*
|
||||
- public-pizza_orders.*
|
||||
|
||||
|
||||
loaders:
|
||||
- name: target-dw
|
||||
config:
|
||||
host: localhost
|
||||
port: 5444
|
||||
user: dw
|
||||
password: dw123
|
||||
database: dw_db
|
||||
ssl: false
|
||||
- name: staging
|
||||
- name: prod
|
||||
|
||||
|
|
@ -26,18 +33,22 @@ plugins:
|
|||
- name: tap-postgres
|
||||
variant: meltanolabs
|
||||
pip_url: meltanolabs-tap-postgres
|
||||
|
||||
- name: tap-app
|
||||
inherit_from: tap-postgres
|
||||
|
||||
select:
|
||||
- public-customers.*
|
||||
- public-pizza_orders.*
|
||||
metadata:
|
||||
public-customers:
|
||||
replication-method: INCREMENTAL
|
||||
replication-key: created_at
|
||||
loaders:
|
||||
- name: target-postgres
|
||||
variant: meltanolabs
|
||||
pip_url: meltanolabs-target-postgres
|
||||
config:
|
||||
host: localhost
|
||||
port: 5444
|
||||
user: dw
|
||||
password: dw123
|
||||
database: dw_db
|
||||
ssl: false
|
||||
- name: target-dw
|
||||
inherit_from: target-postgres
|
||||
utilities:
|
||||
- name: dbt-postgres
|
||||
variant: dbt-labs
|
||||
pip_url: dbt-core dbt-postgres meltano-dbt-ext~=0.3.0
|
||||
|
|
|
|||
172
pizza_dw/plugins/utilities/dbt-postgres--dbt-labs.lock
Normal file
172
pizza_dw/plugins/utilities/dbt-postgres--dbt-labs.lock
Normal file
|
|
@ -0,0 +1,172 @@
|
|||
{
|
||||
"plugin_type": "utilities",
|
||||
"name": "dbt-postgres",
|
||||
"namespace": "dbt_postgres",
|
||||
"variant": "dbt-labs",
|
||||
"label": "dbt PostgreSQL",
|
||||
"docs": "https://hub.meltano.com/utilities/dbt-postgres--dbt-labs",
|
||||
"repo": "https://github.com/dbt-labs/dbt-core",
|
||||
"pip_url": "dbt-core dbt-postgres meltano-dbt-ext~=0.3.0",
|
||||
"executable": "dbt_invoker",
|
||||
"logo_url": "https://hub.meltano.com/assets/logos/utilities/dbt.png",
|
||||
"settings": [
|
||||
{
|
||||
"name": "dbname",
|
||||
"aliases": [
|
||||
"database"
|
||||
],
|
||||
"kind": "string",
|
||||
"label": "Database",
|
||||
"description": "The db to connect to.\n"
|
||||
},
|
||||
{
|
||||
"name": "host",
|
||||
"kind": "string",
|
||||
"label": "Host",
|
||||
"description": "The postgres host to connect to.\n"
|
||||
},
|
||||
{
|
||||
"name": "keepalives_idle",
|
||||
"kind": "integer",
|
||||
"label": "Keep Alives Idle",
|
||||
"description": "Seconds between TCP keepalive packets.\n"
|
||||
},
|
||||
{
|
||||
"name": "password",
|
||||
"kind": "string",
|
||||
"label": "Password",
|
||||
"description": "The password to connect with.\n",
|
||||
"sensitive": true
|
||||
},
|
||||
{
|
||||
"name": "port",
|
||||
"kind": "integer",
|
||||
"label": "Port",
|
||||
"description": "The port to connect to.\n"
|
||||
},
|
||||
{
|
||||
"name": "profiles_dir",
|
||||
"env": "DBT_PROFILES_DIR",
|
||||
"value": "$MELTANO_PROJECT_ROOT/transform/profiles/postgres",
|
||||
"label": "Profiles Directory"
|
||||
},
|
||||
{
|
||||
"name": "project_dir",
|
||||
"env": "DBT_PROJECT_DIR",
|
||||
"value": "$MELTANO_PROJECT_ROOT/transform",
|
||||
"label": "Projects Directory"
|
||||
},
|
||||
{
|
||||
"name": "role",
|
||||
"kind": "string",
|
||||
"label": "Role",
|
||||
"description": "Role for dbt to assume when executing queries.\n"
|
||||
},
|
||||
{
|
||||
"name": "schema",
|
||||
"kind": "string",
|
||||
"label": "Schema",
|
||||
"description": "The schema to use.\n"
|
||||
},
|
||||
{
|
||||
"name": "search_path",
|
||||
"kind": "string",
|
||||
"label": "Search Path",
|
||||
"description": "Overrides the default search path.\n"
|
||||
},
|
||||
{
|
||||
"name": "skip_pre_invoke",
|
||||
"env": "DBT_EXT_SKIP_PRE_INVOKE",
|
||||
"kind": "boolean",
|
||||
"value": false,
|
||||
"label": "Skip Pre-invoke",
|
||||
"description": "Whether to skip pre-invoke hooks which automatically run dbt clean and deps"
|
||||
},
|
||||
{
|
||||
"name": "sslmode",
|
||||
"kind": "array",
|
||||
"label": "SSL Mode",
|
||||
"description": "SSL Mode used to connect to the database.\n"
|
||||
},
|
||||
{
|
||||
"name": "target_path",
|
||||
"env": "DBT_TARGET_PATH",
|
||||
"kind": "string",
|
||||
"value": "$MELTANO_PROJECT_ROOT/.meltano/transformers/dbt/target",
|
||||
"label": "Target Path"
|
||||
},
|
||||
{
|
||||
"name": "type",
|
||||
"env": "DBT_EXT_TYPE",
|
||||
"value": "postgres",
|
||||
"label": "dbt Profile type"
|
||||
},
|
||||
{
|
||||
"name": "user",
|
||||
"kind": "string",
|
||||
"label": "User",
|
||||
"description": "The user to connect as.\n"
|
||||
}
|
||||
],
|
||||
"commands": {
|
||||
"build": {
|
||||
"args": "build",
|
||||
"description": "Will run your models, tests, snapshots and seeds in DAG order."
|
||||
},
|
||||
"clean": {
|
||||
"args": "clean",
|
||||
"description": "Delete all folders in the clean-targets list (usually the dbt_modules and target directories.)"
|
||||
},
|
||||
"compile": {
|
||||
"args": "compile",
|
||||
"description": "Generates executable SQL from source model, test, and analysis files. Compiled SQL files are written to the target/ directory."
|
||||
},
|
||||
"debug": {
|
||||
"args": "debug",
|
||||
"description": "Debug your DBT project and warehouse connection."
|
||||
},
|
||||
"deps": {
|
||||
"args": "deps",
|
||||
"description": "Pull the most recent version of the dependencies listed in packages.yml"
|
||||
},
|
||||
"describe": {
|
||||
"args": "describe",
|
||||
"description": "Describe the",
|
||||
"executable": "dbt_extension"
|
||||
},
|
||||
"docs-generate": {
|
||||
"args": "docs generate",
|
||||
"description": "Generate documentation for your project."
|
||||
},
|
||||
"docs-serve": {
|
||||
"args": "docs serve",
|
||||
"description": "Serve documentation for your project. Make sure you ran `docs-generate` first."
|
||||
},
|
||||
"freshness": {
|
||||
"args": "source freshness",
|
||||
"description": "Check the freshness of your source data."
|
||||
},
|
||||
"initialize": {
|
||||
"args": "initialize",
|
||||
"description": "Initialize a new dbt project. This will create a dbt_project.yml file, a profiles.yml file, and models directory.\n",
|
||||
"executable": "dbt_extension"
|
||||
},
|
||||
"run": {
|
||||
"args": "run",
|
||||
"description": "Compile SQL and execute against the current target database."
|
||||
},
|
||||
"seed": {
|
||||
"args": "seed",
|
||||
"description": "Load data from csv files into your data warehouse."
|
||||
},
|
||||
"snapshot": {
|
||||
"args": "snapshot",
|
||||
"description": "Execute snapshots defined in your project."
|
||||
},
|
||||
"test": {
|
||||
"args": "test",
|
||||
"description": "Runs tests on data in deployed models."
|
||||
}
|
||||
},
|
||||
"ext_repo": "https://github.com/meltano/dbt-ext"
|
||||
}
|
||||
3
pizza_dw/transform/.gitignore
vendored
Normal file
3
pizza_dw/transform/.gitignore
vendored
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
target/
|
||||
dbt_packages/
|
||||
logs/
|
||||
29
pizza_dw/transform/dbt_project.yml
Normal file
29
pizza_dw/transform/dbt_project.yml
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
name: my_meltano_project
|
||||
version: '1.0'
|
||||
profile: meltano
|
||||
config-version: 2
|
||||
require-dbt-version: [">=1.0.0", "<2.0.0"]
|
||||
flags:
|
||||
send_anonymous_usage_stats: False
|
||||
use_colors: True
|
||||
model-paths:
|
||||
- models
|
||||
analysis-paths:
|
||||
- analysis
|
||||
test-paths:
|
||||
- tests
|
||||
seed-paths:
|
||||
- data
|
||||
macro-paths:
|
||||
- macros
|
||||
snapshot-paths:
|
||||
- snapshots
|
||||
target-path: target
|
||||
log-path: logs
|
||||
packages-install-path: dbt_packages
|
||||
clean-targets:
|
||||
- target
|
||||
- dbt_packages
|
||||
- logs
|
||||
models:
|
||||
my_meltano_project: null
|
||||
0
pizza_dw/transform/models/.gitkeep
Normal file
0
pizza_dw/transform/models/.gitkeep
Normal file
46
pizza_dw/transform/profiles/postgres/profiles.yml
Normal file
46
pizza_dw/transform/profiles/postgres/profiles.yml
Normal file
|
|
@ -0,0 +1,46 @@
|
|||
# Postgres config reference:
|
||||
meltano:
|
||||
target: "{{ env_var('MELTANO_ENVIRONMENT', 'dev') }}"
|
||||
outputs:
|
||||
dev:
|
||||
type: postgres
|
||||
host: "{{ env_var('DBT_POSTGRES_HOST') }}"
|
||||
user: "{{ env_var('DBT_POSTGRES_USER') }}"
|
||||
password: "{{ env_var('DBT_POSTGRES_PASSWORD') }}"
|
||||
port: "{{ env_var('DBT_POSTGRES_PORT') | int }}"
|
||||
dbname: "{{ env_var('DBT_POSTGRES_DBNAME', '') }}"
|
||||
schema: "{{ env_var('DBT_POSTGRES_SCHEMA') }}"
|
||||
threads: 2
|
||||
keepalives_idle: 0 # default 0, indicating the system default
|
||||
connect_timeout: 10 # default 10 seconds
|
||||
search_path: "{{ env_var('DBT_POSTGRES_SEARCH_PATH', '') }}" # optional, override the default postgres search_path
|
||||
role: "{{ env_var('DBT_POSTGRES_ROLE', '') }}" # optional, set the role dbt assumes when executing queries
|
||||
# sslmode: "{{ env_var('DBT_POSTGRES_SSLMODE', '').split() }}" # optional, set the sslmode used to connect to the database
|
||||
staging:
|
||||
type: postgres
|
||||
host: "{{ env_var('DBT_POSTGRES_HOST') }}"
|
||||
user: "{{ env_var('DBT_POSTGRES_USER') }}"
|
||||
password: "{{ env_var('DBT_POSTGRES_PASSWORD') }}"
|
||||
port: "{{ env_var('DBT_POSTGRES_PORT') | int }}"
|
||||
dbname: "{{ env_var('DBT_POSTGRES_DBNAME', '') }}"
|
||||
schema: "{{ env_var('DBT_POSTGRES_SCHEMA') }}"
|
||||
threads: 4
|
||||
keepalives_idle: 0 # default 0, indicating the system default
|
||||
connect_timeout: 10 # default 10 seconds
|
||||
search_path: "{{ env_var('DBT_POSTGRES_SEARCH_PATH', '') }}" # optional, override the default postgres search_path
|
||||
role: "{{ env_var('DBT_POSTGRES_ROLE', '') }}" # optional, set the role dbt assumes when executing queries
|
||||
# sslmode: "{{ env_var('DBT_POSTGRES_SSLMODE', '').split() }}" # optional, set the sslmode used to connect to the database
|
||||
prod:
|
||||
type: postgres
|
||||
host: "{{ env_var('DBT_POSTGRES_HOST') }}"
|
||||
user: "{{ env_var('DBT_POSTGRES_USER') }}"
|
||||
password: "{{ env_var('DBT_POSTGRES_PASSWORD') }}"
|
||||
port: "{{ env_var('DBT_POSTGRES_PORT') | int }}"
|
||||
dbname: "{{ env_var('DBT_POSTGRES_DBNAME', '') }}"
|
||||
schema: "{{ env_var('DBT_POSTGRES_SCHEMA') }}"
|
||||
threads: 6
|
||||
keepalives_idle: 0 # default 0, indicating the system default
|
||||
connect_timeout: 10 # default 10 seconds
|
||||
search_path: "{{ env_var('DBT_POSTGRES_SEARCH_PATH', '') }}" # optional, override the default postgres search_path
|
||||
role: "{{ env_var('DBT_POSTGRES_ROLE', '') }}" # optional, set the role dbt assumes when executing queries
|
||||
# sslmode: "{{ env_var('DBT_POSTGRES_SSLMODE', '').split() }}" # optional, set the sslmode used to connect to the database
|
||||
Loading…
Add table
Add a link
Reference in a new issue