diff --git a/pizza_dw/log.txt b/pizza_dw/log.txt new file mode 100644 index 0000000..e69de29 diff --git a/pizza_dw/meltano.yml b/pizza_dw/meltano.yml index 4cecb85..1450e2b 100644 --- a/pizza_dw/meltano.yml +++ b/pizza_dw/meltano.yml @@ -6,38 +6,49 @@ environments: config: plugins: extractors: - - name: tap-app - config: - host: localhost - port: 5432 - user: app - password: app123 - database: app_db - ssl: false - select: - - public-customers.* - - public-pizza_orders.* + - name: tap-app + config: + host: localhost + port: 5432 + user: app + password: app123 + database: app_db + ssl: false + + loaders: + - name: target-dw + config: + host: localhost + port: 5444 + user: dw + password: dw123 + database: dw_db + ssl: false - name: staging - name: prod plugins: extractors: - - name: tap-postgres - variant: meltanolabs - pip_url: meltanolabs-tap-postgres - - - name: tap-app - inherit_from: tap-postgres - + - name: tap-postgres + variant: meltanolabs + pip_url: meltanolabs-tap-postgres + - name: tap-app + inherit_from: tap-postgres + select: + - public-customers.* + - public-pizza_orders.* + metadata: + public-customers: + replication-method: INCREMENTAL + replication-key: created_at loaders: - name: target-postgres variant: meltanolabs pip_url: meltanolabs-target-postgres - config: - host: localhost - port: 5444 - user: dw - password: dw123 - database: dw_db - ssl: false + - name: target-dw + inherit_from: target-postgres + utilities: + - name: dbt-postgres + variant: dbt-labs + pip_url: dbt-core dbt-postgres meltano-dbt-ext~=0.3.0 diff --git a/pizza_dw/plugins/utilities/dbt-postgres--dbt-labs.lock b/pizza_dw/plugins/utilities/dbt-postgres--dbt-labs.lock new file mode 100644 index 0000000..84b87a9 --- /dev/null +++ b/pizza_dw/plugins/utilities/dbt-postgres--dbt-labs.lock @@ -0,0 +1,172 @@ +{ + "plugin_type": "utilities", + "name": "dbt-postgres", + "namespace": "dbt_postgres", + "variant": "dbt-labs", + "label": "dbt PostgreSQL", + "docs": "https://hub.meltano.com/utilities/dbt-postgres--dbt-labs", + "repo": "https://github.com/dbt-labs/dbt-core", + "pip_url": "dbt-core dbt-postgres meltano-dbt-ext~=0.3.0", + "executable": "dbt_invoker", + "logo_url": "https://hub.meltano.com/assets/logos/utilities/dbt.png", + "settings": [ + { + "name": "dbname", + "aliases": [ + "database" + ], + "kind": "string", + "label": "Database", + "description": "The db to connect to.\n" + }, + { + "name": "host", + "kind": "string", + "label": "Host", + "description": "The postgres host to connect to.\n" + }, + { + "name": "keepalives_idle", + "kind": "integer", + "label": "Keep Alives Idle", + "description": "Seconds between TCP keepalive packets.\n" + }, + { + "name": "password", + "kind": "string", + "label": "Password", + "description": "The password to connect with.\n", + "sensitive": true + }, + { + "name": "port", + "kind": "integer", + "label": "Port", + "description": "The port to connect to.\n" + }, + { + "name": "profiles_dir", + "env": "DBT_PROFILES_DIR", + "value": "$MELTANO_PROJECT_ROOT/transform/profiles/postgres", + "label": "Profiles Directory" + }, + { + "name": "project_dir", + "env": "DBT_PROJECT_DIR", + "value": "$MELTANO_PROJECT_ROOT/transform", + "label": "Projects Directory" + }, + { + "name": "role", + "kind": "string", + "label": "Role", + "description": "Role for dbt to assume when executing queries.\n" + }, + { + "name": "schema", + "kind": "string", + "label": "Schema", + "description": "The schema to use.\n" + }, + { + "name": "search_path", + "kind": "string", + "label": "Search Path", + "description": "Overrides the default search path.\n" + }, + { + "name": "skip_pre_invoke", + "env": "DBT_EXT_SKIP_PRE_INVOKE", + "kind": "boolean", + "value": false, + "label": "Skip Pre-invoke", + "description": "Whether to skip pre-invoke hooks which automatically run dbt clean and deps" + }, + { + "name": "sslmode", + "kind": "array", + "label": "SSL Mode", + "description": "SSL Mode used to connect to the database.\n" + }, + { + "name": "target_path", + "env": "DBT_TARGET_PATH", + "kind": "string", + "value": "$MELTANO_PROJECT_ROOT/.meltano/transformers/dbt/target", + "label": "Target Path" + }, + { + "name": "type", + "env": "DBT_EXT_TYPE", + "value": "postgres", + "label": "dbt Profile type" + }, + { + "name": "user", + "kind": "string", + "label": "User", + "description": "The user to connect as.\n" + } + ], + "commands": { + "build": { + "args": "build", + "description": "Will run your models, tests, snapshots and seeds in DAG order." + }, + "clean": { + "args": "clean", + "description": "Delete all folders in the clean-targets list (usually the dbt_modules and target directories.)" + }, + "compile": { + "args": "compile", + "description": "Generates executable SQL from source model, test, and analysis files. Compiled SQL files are written to the target/ directory." + }, + "debug": { + "args": "debug", + "description": "Debug your DBT project and warehouse connection." + }, + "deps": { + "args": "deps", + "description": "Pull the most recent version of the dependencies listed in packages.yml" + }, + "describe": { + "args": "describe", + "description": "Describe the", + "executable": "dbt_extension" + }, + "docs-generate": { + "args": "docs generate", + "description": "Generate documentation for your project." + }, + "docs-serve": { + "args": "docs serve", + "description": "Serve documentation for your project. Make sure you ran `docs-generate` first." + }, + "freshness": { + "args": "source freshness", + "description": "Check the freshness of your source data." + }, + "initialize": { + "args": "initialize", + "description": "Initialize a new dbt project. This will create a dbt_project.yml file, a profiles.yml file, and models directory.\n", + "executable": "dbt_extension" + }, + "run": { + "args": "run", + "description": "Compile SQL and execute against the current target database." + }, + "seed": { + "args": "seed", + "description": "Load data from csv files into your data warehouse." + }, + "snapshot": { + "args": "snapshot", + "description": "Execute snapshots defined in your project." + }, + "test": { + "args": "test", + "description": "Runs tests on data in deployed models." + } + }, + "ext_repo": "https://github.com/meltano/dbt-ext" +} diff --git a/pizza_dw/transform/.gitignore b/pizza_dw/transform/.gitignore new file mode 100644 index 0000000..087d062 --- /dev/null +++ b/pizza_dw/transform/.gitignore @@ -0,0 +1,3 @@ +target/ +dbt_packages/ +logs/ diff --git a/pizza_dw/transform/dbt_project.yml b/pizza_dw/transform/dbt_project.yml new file mode 100644 index 0000000..7aa4999 --- /dev/null +++ b/pizza_dw/transform/dbt_project.yml @@ -0,0 +1,29 @@ +name: my_meltano_project +version: '1.0' +profile: meltano +config-version: 2 +require-dbt-version: [">=1.0.0", "<2.0.0"] +flags: + send_anonymous_usage_stats: False + use_colors: True +model-paths: +- models +analysis-paths: +- analysis +test-paths: +- tests +seed-paths: +- data +macro-paths: +- macros +snapshot-paths: +- snapshots +target-path: target +log-path: logs +packages-install-path: dbt_packages +clean-targets: +- target +- dbt_packages +- logs +models: + my_meltano_project: null diff --git a/pizza_dw/transform/models/.gitkeep b/pizza_dw/transform/models/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/pizza_dw/transform/profiles/postgres/profiles.yml b/pizza_dw/transform/profiles/postgres/profiles.yml new file mode 100644 index 0000000..df9912f --- /dev/null +++ b/pizza_dw/transform/profiles/postgres/profiles.yml @@ -0,0 +1,46 @@ +# Postgres config reference: +meltano: + target: "{{ env_var('MELTANO_ENVIRONMENT', 'dev') }}" + outputs: + dev: + type: postgres + host: "{{ env_var('DBT_POSTGRES_HOST') }}" + user: "{{ env_var('DBT_POSTGRES_USER') }}" + password: "{{ env_var('DBT_POSTGRES_PASSWORD') }}" + port: "{{ env_var('DBT_POSTGRES_PORT') | int }}" + dbname: "{{ env_var('DBT_POSTGRES_DBNAME', '') }}" + schema: "{{ env_var('DBT_POSTGRES_SCHEMA') }}" + threads: 2 + keepalives_idle: 0 # default 0, indicating the system default + connect_timeout: 10 # default 10 seconds + search_path: "{{ env_var('DBT_POSTGRES_SEARCH_PATH', '') }}" # optional, override the default postgres search_path + role: "{{ env_var('DBT_POSTGRES_ROLE', '') }}" # optional, set the role dbt assumes when executing queries + # sslmode: "{{ env_var('DBT_POSTGRES_SSLMODE', '').split() }}" # optional, set the sslmode used to connect to the database + staging: + type: postgres + host: "{{ env_var('DBT_POSTGRES_HOST') }}" + user: "{{ env_var('DBT_POSTGRES_USER') }}" + password: "{{ env_var('DBT_POSTGRES_PASSWORD') }}" + port: "{{ env_var('DBT_POSTGRES_PORT') | int }}" + dbname: "{{ env_var('DBT_POSTGRES_DBNAME', '') }}" + schema: "{{ env_var('DBT_POSTGRES_SCHEMA') }}" + threads: 4 + keepalives_idle: 0 # default 0, indicating the system default + connect_timeout: 10 # default 10 seconds + search_path: "{{ env_var('DBT_POSTGRES_SEARCH_PATH', '') }}" # optional, override the default postgres search_path + role: "{{ env_var('DBT_POSTGRES_ROLE', '') }}" # optional, set the role dbt assumes when executing queries + # sslmode: "{{ env_var('DBT_POSTGRES_SSLMODE', '').split() }}" # optional, set the sslmode used to connect to the database + prod: + type: postgres + host: "{{ env_var('DBT_POSTGRES_HOST') }}" + user: "{{ env_var('DBT_POSTGRES_USER') }}" + password: "{{ env_var('DBT_POSTGRES_PASSWORD') }}" + port: "{{ env_var('DBT_POSTGRES_PORT') | int }}" + dbname: "{{ env_var('DBT_POSTGRES_DBNAME', '') }}" + schema: "{{ env_var('DBT_POSTGRES_SCHEMA') }}" + threads: 6 + keepalives_idle: 0 # default 0, indicating the system default + connect_timeout: 10 # default 10 seconds + search_path: "{{ env_var('DBT_POSTGRES_SEARCH_PATH', '') }}" # optional, override the default postgres search_path + role: "{{ env_var('DBT_POSTGRES_ROLE', '') }}" # optional, set the role dbt assumes when executing queries + # sslmode: "{{ env_var('DBT_POSTGRES_SSLMODE', '').split() }}" # optional, set the sslmode used to connect to the database