more meltano stuff

pizza_dw/meltano.yml

@@ -6,38 +6,49 @@ environments:
   config:
     plugins:
       extractors:
-        - name: tap-app
+      - name: tap-app
-          config:
+        config:
-            host: localhost
+          host: localhost
-            port: 5432
+          port: 5432
-            user: app
+          user: app
-            password: app123
+          password: app123
-            database: app_db
+          database: app_db
-            ssl: false
+          ssl: false
-          select:
-            - public-customers.*
-            - public-pizza_orders.*
 
+
+      loaders:
+      - name: target-dw
+        config:
+          host: localhost
+          port: 5444
+          user: dw
+          password: dw123
+          database: dw_db
+          ssl: false
 - name: staging
 - name: prod
 
 plugins:
   extractors:
-    - name: tap-postgres
+  - name: tap-postgres
-      variant: meltanolabs
+    variant: meltanolabs
-      pip_url: meltanolabs-tap-postgres
+    pip_url: meltanolabs-tap-postgres
-
+  - name: tap-app
-    - name: tap-app
+    inherit_from: tap-postgres
-      inherit_from: tap-postgres
+    select:
-
+    - public-customers.*
+    - public-pizza_orders.*
+    metadata:
+      public-customers:
+        replication-method: INCREMENTAL
+        replication-key: created_at
   loaders:
   - name: target-postgres
     variant: meltanolabs
     pip_url: meltanolabs-target-postgres
-    config:
+  - name: target-dw
-      host: localhost
+    inherit_from: target-postgres
-      port: 5444
+  utilities:
-      user: dw
+  - name: dbt-postgres
-      password: dw123
+    variant: dbt-labs
-      database: dw_db
+    pip_url: dbt-core dbt-postgres meltano-dbt-ext~=0.3.0
-      ssl: false

pizza_dw/plugins/utilities/dbt-postgres--dbt-labs.lock

@@ -0,0 +1,172 @@
+{
+  "plugin_type": "utilities",
+  "name": "dbt-postgres",
+  "namespace": "dbt_postgres",
+  "variant": "dbt-labs",
+  "label": "dbt PostgreSQL",
+  "docs": "https://hub.meltano.com/utilities/dbt-postgres--dbt-labs",
+  "repo": "https://github.com/dbt-labs/dbt-core",
+  "pip_url": "dbt-core dbt-postgres meltano-dbt-ext~=0.3.0",
+  "executable": "dbt_invoker",
+  "logo_url": "https://hub.meltano.com/assets/logos/utilities/dbt.png",
+  "settings": [
+    {
+      "name": "dbname",
+      "aliases": [
+        "database"
+      ],
+      "kind": "string",
+      "label": "Database",
+      "description": "The db to connect to.\n"
+    },
+    {
+      "name": "host",
+      "kind": "string",
+      "label": "Host",
+      "description": "The postgres host to connect to.\n"
+    },
+    {
+      "name": "keepalives_idle",
+      "kind": "integer",
+      "label": "Keep Alives Idle",
+      "description": "Seconds between TCP keepalive packets.\n"
+    },
+    {
+      "name": "password",
+      "kind": "string",
+      "label": "Password",
+      "description": "The password to connect with.\n",
+      "sensitive": true
+    },
+    {
+      "name": "port",
+      "kind": "integer",
+      "label": "Port",
+      "description": "The port to connect to.\n"
+    },
+    {
+      "name": "profiles_dir",
+      "env": "DBT_PROFILES_DIR",
+      "value": "$MELTANO_PROJECT_ROOT/transform/profiles/postgres",
+      "label": "Profiles Directory"
+    },
+    {
+      "name": "project_dir",
+      "env": "DBT_PROJECT_DIR",
+      "value": "$MELTANO_PROJECT_ROOT/transform",
+      "label": "Projects Directory"
+    },
+    {
+      "name": "role",
+      "kind": "string",
+      "label": "Role",
+      "description": "Role for dbt to assume when executing queries.\n"
+    },
+    {
+      "name": "schema",
+      "kind": "string",
+      "label": "Schema",
+      "description": "The schema to use.\n"
+    },
+    {
+      "name": "search_path",
+      "kind": "string",
+      "label": "Search Path",
+      "description": "Overrides the default search path.\n"
+    },
+    {
+      "name": "skip_pre_invoke",
+      "env": "DBT_EXT_SKIP_PRE_INVOKE",
+      "kind": "boolean",
+      "value": false,
+      "label": "Skip Pre-invoke",
+      "description": "Whether to skip pre-invoke hooks which automatically run dbt clean and deps"
+    },
+    {
+      "name": "sslmode",
+      "kind": "array",
+      "label": "SSL Mode",
+      "description": "SSL Mode used to connect to the database.\n"
+    },
+    {
+      "name": "target_path",
+      "env": "DBT_TARGET_PATH",
+      "kind": "string",
+      "value": "$MELTANO_PROJECT_ROOT/.meltano/transformers/dbt/target",
+      "label": "Target Path"
+    },
+    {
+      "name": "type",
+      "env": "DBT_EXT_TYPE",
+      "value": "postgres",
+      "label": "dbt Profile type"
+    },
+    {
+      "name": "user",
+      "kind": "string",
+      "label": "User",
+      "description": "The user to connect as.\n"
+    }
+  ],
+  "commands": {
+    "build": {
+      "args": "build",
+      "description": "Will run your models, tests, snapshots and seeds in DAG order."
+    },
+    "clean": {
+      "args": "clean",
+      "description": "Delete all folders in the clean-targets list (usually the dbt_modules and target directories.)"
+    },
+    "compile": {
+      "args": "compile",
+      "description": "Generates executable SQL from source model, test, and analysis files. Compiled SQL files are written to the target/ directory."
+    },
+    "debug": {
+      "args": "debug",
+      "description": "Debug your DBT project and warehouse connection."
+    },
+    "deps": {
+      "args": "deps",
+      "description": "Pull the most recent version of the dependencies listed in packages.yml"
+    },
+    "describe": {
+      "args": "describe",
+      "description": "Describe the",
+      "executable": "dbt_extension"
+    },
+    "docs-generate": {
+      "args": "docs generate",
+      "description": "Generate documentation for your project."
+    },
+    "docs-serve": {
+      "args": "docs serve",
+      "description": "Serve documentation for your project. Make sure you ran `docs-generate` first."
+    },
+    "freshness": {
+      "args": "source freshness",
+      "description": "Check the freshness of your source data."
+    },
+    "initialize": {
+      "args": "initialize",
+      "description": "Initialize a new dbt project. This will create a dbt_project.yml file, a profiles.yml file, and models directory.\n",
+      "executable": "dbt_extension"
+    },
+    "run": {
+      "args": "run",
+      "description": "Compile SQL and execute against the current target database."
+    },
+    "seed": {
+      "args": "seed",
+      "description": "Load data from csv files into your data warehouse."
+    },
+    "snapshot": {
+      "args": "snapshot",
+      "description": "Execute snapshots defined in your project."
+    },
+    "test": {
+      "args": "test",
+      "description": "Runs tests on data in deployed models."
+    }
+  },
+  "ext_repo": "https://github.com/meltano/dbt-ext"
+}

pizza_dw/transform/.gitignore

@@ -0,0 +1,3 @@
+target/
+dbt_packages/
+logs/

pizza_dw/transform/dbt_project.yml

@@ -0,0 +1,29 @@
+name: my_meltano_project
+version: '1.0'
+profile: meltano
+config-version: 2
+require-dbt-version: [">=1.0.0", "<2.0.0"]
+flags:
+  send_anonymous_usage_stats: False
+  use_colors: True
+model-paths:
+- models
+analysis-paths:
+- analysis
+test-paths:
+- tests
+seed-paths:
+- data
+macro-paths:
+- macros
+snapshot-paths:
+- snapshots
+target-path: target
+log-path: logs
+packages-install-path: dbt_packages
+clean-targets:
+- target
+- dbt_packages
+- logs
+models:
+  my_meltano_project: null

pizza_dw/transform/profiles/postgres/profiles.yml

@@ -0,0 +1,46 @@
+# Postgres config reference:
+meltano:
+  target: "{{ env_var('MELTANO_ENVIRONMENT', 'dev') }}"
+  outputs:
+    dev:
+      type: postgres
+      host: "{{ env_var('DBT_POSTGRES_HOST') }}"
+      user: "{{ env_var('DBT_POSTGRES_USER') }}"
+      password: "{{ env_var('DBT_POSTGRES_PASSWORD') }}"
+      port: "{{ env_var('DBT_POSTGRES_PORT') | int }}"
+      dbname: "{{ env_var('DBT_POSTGRES_DBNAME', '') }}"
+      schema: "{{ env_var('DBT_POSTGRES_SCHEMA') }}"
+      threads: 2
+      keepalives_idle: 0 # default 0, indicating the system default
+      connect_timeout: 10 # default 10 seconds
+      search_path: "{{ env_var('DBT_POSTGRES_SEARCH_PATH', '') }}" # optional, override the default postgres search_path
+      role: "{{ env_var('DBT_POSTGRES_ROLE', '') }}" # optional, set the role dbt assumes when executing queries
+      # sslmode: "{{ env_var('DBT_POSTGRES_SSLMODE', '').split() }}" # optional, set the sslmode used to connect to the database
+    staging:
+      type: postgres
+      host: "{{ env_var('DBT_POSTGRES_HOST') }}"
+      user: "{{ env_var('DBT_POSTGRES_USER') }}"
+      password: "{{ env_var('DBT_POSTGRES_PASSWORD') }}"
+      port: "{{ env_var('DBT_POSTGRES_PORT') | int }}"
+      dbname: "{{ env_var('DBT_POSTGRES_DBNAME', '') }}"
+      schema: "{{ env_var('DBT_POSTGRES_SCHEMA') }}"
+      threads: 4
+      keepalives_idle: 0 # default 0, indicating the system default
+      connect_timeout: 10 # default 10 seconds
+      search_path: "{{ env_var('DBT_POSTGRES_SEARCH_PATH', '') }}" # optional, override the default postgres search_path
+      role: "{{ env_var('DBT_POSTGRES_ROLE', '') }}" # optional, set the role dbt assumes when executing queries
+      # sslmode: "{{ env_var('DBT_POSTGRES_SSLMODE', '').split() }}" # optional, set the sslmode used to connect to the database
+    prod:
+      type: postgres
+      host: "{{ env_var('DBT_POSTGRES_HOST') }}"
+      user: "{{ env_var('DBT_POSTGRES_USER') }}"
+      password: "{{ env_var('DBT_POSTGRES_PASSWORD') }}"
+      port: "{{ env_var('DBT_POSTGRES_PORT') | int }}"
+      dbname: "{{ env_var('DBT_POSTGRES_DBNAME', '') }}"
+      schema: "{{ env_var('DBT_POSTGRES_SCHEMA') }}"
+      threads: 6
+      keepalives_idle: 0 # default 0, indicating the system default
+      connect_timeout: 10 # default 10 seconds
+      search_path: "{{ env_var('DBT_POSTGRES_SEARCH_PATH', '') }}" # optional, override the default postgres search_path
+      role: "{{ env_var('DBT_POSTGRES_ROLE', '') }}" # optional, set the role dbt assumes when executing queries
+      # sslmode: "{{ env_var('DBT_POSTGRES_SSLMODE', '').split() }}" # optional, set the sslmode used to connect to the database