diff --git a/.vscode/settings.json b/.vscode/settings.json index ddb2cf2..d8d206c 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -11,5 +11,24 @@ ] }, "dbt.enableNewLineagePanel": true, - "dbt.enableNewDocsPanel": true + "dbt.enableNewDocsPanel": true, + "yaml.schemas": { + "https://raw.githubusercontent.com/dbt-labs/dbt-jsonschema/main/schemas/dbt_yml_files.json": [ + "/**/*.yml", + "!profiles.yml", + "!dbt_project.yml", + "!packages.yml", + "!selectors.yml", + "!profile_template.yml" + ], + "https://raw.githubusercontent.com/dbt-labs/dbt-jsonschema/main/schemas/dbt_project.json": [ + "dbt_project.yml" + ], + "https://raw.githubusercontent.com/dbt-labs/dbt-jsonschema/main/schemas/selectors.json": [ + "selectors.yml" + ], + "https://raw.githubusercontent.com/dbt-labs/dbt-jsonschema/main/schemas/packages.json": [ + "packages.yml" + ] + }, } \ No newline at end of file diff --git a/macros/generate_stripe_sources_unioned_select.sql b/macros/generate_stripe_sources_unioned_select.sql new file mode 100644 index 0000000..d2803c3 --- /dev/null +++ b/macros/generate_stripe_sources_unioned_select.sql @@ -0,0 +1,15 @@ +/* +This macro runs for every country code and: + - Reads from the right source (stripe_) + - Adds a column identifying the source account with a string like "stripe__account" +*/ +{% macro generate_stripe_sources_unioned_select(source_table) -%} + {% set countries = ["us", "uk"] %} + {% for country in countries %} + select *, 'stripe_{{ country }}_account' as stripe_source_account + from {{ source("stripe_" ~ country, source_table) }} + {% if not loop.last -%} + union all + {%- endif %} + {% endfor %} +{%- endmacro %} diff --git a/models/doc_blocks.md b/models/doc_blocks.md index beeef7c..d78a8e3 100644 --- a/models/doc_blocks.md +++ b/models/doc_blocks.md @@ -28,8 +28,20 @@ You can query the _airbyte_meta column to see which rows failed for content reas {% enddocs %} -{% docs dwh_extracted_at_date_desc %} +{% docs dwh_extracted_at_utc_desc %} -A timestamp for when the event was pulled by Airbyte from the original data source. +A timestamp for when the event was pulled from the original data source. -{% enddocs %} \ No newline at end of file +{% enddocs %} + +{% docs dwh_extracted_date_utc_desc %} + +The date when the event was pulled from the original data source. + +{% enddocs %} + +{% docs currency_desc %} + +Three-letter ISO 4217 currency code, in uppercase. Must be a supported currency. + +{% enddocs %} diff --git a/models/staging/stripe/_stripe_sources.yml b/models/staging/stripe/_stripe_sources.yml index cf09e19..2d378fa 100644 --- a/models/staging/stripe/_stripe_sources.yml +++ b/models/staging/stripe/_stripe_sources.yml @@ -1,9 +1,13 @@ version: 2 +# The design of this source was heavily inspired by this post: +# https://discourse.getdbt.com/t/unioning-identically-structured-data-sources/921 + sources: - name: stripe_uk + description: Data from our Stripe UK account. schema: sync_stripe_uk - tables: + tables: &stripe_tables - name: balance_transactions identifier: balance_transactions description: | @@ -79,6 +83,8 @@ sources: "transfer_cancel", "transfer_failure", "transfer_refund", + "balance_transfer_outbound", + "balance_transfer_inbound" ] - name: amount data_type: bigint @@ -108,7 +114,7 @@ sources: - not_null - name: currency data_type: character varying - description: "{{ doc('stripe_currency_desc') }}" + description: "{{ doc('raw_stripe_currency_desc') }}" tests: - not_null - dbt_expectations.expect_column_value_lengths_to_equal: @@ -145,6 +151,7 @@ sources: "refund", "topup", "dispute", + "other_adjustment" ] - name: _airbyte_raw_id data_type: character varying @@ -241,7 +248,7 @@ sources: - not_null - name: currency data_type: character varying - description: "{{ doc('stripe_currency_desc') }}" + description: "{{ doc('raw_stripe_currency_desc') }}" tests: - not_null - dbt_expectations.expect_column_value_lengths_to_equal: @@ -457,7 +464,7 @@ sources: - not_null - name: currency data_type: character varying - description: "{{ doc('stripe_currency_desc') }}" + description: "{{ doc('raw_stripe_currency_desc') }}" tests: - not_null - dbt_expectations.expect_column_value_lengths_to_equal: @@ -664,7 +671,7 @@ sources: description: "{{ doc('stripe_created_desc') }}" - name: currency data_type: character varying - description: "{{ doc('stripe_currency_desc') }}" + description: "{{ doc('raw_stripe_currency_desc') }}" tests: - not_null - dbt_expectations.expect_column_value_lengths_to_equal: @@ -712,3 +719,7 @@ sources: - name: _airbyte_meta data_type: jsonb description: "{{ doc('_airbyte_meta_desc') }}" + - name: stripe_us + description: Data from our Stripe US account. + schema: sync_stripe_us + tables: *stripe_tables diff --git a/models/staging/stripe/schema.yml b/models/staging/stripe/schema.yml index 401a3db..b7a6ac4 100644 --- a/models/staging/stripe/schema.yml +++ b/models/staging/stripe/schema.yml @@ -6,61 +6,95 @@ models: Individual transactions happening in our different Stripe currency accounts. You can read more about this object here: https://docs.stripe.com/api/balance_transactions/object. - columns: - name: id data_type: character varying description: "{{ doc('generic_id_desc') }}" + - name: fee - data_type: bigint - description: "Fees (in cents) paid for this transaction. Represented as a positive integer when assessed." + data_type: numeric + description: "Fees paid for this transaction. Represented as a positive integer when assessed." + - name: net - data_type: bigint - description: "Net impact to a Stripe balance (in cents). A positive value represents incrementing a Stripe balance, and a negative value decrementing a Stripe balance. You can calculate the net impact of a transaction on a balance by amount - fee" + data_type: numeric + description: "Net impact to a Stripe balance. A positive value represents incrementing a Stripe balance, and a negative value decrementing a Stripe balance. You can calculate the net impact of a transaction on a balance by amount - fee" + - name: type data_type: character varying description: "Transaction type: adjustment, advance, advance_funding, anticipation_repayment, application_fee, application_fee_refund, charge, climate_order_purchase, climate_order_refund, connect_collection_transfer, contribution, issuing_authorization_hold, issuing_authorization_release, issuing_dispute, issuing_transaction, obligation_outbound, obligation_reversal_inbound, payment, payment_failure_refund, payment_network_reserve_hold, payment_network_reserve_release, payment_refund, payment_reversal, payment_unreconciled, payout, payout_cancel, payout_failure, refund, refund_failure, reserve_transaction, reserved_funds, stripe_fee, stripe_fx_fee, tax_fee, topup, topup_reversal, transfer, transfer_cancel, transfer_failure, or transfer_refund. Learn more about balance transaction types and what they represent. To classify transactions for accounting purposes, consider reporting_category instead." + - name: amount - data_type: bigint - description: "Gross amount of this transaction (in cents). A positive value represents funds charged to another party, and a negative value represents funds sent to another party." + data_type: numeric + description: "Gross amount of this transaction. A positive value represents funds charged to another party, and a negative value represents funds sent to another party." + - name: object data_type: character varying description: "Silly column. The value is always `balance_transaction`." + - name: source data_type: character varying description: "This transaction relates to the Stripe object indicated here. Can be different object types (charge, refund, payment, etc)." + - name: status data_type: character varying description: "The transaction’s net funds status in the Stripe balance, which are either available or pending." + - name: created_at_utc - data_type: bigint + data_type: timestamp with time zone description: "The timestamp at which the object was created." + + - name: created_date_utc + data_type: date + description: "The date when the object was created." + - name: currency data_type: character varying - description: "{{ doc('stripe_currency_desc') }}" + description: "{{ doc('currency_desc') }}" + - name: description data_type: character varying description: "An arbitrary string attached to the object. Often useful for displaying to users." + - name: fee_details data_type: jsonb description: "Detailed breakdown of fees (in cents) paid for this transaction." + - name: available_at_utc - data_type: bigint + data_type: timestamp with time zone description: "The timestamp at which the transaction’s net funds become available in the Stripe balance." + + - name: available_date_utc + data_type: date + description: "The date when the object was available." + - name: exchange_rate data_type: numeric description: "If applicable, this transaction uses an exchange rate. If money converts from currency A to currency B, then the amount in currency A, multipled by the exchange_rate, equals the amount in currency B. For example, if you charge a customer 10.00 EUR, the PaymentIntent’s amount is 1000 and currency is eur. If this converts to 12.34 USD in your Stripe account, the BalanceTransaction’s amount is 1234, its currency is usd, and the exchange_rate is 1.234." + - name: sourced_transfers data_type: jsonb description: "{{ doc('field_docs_na_desc') }}" + - name: reporting_category data_type: character varying description: "Read more at https://stripe.com/docs/reports/reporting-categories." - - name: dwh_extracted_at_date + + - name: stripe_source_account + data_type: character varying + description: "{{ doc('stripe_source_account_desc')}}" + + - name: dwh_extracted_at_utc data_type: timestamp with time zone - description: "{{ doc('dwh_extracted_at_date_desc')}}" + description: "{{ doc('dwh_extracted_at_utc_desc')}}" + + - name: dwh_extracted_date_utc + data_type: date + description: "{{ doc('dwh_extracted_date_utc_desc')}}" - name: stg_stripe__charges - description: "" + description: | + The Charge object represents a single attempt to move money into your Stripe account. PaymentIntent confirmation is the most common way to create Charges, but transferring money to a different Stripe account through Connect also creates Charges. Some legacy payment flows create Charges directly, which is not recommended for new integrations. + + You can read more about this object here: https://docs.stripe.com/api/charges. columns: - name: id data_type: character varying @@ -79,8 +113,8 @@ models: description: "{{ doc('field_docs_na_desc') }}" - name: amount - data_type: bigint - description: "Amount intended to be collected by this payment. A positive integer representing how much to charge in the smallest currency unit (e.g., 100 cents to charge $1.00 or 100 to charge ¥100, a zero-decimal currency). The minimum amount is $0.50 US or equivalent in charge currency. The amount value supports up to eight digits (e.g., a value of 99999999 for a USD charge of $999,999.99)." + data_type: numeric + description: "Amount intended to be collected by this payment. A positive integer representing how much to charge in the regular unit for the currency (1 for $1)." - name: object data_type: character varying @@ -102,6 +136,10 @@ models: data_type: timestamp with time zone description: "The timestamp at which the object was created." + - name: created_date_utc + data_type: date + description: "The date when the object was created." + - name: dispute data_type: character varying description: "{{ doc('field_docs_na_desc')}}" @@ -122,13 +160,17 @@ models: data_type: timestamp with time zone description: "The timestamp at which the object was last updated." + - name: updated_date_utc + data_type: date + description: "The date when the object was last updated." + - name: is_captured data_type: boolean description: "If the charge was created without capturing, this Boolean represents whether it is still uncaptured or has since been captured." - name: currency data_type: character varying - description: "{{ doc('stripe_currency_desc') }}" + description: "{{ doc('currency_desc') }}" - name: customer data_type: character varying @@ -211,12 +253,12 @@ models: description: "A string that identifies this transaction as part of a group. See the Connect documentation for details." - name: amount_captured - data_type: bigint - description: "Amount in cents captured (can be less than the amount attribute on the charge if a partial capture was made)." + data_type: numeric + description: "Amount captured (can be less than the amount attribute on the charge if a partial capture was made)." - name: amount_refunded - data_type: bigint - description: "Amount in cents refunded (can be less than the amount attribute on the charge if a partial refund was issued)." + data_type: numeric + description: "Amount refunded (can be less than the amount attribute on the charge if a partial refund was issued)." - name: application_fee data_type: character varying @@ -266,9 +308,17 @@ models: data_type: character varying description: "The full statement descriptor that is passed to card networks, and that is displayed on your customers’ credit card and bank statements. Allows you to see what the statement descriptor looks like after the static and dynamic portions are combined." - - name: dwh_extracted_at_date + - name: stripe_source_account + data_type: character varying + description: "{{ doc('stripe_source_account_desc')}}" + + - name: dwh_extracted_at_utc data_type: timestamp with time zone - description: "{{ doc('dwh_extracted_at_date_desc') }}" + description: "{{ doc('dwh_extracted_at_utc_desc') }}" + + - name: dwh_extracted_date_utc + data_type: date + description: "{{ doc('dwh_extracted_date_utc_desc') }}" - name: stg_stripe__payment_intents description: | A PaymentIntent guides you through the process of collecting a payment from your customer. We recommend that you create exactly one PaymentIntent for each order or customer session in your system. You can reference the PaymentIntent later to see the history of payment attempts for a particular session. @@ -282,8 +332,8 @@ models: description: "{{ doc('generic_id_desc') }}" - name: amount - data_type: bigint - description: "Amount intended to be collected by this PaymentIntent. A positive integer representing how much to charge in the smallest currency unit (e.g., 100 cents to charge $1.00 or 100 to charge ¥100, a zero-decimal currency). The minimum amount is $0.50 US or equivalent in charge currency. The amount value supports up to eight digits (e.g., a value of 99999999 for a USD charge of $999,999.99)." + data_type: numeric + description: "Amount intended to be collected by this PaymentIntent. A positive integer representing how much to charge. The minimum amount is $0.50 US or equivalent in charge currency." - name: object data_type: character varying @@ -309,6 +359,10 @@ models: data_type: timestamp with time zone description: "The timestamp at which the object was created." + - name: created_date_utc + data_type: timestamp with time zone + description: "The date when which the object was created." + - name: invoice data_type: character varying description: "ID of the invoice that created this PaymentIntent, if it exists." @@ -317,9 +371,13 @@ models: data_type: timestamp with time zone description: "The timestamp at which the object was last updated." + - name: updated_date_utc + data_type: date + description: "The date when the object was last updated." + - name: currency data_type: character varying - description: "{{ doc('stripe_currency_desc') }}" + description: "{{ doc('currency_desc') }}" - name: customer data_type: character varying @@ -354,6 +412,10 @@ models: data_type: timestamp with time zone description: "Populated when status is canceled, this is the time at which the PaymentIntent was canceled." + - name: canceled_date_utc + data_type: date + description: "Populated when status is canceled, this is the date at which the PaymentIntent was canceled." + - name: description data_type: character varying description: "An arbitrary string attached to the object. Often useful for displaying to users." @@ -464,14 +526,22 @@ models: data_type: jsonb description: "{{ doc('field_docs_na_desc') }}" - - name: dwh_extracted_at_date + - name: stripe_source_account + data_type: character varying + description: "{{ doc('stripe_source_account_desc')}}" + + - name: dwh_extracted_at_utc data_type: timestamp with time zone - description: "{{ doc('dwh_extracted_at_date_desc') }}" + description: "{{ doc('dwh_extracted_at_utc_desc') }}" + + - name: dwh_extracted_date_utc + data_type: date + description: "{{ doc('dwh_extracted_date_utc_desc') }}" - name: stg_stripe__refunds description: | - Refund objects allow you to refund a previously created charge that isn’t refunded yet. Funds are refunded to the credit or debit card that’s initially charged. + Refund objects allow you to refund a previously created charge that isn’t refunded yet. Funds are refunded to the credit or debit card that’s initially charged. - You can read more about this object here: https://docs.stripe.com/api/refunds. + You can read more about this object here: https://docs.stripe.com/api/refunds. columns: - name: id data_type: character varying @@ -479,7 +549,7 @@ models: - name: amount data_type: bigint - description: "Amount, in cents." + description: "Amount for the refund." - name: charge data_type: character varying @@ -501,13 +571,21 @@ models: data_type: timestamp with time zone description: "Time at which the object was created." + - name: created_date_utc + data_type: timestamp with time zone + description: "The date when which the object was created." + - name: updated_at_utc data_type: timestamp with time zone description: "Time at which the object was last updated." + - name: updated_date_utc + data_type: date + description: "The date when the object was last updated." + - name: currency data_type: character varying - description: "{{ doc('stripe_currency_desc') }}" + description: "{{ doc('currency_desc') }}" - name: metadata data_type: jsonb @@ -537,6 +615,14 @@ models: data_type: character varying description: "The transfer reversal that’s associated with the refund. Only present if the charge came from another Stripe account." - - name: dwh_extracted_at_date + - name: stripe_source_account + data_type: character varying + description: "{{ doc('stripe_source_account_desc')}}" + + - name: dwh_extracted_at_utc data_type: timestamp with time zone - description: "{{ doc ('dwh_extracted_at_date_desc') }}" + description: "{{ doc('dwh_extracted_at_utc_desc') }}" + + - name: dwh_extracted_date_utc + data_type: date + description: "{{ doc('dwh_extracted_date_utc_desc') }}" diff --git a/models/staging/stripe/stg_stripe__balance_transactions.sql b/models/staging/stripe/stg_stripe__balance_transactions.sql index 6a0c172..b16b433 100644 --- a/models/staging/stripe/stg_stripe__balance_transactions.sql +++ b/models/staging/stripe/stg_stripe__balance_transactions.sql @@ -1,28 +1,51 @@ with raw_balance_transactions as ( - select * from {{ source("stripe_uk", "balance_transactions") }} + {{ generate_stripe_sources_unioned_select("balance_transactions") }} ), + stg_seed__currencies as (select * from {{ ref("stg_seed__currencies") }}), stg_stripe__balance_transactions as ( select {{ adapter.quote("id") }}, - {{ adapter.quote("fee") }}, - {{ adapter.quote("net") }}, + cast( + {{ adapter.quote("fee") }} + / (10 ^ cur.decimal_positions) as numeric(19, 4) + ) as fee, + cast( + {{ adapter.quote("net") }} + / (10 ^ cur.decimal_positions) as numeric(19, 4) + ) as net, {{ adapter.quote("type") }}, - {{ adapter.quote("amount") }}, + cast( + {{ adapter.quote("amount") }} + / (10 ^ cur.decimal_positions) as numeric(19, 4) + ) as amount, {{ adapter.quote("object") }}, {{ adapter.quote("source") }}, {{ adapter.quote("status") }}, to_timestamp({{ adapter.quote("created") }}) as created_at_utc, - {{ adapter.quote("currency") }}, + cast( + to_timestamp({{ adapter.quote("created") }}) as date + ) as created_date_utc, + upper({{ adapter.quote("currency") }}) as currency, {{ adapter.quote("description") }}, {{ adapter.quote("fee_details") }}, to_timestamp({{ adapter.quote("available_on") }}) as available_at_utc, + cast( + to_timestamp({{ adapter.quote("available_on") }}) as date + ) as available_date_utc, {{ adapter.quote("exchange_rate") }}, {{ adapter.quote("sourced_transfers") }}, {{ adapter.quote("reporting_category") }}, - {{ adapter.quote("_airbyte_extracted_at") }} as dwh_extracted_at_date + {{ adapter.quote("stripe_source_account") }}, + {{ adapter.quote("_airbyte_extracted_at") }} as dwh_extracted_at_utc, + cast( + {{ adapter.quote("_airbyte_extracted_at") }} as date + ) as dwh_extracted_date_utc - from raw_balance_transactions + from raw_balance_transactions bt + left join + stg_seed__currencies cur + on upper(bt.{{ adapter.quote("currency") }}) = cur.iso_4217_code ) select * from stg_stripe__balance_transactions diff --git a/models/staging/stripe/stg_stripe__charges.sql b/models/staging/stripe/stg_stripe__charges.sql index 3267009..d898ea3 100644 --- a/models/staging/stripe/stg_stripe__charges.sql +++ b/models/staging/stripe/stg_stripe__charges.sql @@ -1,24 +1,34 @@ with - raw_charges as (select * from {{ source("stripe_uk", "charges") }}), + raw_charges as ({{ generate_stripe_sources_unioned_select("charges") }}), + stg_seed__currencies as (select * from {{ ref("stg_seed__currencies") }}), stg_stripe__charges as ( select {{ adapter.quote("id") }}, {{ adapter.quote("card") }}, cast({{ adapter.quote("paid") }} as bool) as is_paid, {{ adapter.quote("order") }}, - {{ adapter.quote("amount") }}, + cast( + {{ adapter.quote("amount") }} + / (10 ^ cur.decimal_positions) as numeric(19, 4) + ) as amount, {{ adapter.quote("object") }}, {{ adapter.quote("review") }}, {{ adapter.quote("source") }}, {{ adapter.quote("status") }}, to_timestamp({{ adapter.quote("created") }}) as created_at_utc, + cast( + to_timestamp({{ adapter.quote("created") }}) as date + ) as created_date_utc, {{ adapter.quote("dispute") }}, {{ adapter.quote("invoice") }}, {{ adapter.quote("outcome") }}, {{ adapter.quote("refunds") }}, to_timestamp({{ adapter.quote("updated") }}) as updated_at_utc, + cast( + to_timestamp({{ adapter.quote("updated") }}) as date + ) as updated_date_utc, cast({{ adapter.quote("captured") }} as bool) as is_captured, - {{ adapter.quote("currency") }}, + upper({{ adapter.quote("currency") }}) as currency, {{ adapter.quote("customer") }}, cast({{ adapter.quote("disputed") }} as bool) as is_disputed, cast({{ adapter.quote("livemode") }} as bool) as is_livemode, @@ -39,8 +49,14 @@ with {{ adapter.quote("payment_method") }}, {{ adapter.quote("receipt_number") }}, {{ adapter.quote("transfer_group") }}, - {{ adapter.quote("amount_captured") }}, - {{ adapter.quote("amount_refunded") }}, + cast( + {{ adapter.quote("amount_captured") }} + / (10 ^ cur.decimal_positions) as numeric(19, 4) + ) as amount_captured, + cast( + {{ adapter.quote("amount_refunded") }} + / (10 ^ cur.decimal_positions) as numeric(19, 4) + ) as amount_refunded, {{ adapter.quote("application_fee") }}, {{ adapter.quote("billing_details") }}, {{ adapter.quote("failure_message") }}, @@ -53,9 +69,16 @@ with {{ adapter.quote("failure_balance_transaction") }}, {{ adapter.quote("statement_descriptor_suffix") }}, {{ adapter.quote("calculated_statement_descriptor") }}, - {{ adapter.quote("_airbyte_extracted_at") }} as dwh_extracted_at_date + {{ adapter.quote("stripe_source_account") }}, + {{ adapter.quote("_airbyte_extracted_at") }} as dwh_extracted_at_utc, + cast( + {{ adapter.quote("_airbyte_extracted_at") }} as date + ) as dwh_extracted_date_utc - from raw_charges + from raw_charges ch + left join + stg_seed__currencies cur + on upper(ch.{{ adapter.quote("currency") }}) = cur.iso_4217_code ) select * from stg_stripe__charges diff --git a/models/staging/stripe/stg_stripe__payment_intents.sql b/models/staging/stripe/stg_stripe__payment_intents.sql index aedcc39..05482c0 100644 --- a/models/staging/stripe/stg_stripe__payment_intents.sql +++ b/models/staging/stripe/stg_stripe__payment_intents.sql @@ -1,18 +1,30 @@ with - raw_payment_intents as (select * from {{ source("stripe_uk", "payment_intents") }}), + raw_payment_intents as ( + {{ generate_stripe_sources_unioned_select("payment_intents") }} + ), + stg_seed__currencies as (select * from {{ ref("stg_seed__currencies") }}), stg_stripe__payment_intents as ( select {{ adapter.quote("id") }}, - {{ adapter.quote("amount") }}, + cast( + {{ adapter.quote("amount") }} + / (10 ^ cur.decimal_positions) as numeric(19, 4) + ) as amount, {{ adapter.quote("object") }}, {{ adapter.quote("review") }}, {{ adapter.quote("source") }}, {{ adapter.quote("status") }}, {{ adapter.quote("charges") }}, to_timestamp({{ adapter.quote("created") }}) as created_at_utc, + cast( + to_timestamp({{ adapter.quote("created") }}) as date + ) as created_date_utc, {{ adapter.quote("invoice") }}, to_timestamp({{ adapter.quote("updated") }}) as updated_at_utc, - {{ adapter.quote("currency") }}, + cast( + to_timestamp({{ adapter.quote("updated") }}) as date + ) as updated_date_utc, + upper({{ adapter.quote("currency") }}) as currency, {{ adapter.quote("customer") }}, cast({{ adapter.quote("livemode") }} as bool) as is_livemode, {{ adapter.quote("metadata") }}, @@ -20,6 +32,9 @@ with {{ adapter.quote("processing") }}, {{ adapter.quote("application") }}, to_timestamp({{ adapter.quote("canceled_at") }}) as canceled_at_utc, + cast( + to_timestamp({{ adapter.quote("canceled_at") }}) as date + ) as canceled_date_utc, {{ adapter.quote("description") }}, {{ adapter.quote("next_action") }}, {{ adapter.quote("on_behalf_of") }}, @@ -31,8 +46,14 @@ with {{ adapter.quote("capture_method") }}, {{ adapter.quote("payment_method") }}, {{ adapter.quote("transfer_group") }}, - {{ adapter.quote("amount_received") }}, - {{ adapter.quote("amount_capturable") }}, + cast( + {{ adapter.quote("amount_received") }} + / (10 ^ cur.decimal_positions) as numeric(19, 4) + ) as amount_received, + cast( + {{ adapter.quote("amount_capturable") }} + / (10 ^ cur.decimal_positions) as numeric(19, 4) + ) as amount_capturable, {{ adapter.quote("last_payment_error") }}, {{ adapter.quote("setup_future_usage") }}, {{ adapter.quote("cancellation_reason") }}, @@ -45,9 +66,16 @@ with {{ adapter.quote("automatic_payment_methods") }}, {{ adapter.quote("statement_descriptor_suffix") }}, {{ adapter.quote("payment_method_configuration_details") }}, - {{ adapter.quote("_airbyte_extracted_at") }} as dwh_extracted_at_date + {{ adapter.quote("stripe_source_account") }}, + {{ adapter.quote("_airbyte_extracted_at") }} as dwh_extracted_at_utc, + cast( + {{ adapter.quote("_airbyte_extracted_at") }} as date + ) as dwh_extracted_date_utc - from raw_payment_intents + from raw_payment_intents pi + left join + stg_seed__currencies cur + on upper(pi.{{ adapter.quote("currency") }}) = cur.iso_4217_code ) select * from stg_stripe__payment_intents diff --git a/models/staging/stripe/stg_stripe__refunds.sql b/models/staging/stripe/stg_stripe__refunds.sql index 91a2cad..48a2c7d 100644 --- a/models/staging/stripe/stg_stripe__refunds.sql +++ b/models/staging/stripe/stg_stripe__refunds.sql @@ -1,16 +1,26 @@ with - raw_refunds as (select * from {{ source("stripe_uk", "refunds") }}), + raw_refunds as ({{ generate_stripe_sources_unioned_select("refunds") }}), + stg_seed__currencies as (select * from {{ ref("stg_seed__currencies") }}), stg_stripe__refunds as ( select {{ adapter.quote("id") }}, - {{ adapter.quote("amount") }}, + cast( + {{ adapter.quote("amount") }} + / (10 ^ cur.decimal_positions) as numeric(19, 4) + ) as amount, {{ adapter.quote("charge") }}, {{ adapter.quote("object") }}, {{ adapter.quote("reason") }}, {{ adapter.quote("status") }}, to_timestamp({{ adapter.quote("created") }}) as created_at_utc, + cast( + to_timestamp({{ adapter.quote("created") }}) as date + ) as created_date_utc, to_timestamp({{ adapter.quote("updated") }}) as updated_at_utc, - {{ adapter.quote("currency") }}, + cast( + to_timestamp({{ adapter.quote("updated") }}) as date + ) as updated_date_utc, + upper({{ adapter.quote("currency") }}) as currency, {{ adapter.quote("metadata") }}, {{ adapter.quote("payment_intent") }}, {{ adapter.quote("receipt_number") }}, @@ -18,9 +28,16 @@ with {{ adapter.quote("balance_transaction") }}, {{ adapter.quote("destination_details") }}, {{ adapter.quote("source_transfer_reversal") }}, - {{ adapter.quote("_airbyte_extracted_at") }} as dwh_extracted_at_date + {{ adapter.quote("stripe_source_account") }}, + {{ adapter.quote("_airbyte_extracted_at") }} as dwh_extracted_at_utc, + cast( + {{ adapter.quote("_airbyte_extracted_at") }} as date + ) as dwh_extracted_date_utc - from raw_refunds + from raw_refunds r + left join + stg_seed__currencies cur + on upper(r.{{ adapter.quote("currency") }}) = cur.iso_4217_code ) select * from stg_stripe__refunds diff --git a/models/staging/stripe/stripe_doc_blocks.md b/models/staging/stripe/stripe_doc_blocks.md index a8c073c..2ed7181 100644 --- a/models/staging/stripe/stripe_doc_blocks.md +++ b/models/staging/stripe/stripe_doc_blocks.md @@ -10,7 +10,7 @@ Time at which the object was last updated. Measured in seconds since the Unix ep {% enddocs %} -{% docs stripe_currency_desc %} +{% docs raw_stripe_currency_desc %} Three-letter ISO currency code, in lowercase. Must be a supported currency. @@ -27,3 +27,9 @@ Has the value true if the object exists in live mode or the value false if the o Additional properties supplied by Superhog. {% enddocs %} + +{% docs stripe_source_account_desc %} + +Superhog uses multiple different Stripe account. This column indicates from which of them is the data coming from. + +{% enddocs %}