Merged PR 2113: Bank Transactions to reporting

- Pushes the `int` model for bank transactions to reporting.
- A few silly fixes along the way.

Related work items: #17551

models/intermediate/xero/int_xero__bank_transactions.sql

@@ -32,7 +32,6 @@ select
     -- oficial dwh rate.
     coalesce(bt.exchange_rate_to_gbp, ser.rate) as exchange_rate_to_gbp,
     bt.line_amount_tax_inclusiveness,
-    bt.line_items,
     bt.is_reconciled,
     bt.has_attachments,
     bt.url,

models/reporting/xero/schema.yaml

@@ -445,411 +445,6 @@ models:
       - name: dwh_extracted_at_utc
         data_type: timestamp with time zone
         description: ""
-  - name: xero__denom_invoicing_mart
-    description: |
-      This is denormalized table built upon contacts, invoice and invoice line
-      items. It's just built to save you time in joining and for dealing with
-      certain limitations in PBI.
-
-      The level of granularity is at line item. Line items there are joined 
-      with invoices by the id of the invoice present in the line item, and 
-      contacts are joined by the id of the contact present in the invoice.
-      Be careful when aggregating figures: sums on the invoice line item level
-      make sense, sums on the invoice level do not.
-
-      Data from the original entities is brought as is with no transformation.
-      Please, when modelling here, try to respect this approach.
-
-    columns:
-      - name: id_line_item
-        data_type: text
-        description: "Xero's unique identifier for the line item."
-
-      - name: id_item
-        data_type: text
-        description: "Xero's unique identifier for the item of this line."
-
-      - name: item_code
-        data_type: text
-        description: "A text based code that describes the Item."
-
-      - name: quantity
-        data_type: numeric
-        description: "How much of the item is sold in its invoice."
-
-      - name: unit_amount
-        data_type: numeric
-        description: "The price of each unit of the item."
-
-      - name: line_amount_local_curr
-        data_type: numeric
-        description: |
-          The total value to be charged for this line, in local currency.
-
-      - name: line_amount_in_gbp
-        data_type: numeric
-        description: "The total value to be charged for this line, in GBP."
-
-      - name: line_amount_wo_taxes_local_curr
-        data_type: numeric
-        description: |
-
-          The total value to be charged for this line, minus taxes, in local 
-          currency.
-
-          In some cases, this will be the same value as the line amount. This
-          depends on whether there is any tax in the invoice, and whether the
-          invoice amounts are tax inclusive or tax exclusive. You can find this
-          out in the details of the invoice this line item belongs to.
-
-      - name: line_amount_wo_taxes_in_gbp
-        data_type: numeric
-        description: |
-          The total value to be charged for this line, minus taxes, in GBP.
-
-          In some cases, this will be the same value as the line amount. This
-          depends on whether there is any tax in the invoice, and whether the
-          invoice amounts are tax inclusive or tax exclusive. You can find this
-          out in the details of the invoice this line item belongs to.
-
-      - name: tax_amount_local_curr
-        data_type: numeric
-        description: "The total value of taxes for this line, in local currency."
-
-      - name: tax_amount_in_gbp
-        data_type: numeric
-        description: "The total value of taxes for this line, in GBP."
-
-      - name: tax_type
-        data_type: text
-        description: ""
-
-      - name: line_description
-        data_type: text
-        description: "A text description of the line item."
-
-      - name: id_invoice
-        data_type: character varying
-        description: "Xero's unique identifier for the invoice."
-
-      - name: reference
-        data_type: character varying
-        description: |
-          The Superhog-set reference for the invoices. Only relevant for 
-          records of invoice type (ACCREC).
-
-      - name: invoice_number
-        data_type: character varying
-        description: ""
-
-      - name: invoice_issued_date_utc
-        data_type: timestamp with time zone
-        description: "Date on which the invoice was issued."
-
-      - name: invoice_due_date_utc
-        data_type: timestamp with time zone
-        description: "The date were the invoice should be paid the latest."
-
-      - name: was_fully_paid_on_date_utc
-        data_type: timestamp with time zone
-        description: |
-          The date on which the invoice was fully paid. If the invoice is still
-          not fully paid, it returns null.
-
-      - name: invoice_type
-        data_type: character varying
-        description: |
-          This field indicates whether the invoice is from Superhog towards a customer 
-          (value is "ACCREC") or from a supplier towards Superhog (value is "ACCPAY").
-
-      - name: invoice_currency_iso_4217
-        data_type: character varying
-        description: |
-          The currency in which this line item amounts are defined.
-
-          Line items don't really have a currency definition of its own, this
-          field is populated for your convenience when working with this data.
-          The invoice currency is defined at the invoice level, and the values
-          you see in this field should always be the same as the currency of
-          the invoice this line item belongs to.
-
-      - name: exchange_rate_to_gbp
-        data_type: numeric
-        description: |
-          The implied exchange rate between the invoice currency and GBP.
-
-          This is the XXX to GBP rate. So the number represented in this field 
-          is the units of currency GBP that 1 unit of XXX buys you.
-
-          For example, if the invoice is 135 ZAR, and this rate is 0.0167, the
-          invoice value in GBP is 2.25 (135ZAR * 0.0167GBP/ZAR = 2.25GBP).
-
-      - name: total_amount_local_curr
-        data_type: numeric
-        description: |
-          The total amount to be paid, in the currency the invoice is 
-          denominated in.
-
-      - name: total_amount_in_gbp
-        data_type: numeric
-        description: "The total amount to be paid, converted to GBP."
-
-      - name: total_amount_wo_tax_local_curr
-        data_type: numeric
-        description: |
-          The total amount to be paid minus taxes, in the currency the invoice
-          is denominated in.
-
-      - name: total_amount_wo_tax_in_gbp
-        data_type: numeric
-        description: |
-          The total amount to be paid minus taxes, converted to GBP.
-
-      - name: total_tax_local_curr
-        data_type: numeric
-        description: |
-          The total tax, in the currency the invoice is denominated in.
-
-      - name: total_tax_in_gbp
-        data_type: numeric
-        description: "The total tax, converted to GBP."
-
-      - name: total_due_local_curr
-        data_type: numeric
-        description: |
-          The total amount outstanding right now, in the currency the invoice 
-          is denominated in.
-
-      - name: total_due_in_gbp
-        data_type: numeric
-        description: |
-          The total amount outstanding right now, converted to GBP.
-
-      - name: total_paid_local_curr
-        data_type: numeric
-        description: |
-          The total amount that has already been paid, in the currency the 
-          invoice is denominated in.
-
-      - name: total_paid_in_gbp
-        data_type: numeric
-        description: "The total amount that has already been paid, in GBP."
-
-      - name: invoice_status
-        data_type: character varying
-        description: |
-
-          The status of the invoice.
-
-          Can be one of PAID, VOIDED, DRAFT, DELETED, AUTHORISED, SUBMITTED.
-
-          You can read more here https://developer.xero.com/documentation/api/accounting/types#invoices
-
-      - name: has_errors
-        data_type: boolean
-        description: ""
-
-      - name: is_discounted
-        data_type: boolean
-        description: ""
-
-      - name: date_string
-        data_type: timestamp with time zone
-        description: ""
-
-      - name: due_date_string
-        data_type: timestamp with time zone
-        description: ""
-
-      - name: has_been_sent_to_contact
-        data_type: boolean
-        description: ""
-
-      - name: total_discount
-        data_type: numeric
-        description: ""
-
-      - name: total_credited_local_curr
-        data_type: numeric
-        description: |
-          Total amount credited by related credit notes, 
-          over-payments and pre-payments.
-
-      - name: has_attachments
-        data_type: boolean
-        description: ""
-
-      - name: line_amount_tax_inclusiveness
-        data_type: character varying
-        description: |
-          Indicates whether the amounts included in line items are tax
-          inclusive (Inclusive), tax exclusive (Exclusive) or simply,
-          there are no taxes on the invoice (NoTax).
-
-      - name: payment_planned_date_utc
-        data_type: timestamp with time zone
-        description: ""
-
-      - name: payment_expected_date_utc
-        data_type: timestamp with time zone
-        description: ""
-
-      - name: id_contact
-        data_type: character varying
-        description: "Xero's unique identifier for the contact."
-
-      - name: account_number
-        data_type: character varying
-        description: |
-          A Superhog set identifier. For customers, this is Hubspot's Deal Id.
-
-      - name: id_deal
-        data_type: text
-        description: |
-          For those account numbers that look like a Deal Id, we extract them
-          into this field. Other contacs will have a blank.
-
-          You can use this to find specific deal ids or even as a proxy bool to
-          keep or remove deal id customers.
-
-      - name: contact_name
-        data_type: character varying
-        description: "The name for the customer."
-
-      - name: is_customer
-        data_type: boolean
-        description: "Flag that shows if the contact is a Superhog customer."
-
-      - name: is_supplier
-        data_type: boolean
-        description: "Flag that shows if the contact is a Superhog supplier."
-
-      - name: tax_number
-        data_type: character varying
-        description: "Tax number for this contact."
-
-      - name: contact_first_name
-        data_type: character varying
-        description: ""
-
-      - name: contact_last_name
-        data_type: character varying
-        description: ""
-
-      - name: phones
-        data_type: jsonb
-        description: ""
-
-      - name: website
-        data_type: character varying
-        description: ""
-
-      - name: balances
-        data_type: jsonb
-        description: ""
-
-      - name: discount
-        data_type: numeric
-        description: ""
-
-      - name: addresses
-        data_type: jsonb
-        description: ""
-
-      - name: attachments
-        data_type: jsonb
-        description: ""
-
-      - name: email_addresses
-        data_type: character varying
-        description: ""
-
-      - name: payment_terms
-        data_type: jsonb
-        description: ""
-
-      - name: batch_payments
-        data_type: jsonb
-        description: ""
-
-      - name: branding_theme
-        data_type: jsonb
-        description: ""
-
-      - name: contact_groups
-        data_type: jsonb
-        description: ""
-
-      - name: contact_number
-        data_type: character varying
-        description: ""
-
-      - name: contact_status
-        data_type: character varying
-        description: ""
-
-      - name: skyper_user_name
-        data_type: character varying
-        description: ""
-
-      - name: contact_persons
-        data_type: jsonb
-        description: ""
-
-      - name: xero_network_key
-        data_type: character varying
-        description: ""
-
-      - name: default_currency_iso_4217
-        data_type: character varying
-        description: |
-          The default currency used to interact with this contact.
-
-          For customers, this is the currency we will usually use to invoice.
-
-      - name: validation_errors
-        data_type: jsonb
-        description: ""
-
-      - name: bank_account_details
-        data_type: character varying
-        description: ""
-
-      - name: has_validation_errors
-        data_type: boolean
-        description: ""
-
-      - name: tracking_category_name
-        data_type: character varying
-        description: ""
-
-      - name: account_payable_tax_type
-        data_type: character varying
-        description: ""
-
-      - name: tracking_category_option
-        data_type: character varying
-        description: ""
-
-      - name: sales_default_account_code
-        data_type: character varying
-        description: ""
-
-      - name: sales_tracking_categories
-        data_type: jsonb
-        description: ""
-
-      - name: account_receivable_tax_type
-        data_type: character varying
-        description: ""
-
-      - name: purchases_default_account_code
-        data_type: character varying
-        description: ""
-
-      - name: purchases_tracking_categories
-        data_type: jsonb
-        description: ""
   - name: xero__credit_notes
     description: |
       Credit notes that we have raised within our accounting books.
@@ -882,7 +477,7 @@ models:
       - name: reference
         data_type: character varying
         description: |
-          The Superhog-set reference for the credit note. 
+          The Superhog-set reference for the credit note.
 
       - name: credit_note_number
         data_type: character varying
@@ -909,7 +504,7 @@ models:
       - name: credit_note_currency_iso_4217
         data_type: character varying
         description: |
-        
+
           The ISO 4217 code for the currency which this credit note uses as 
           unit of account.
 
@@ -926,7 +521,7 @@ models:
 
       - name: total_amount_local_curr
         data_type: numeric
-        description:  | 
+        description: |
           The total amount to be credited, in the currency the note is 
           denominated in.
 
@@ -937,8 +532,8 @@ models:
       - name: subtotal_local_curr
         data_type: numeric
         description: |
-           The total amount to be credited minus taxes, in the currency the 
+          The total amount to be credited minus taxes, in the currency the 
-           note is denominated in.
+          note is denominated in.
 
       - name: subtotal_in_gbp
         data_type: numeric
@@ -1034,7 +629,7 @@ models:
     columns:
       - name: id_credit_note
         data_type: character varying
-        description:  Xero's unique identifier for the credit note.
+        description: Xero's unique identifier for the credit note.
 
       - name: id_line_item
         data_type: text
@@ -1133,7 +728,7 @@ models:
 
       - name: fee_category
         data_type: text
-        description: | 
+        description: |
           A fee category, one of out of booking_fees, listing_fees, 
           verification_fees, waiver fees.
 
@@ -1147,7 +742,7 @@ models:
 
       - name: net_fees_in_gbp
         data_type: numeric
-        description: | 
+        description: |
           Net fees are calculated by subtracting credited fees from invoiced
           fees. It might be the case that net fees are negative in some month,
           if it happens to be that we have given back to customers more than we
@@ -1172,7 +767,7 @@ models:
 
       - name: id_deal
         data_type: text
-        description: | 
+        description: |
           A fee category per deal.
 
       - name: fees_invoiced_in_gbp
@@ -1185,8 +780,206 @@ models:
 
       - name: net_fees_in_gbp
         data_type: numeric
-        description: | 
+        description: |
           Net fees are calculated by subtracting credited fees from invoiced
           fees. It might be the case that net fees are negative in some month,
           if it happens to be that we have given back to customers more than we
           have invoiced them.
+  - name: xero__bank_transactions
+    description: |
+      Bank transactions that have happened in any of our bank accounts.
+
+      You can read more here: https://developer.xero.com/documentation/api/accounting/banktransactions
+
+    columns:
+      - name: id_bank_transaction
+        data_type: character varying
+        description: Xero's unique identifier for the transaction.
+        tests:
+          - not_null
+          - unique
+      
+      - name: id_prepayment
+        data_type: character varying
+        description: |
+          Xero generated identifier for a Prepayment (unique within 
+          organisations). This will be returned on BankTransactions with a Type 
+          of SPEND-PREPAYMENT or RECEIVE-PREPAYMENT.
+
+      - name: id_overpayment
+        data_type: character varying
+        description: Xero generated identifier for an Overpayment (unique within
+          organisations). This will be returned on BankTransactions with a Type
+          of SPEND-OVERPAYMENT or RECEIVE-OVERPAYMENT.
+
+      - name: id_contact
+        data_type: text
+        description: |
+          Xero's unique id for the contact related to this transaction.
+
+      - name: id_bank_account
+        data_type: text
+        description: |
+          Xero's unique id for the bank account related to this transaction.
+
+      - name: reference
+        data_type: character varying
+        description: |
+          Reference for the transaction. Only supported for SPEND and RECEIVE 
+          transactions.
+
+      - name: transaction_type
+        data_type: character varying
+        description: |
+          What kind of transaction is this record.
+
+          Can be one of: RECEIVE, RECEIVE-OVERPAYMENT, RECEIVE-PREPAYMENT, SPEND, SPEND-OVERPAYMENT, SPEND-PREPAYMENT, RECEIVE-TRANSFER, SPEND-TRANSFER
+
+          You can read more here: https://developer.xero.com/documentation/api/accounting/types#bank-transactions
+        tests:
+          - not_null
+          - accepted_values:
+              values:
+                - RECEIVE
+                - RECEIVE-OVERPAYMENT
+                - RECEIVE-PREPAYMENT
+                - SPEND
+                - SPEND-OVERPAYMENT
+                - SPEND-PREPAYMENT
+                - RECEIVE-TRANSFER
+                - SPEND-TRANSFER
+
+
+      - name: transaction_status
+        data_type: character varying
+        description: |
+          The status of this transaction.
+
+          Can be one of: AUTHORISED, DELETED.
+
+          You can read more here: https://developer.xero.com/documentation/api/accounting/types#bank-transaction-status-codes
+        tests:
+          - not_null
+          - accepted_values:
+              values:
+                - AUTHORISED
+                - DELETED
+
+      - name: transaction_at_utc
+        data_type: timestamp with time zone
+        description: When did the transaction happen.
+
+      - name: transaction_date_utc
+        data_type: date
+        description: The date on which the transaction happened.
+
+      - name: date_string
+        data_type: timestamp with time zone
+        description: When did the transaction happened, as a string.
+
+      - name: total_amount_local_curr
+        data_type: numeric
+        description: |
+          Total of bank transaction tax inclusive, in the currency the 
+          transaction is denominated in.
+        tests:
+          - not_null
+
+      - name: total_amount_in_gbp
+        data_type: numeric
+        description: |
+          Total of bank transaction tax inclusive, presented in GBP.
+
+      - name: total_amount_wo_tax_local_curr
+        data_type: numeric
+        description: |
+          Total of bank transaction excluding taxes, in the currency the 
+          transaction is denominated in.
+
+      - name: total_amount_wo_tax_in_gbp
+        data_type: numeric
+        description: |
+          Total of bank transaction excluding taxes, presented in GBP.
+
+      - name: total_tax_local_curr
+        data_type: numeric
+        description: |
+          Total tax on bank transaction, in the currency the transaction is 
+          denominated in.
+
+      - name: total_tax_in_gbp
+        data_type: numeric
+        description: |
+          Total tax on bank transaction, presented in GBP.
+
+      - name: transaction_currency_iso_4217
+        data_type: character varying
+        description: The ISO 4217 code for the currency of the transaction.
+        tests:
+          - not_null
+          - length_between:
+              min_length: 3
+              max_length: 3
+
+      - name: exchange_rate_to_gbp
+        data_type: numeric
+        description: |
+
+          ATTENTION! unlike in many other Xero entities, here this field is not 
+          completed by Xero always. It has been complemented with the general 
+          rates data used in the DWH. Xero's rates are preferent, so DWH rates 
+          will only be used where Xero didn't provide a rate.
+
+          The implied exchange rate between the transaction currency and GBP.
+
+          This is the XXX to GBP rate. So the number represented in this field 
+          is the units of currency GBP that 1 unit of XXX buys you. 
+
+          For example, if the transaction is 135 ZAR, and this rate is 0.0167, 
+          the transaction value in GBP is 2.25 (135ZAR * 0.0167GBP/ZAR = 
+          2.25GBP).
+        tests:
+          - not_negative
+
+      - name: line_amount_tax_inclusiveness
+        data_type: character varying
+        description: |
+          Indicates whether the amounts included in line items are tax 
+          inclusive (Inclusive), tax exclusive (Exclusive) or simply,
+          there are no taxes on the invoice (NoTax).
+        tests:
+          - accepted_values:
+              values:
+                - Inclusive
+                - Exclusive
+                - NoTax
+
+      - name: is_reconciled
+        data_type: boolean
+        description: A flag indicating if the transaction has been reconciled.
+        tests:
+          - not_null
+
+      - name: has_attachments
+        data_type: boolean
+        description: ""
+
+      - name: url
+        data_type: character varying
+        description: ""
+
+      - name: external_link_provider_name
+        data_type: character varying
+        description: ""
+
+      - name: updated_at_utc
+        data_type: timestamp with time zone
+        description: ""
+
+      - name: updated_date_utc
+        data_type: date
+        description: ""
+
+      - name: dwh_extracted_at_utc
+        data_type: timestamp with time zone
+        description: ""

models/reporting/xero/xero__bank_transactions.sql

@@ -0,0 +1,33 @@
+with
+    int_xero__bank_transactions as (
+        select * from {{ ref("int_xero__bank_transactions") }}
+    )
+select
+    bt.id_bank_transaction as id_bank_transaction,
+    bt.id_prepayment as id_prepayment,
+    bt.id_overpayment as id_overpayment,
+    bt.id_contact as id_contact,
+    bt.id_bank_account as id_bank_account,
+    bt.reference as reference,
+    bt.transaction_type as transaction_type,
+    bt.transaction_status as transaction_status,
+    bt.transaction_at_utc as transaction_at_utc,
+    bt.transaction_date_utc as transaction_date_utc,
+    bt.date_string as date_string,
+    bt.total_amount_local_curr as total_amount_local_curr,
+    bt.total_amount_in_gbp as total_amount_in_gbp,
+    bt.total_amount_wo_tax_local_curr as total_amount_wo_tax_local_curr,
+    bt.total_amount_wo_tax_in_gbp as total_amount_wo_tax_in_gbp,
+    bt.total_tax_local_curr as total_tax_local_curr,
+    bt.total_tax_in_gbp as total_tax_in_gbp,
+    bt.transaction_currency_iso_4217 as transaction_currency_iso_4217,
+    bt.exchange_rate_to_gbp as exchange_rate_to_gbp,
+    bt.line_amount_tax_inclusiveness as line_amount_tax_inclusiveness,
+    bt.is_reconciled as is_reconciled,
+    bt.has_attachments as has_attachments,
+    bt.url as url,
+    bt.external_link_provider_name as external_link_provider_name,
+    bt.updated_at_utc as updated_at_utc,
+    bt.updated_date_utc as updated_date_utc,
+    bt.dwh_extracted_at_utc as dwh_extracted_at_utc
+from int_xero__bank_transactions bt

models/staging/xero/schema.yml

@@ -672,7 +672,7 @@ models:
         tests:
           - not_null
 
-      - name: subtotal_local_curr
+      - name: total_amount_wo_tax_local_curr
         data_type: numeric
         description: |
           Total of bank transaction excluding taxes, in the currency the 
@@ -700,7 +700,7 @@ models:
           ATTENTION! unlike in many other Xero entities, here this field is not 
           completed by Xero always. Some records will have a rate, some won't.
           You probably need to complement these rates with some other source if
-          you want to 
+          you want to have no nulls.
 
           The implied exchange rate between the transaction currency and GBP.
 
@@ -774,6 +774,7 @@ models:
       - name: dwh_extracted_at_utc
         data_type: timestamp with time zone
         description: ""
+  
   - name: stg_xero__accounts
     description: |
       Accounts in our accounting tree.