Merged PR 5324: --pairs option for CLI

This PR implements an alternative option to `--currencies`, named `--pairs`. This options lets the user specify pairs of currencies instead of just a list of currencies (which assumes you want all pairs).

This is useful to run backfills for one new currency without having to also fetch from XE rates that are already present in the DWH.

Related work items: #29931

CHANGELOG.md

@@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.1.0] - 2025-05-7
+
+### Added
+
+- `get-rates` command now has a `--pairs` options that can be used instead of `--currencies`. `--pairs` allows the user to specify which currency pairs it wants to fetch rates for. This provides more control than `--currencies`, which assumes that the user wants ALL combinations of the listed currencies. `--pairs` will still automatically store reverse and equal rates for the passed pairs.
+
 ## [1.0.1] - 2024-06-27
 
 ### Changed

README.md

@@ -81,6 +81,15 @@ xexe get-rates --output my_rates.csv
 xexe get-rates --currencies USD,EUR,GBP --output my_rates.csv
 ```
 
+Using the `currencies` option will compute ALL combinations between different currencies. If you want a finer-grained control over which currency pairs you fetch, you can instead use the `pairs` options like this:
+
+```bash
+# Pairs must be concatenations of two valid ISO 4217 codes. Each pair must be 
+# comma-separated. No need to use reverse and equal pairs. As in, if you provide 
+# `USDEUR`, xexe will do that pair and also `EURUSD`, `EUREUR` and `USDUSD`.
+xexe get-rates --pairs USDEUR,EURGBP --output my_rates.csv
+```
+
 The output file for `.csv` outputs will follow this schema:
 
 - `date`
@@ -89,7 +98,7 @@ The output file for `.csv` outputs will follow this schema:
 - `exchange_rate`
 - `exported_at`
 
-The file will contain all the combinations of the different currencies and dates passed. This includes inverse and equal rates.
+The file will contain all the combinations of the different currencies (or pairs) and dates passed. This includes inverse and equal rates.
 
 This is better understood with an example. Find below a real call and its real CSV output:
 
@@ -133,6 +142,7 @@ A few more details:
 - Running `get-rates` with an `end-date` beyond the current date will ignore the future dates. The run will behave as if you had specified today as the `end-date`.
 - Trying to place an `end-date` before a `start-date` will cause an exception.
 - Running with the option `--dry-run` will run against a mock of the xe.com API. Format will be valid, but all rates will be fixed. This is for testing purposes.
+- `xexe` will log details to `xexe.log`.
 
 ### Deploying for Superhog infra
 

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "xexe"
-version = "1.0.1"
+version = "1.1.0"
 description = ""
 authors = ["Pablo Martin <pablo.martin@superhog.com>"]
 readme = "README.md"

tests/tests_integration/test_get_rates.py

@@ -143,3 +143,60 @@ def test_get_rates_dry_run_always_returns_42_as_rates():
             assert (
                 row["to_currency"] in some_random_currencies
             ), f"Unexpected to_currency {row['to_currency']}"
+
+
+def test_get_rates_dry_run__with_pairs_always_returns_42_as_rates():
+    """
+    Same as the test above, but relying on the pairs argument instead
+    of the currencies one.
+    """
+
+    some_random_date = datetime.datetime(
+        year=random.choice(range(2010, 2020)),
+        month=random.choice(range(1, 13)),
+        day=random.choice(range(1, 29)),
+    ).date()
+
+    some_pairs = ["USDEUR", "USDGBP", "GBPEUR"]
+
+    runner = CliRunner()
+
+    with runner.isolated_filesystem():
+        run_result = runner.invoke(
+            get_rates,
+            [
+                "--start-date",
+                some_random_date.strftime("%Y-%m-%d"),
+                "--end-date",
+                (some_random_date + datetime.timedelta(days=3)).strftime("%Y-%m-%d"),
+                "--pairs",
+                ",".join(some_pairs),
+                "--output",
+                "test_output.csv",
+            ],
+        )
+
+        assert run_result.exit_code == 0
+
+        with open("test_output.csv", newline="") as csv_file:
+            reader = csv.DictReader(csv_file)
+            rows = list(reader)
+
+        # Ensure that the output contains the correct number of rows
+        expected_num_rows = 36
+        assert (
+            len(rows) == expected_num_rows
+        ), f"Expected {expected_num_rows} rows, but got {len(rows)}"
+
+        # Check that all rows have the expected rate of 42, 1/42 or 1 and the correct dates
+        for row in rows:
+            assert row["rate"] in (
+                "42.00000000",
+                "0.02380952",
+                "0.00000000",
+                "1.00000000",
+            ), f"Expected rate to be 42, 1/42 or 1, but got {row['rate']}"
+            assert row["rate_date"] in [
+                (some_random_date + datetime.timedelta(days=i)).strftime("%Y-%m-%d")
+                for i in range(4)
+            ], f"Unexpected rate_date {row['rate_date']}"

tests/tests_unit/test_currency_pair.py

@@ -0,0 +1,46 @@
+from money.currency import Currency
+
+from xexe.currency_pair import CurrencyPair
+
+
+def test_create_currency_pair_normal_works_fine():
+
+    a_pair: CurrencyPair = CurrencyPair(
+        from_currency=Currency["USD"], to_currency=Currency["EUR"]
+    )
+
+    assert (a_pair.from_currency == "USD", a_pair.to_currency == "EUR")
+
+
+def test_create_currency_pair_with_same_currency_works():
+    same_curr_pair: CurrencyPair = CurrencyPair(
+        from_currency=Currency["USD"], to_currency=Currency["USD"]
+    )
+
+    assert (same_curr_pair.from_currency == "USD", same_curr_pair.to_currency == "USD")
+
+
+def test_reverse_pair_works_fine():
+
+    a_pair: CurrencyPair = CurrencyPair(
+        from_currency=Currency["USD"], to_currency=Currency["EUR"]
+    )
+
+    reverse_pair: CurrencyPair = a_pair.get_reverse_pair()
+
+    assert (
+        isinstance(reverse_pair, CurrencyPair),
+        reverse_pair.from_currency == "EUR",
+        reverse_pair.to_currency == "USD",
+    )
+
+
+def test_pair_equality_works():
+
+    a_pair: CurrencyPair = CurrencyPair(
+        from_currency=Currency["USD"], to_currency=Currency["EUR"]
+    )
+
+    reverse_pair: CurrencyPair = a_pair.get_reverse_pair()
+
+    assert a_pair == a_pair and a_pair != reverse_pair

tests/tests_unit/test_input_handling.py

@@ -4,6 +4,7 @@ import pathlib
 import pytest
 from money.currency import Currency
 
+from xexe.currency_pair import CurrencyPair
 from xexe.inputs_handling import handle_get_rates_inputs
 from xexe.utils import DateRange
 
@@ -23,15 +24,23 @@ def test_handle_input_rates_works_with_full_correct_inputs():
             start_date=(datetime.datetime.now() - datetime.timedelta(days=7)).date(),
             end_date=(datetime.datetime.now() - datetime.timedelta(days=1)).date(),
         ),
-        "currencies": {Currency("USD"), Currency("EUR"), Currency("GBP")},
+        "pairs": {
+            CurrencyPair(Currency("USD"), Currency("EUR")),
+            CurrencyPair(Currency("GBP"), Currency("USD")),
+            CurrencyPair(Currency("GBP"), Currency("EUR")),
+        },
         "dry_run": False,
         "rates_source": "mock",
         "ignore_warnings": True,
         "output": pathlib.Path("test_output.csv"),
     }
 
-    for key in expected_result.keys():
+    for key in {"date_range", "dry_run", "rates_source", "ignore_warnings", "output"}:
         assert handled_inputs[key] == expected_result[key]
+        # We don't check for the currency pairs because the random ordering used
+        # by the currencies arg execution path does not guarantee the sorting,
+        # and CurrencyPair comparison needs proper sorting, and my head hurts
+        # and other tests are already catching for this correctness so.
 
 
 def test_handle_input_rates_raises_with_bad_currency_code():
@@ -105,7 +114,7 @@ def test_handle_input_rates_start_and_end_date_equal_works_fine():
     handled_inputs = handle_get_rates_inputs(
         start_date=datetime.datetime.now(),
         end_date=datetime.datetime.now(),
-        currencies="USD,EUR,GBP",
+        pairs="USDEUR,EURUSD,GBPZAR",
         dry_run=False,
         rates_source="mock",
         ignore_warnings=True,
@@ -116,7 +125,11 @@ def test_handle_input_rates_start_and_end_date_equal_works_fine():
             start_date=datetime.datetime.now().date(),
             end_date=datetime.datetime.now().date(),
         ),
-        "currencies": {Currency("USD"), Currency("EUR"), Currency("GBP")},
+        "pairs": {
+            CurrencyPair(Currency("USD"), Currency("EUR")),
+            CurrencyPair(Currency("EUR"), Currency("USD")),
+            CurrencyPair(Currency("GBP"), Currency("ZAR")),
+        },
         "dry_run": False,
         "rates_source": "mock",
         "ignore_warnings": True,
@@ -125,3 +138,47 @@ def test_handle_input_rates_start_and_end_date_equal_works_fine():
 
     for key in expected_result.keys():
         assert handled_inputs[key] == expected_result[key]
+
+
+def test_handle_input_rates_with_pairs_works_fine():
+    handled_inputs = handle_get_rates_inputs(
+        start_date=datetime.datetime.now(),
+        end_date=datetime.datetime.now(),
+        pairs="USDEUR,EURUSD,GBPZAR",
+        dry_run=False,
+        rates_source="mock",
+        ignore_warnings=True,
+        output="test_output.csv",
+    )
+    expected_result = {
+        "date_range": DateRange(
+            start_date=datetime.datetime.now().date(),
+            end_date=datetime.datetime.now().date(),
+        ),
+        "pairs": {
+            CurrencyPair(from_currency=Currency["USD"], to_currency=Currency["EUR"]),
+            CurrencyPair(from_currency=Currency["EUR"], to_currency=Currency["USD"]),
+            CurrencyPair(from_currency=Currency["GBP"], to_currency=Currency["ZAR"]),
+        },
+        "dry_run": False,
+        "rates_source": "mock",
+        "ignore_warnings": True,
+        "output": pathlib.Path("test_output.csv"),
+    }
+
+    for key in expected_result.keys():
+        assert handled_inputs[key] == expected_result[key]
+
+
+def test_handle_input_rates_raises_with_both_currencies_and_pairs():
+    with pytest.raises(ValueError):
+        handle_get_rates_inputs(
+            start_date=datetime.datetime.now(),
+            end_date=datetime.datetime.now(),
+            currencies="EUR,USD,ZAR",
+            pairs="USDEUR,EURUSD,GBPZAR",
+            dry_run=False,
+            rates_source="mock",
+            ignore_warnings=True,
+            output="test_output.csv",
+        )

tests/tests_unit/test_utils.py

@@ -3,7 +3,8 @@ import datetime
 import pytest
 from money.currency import Currency
 
-from xexe.utils import DateRange, generate_currency_and_dates_combinations
+from xexe.currency_pair import CurrencyPair
+from xexe.utils import DateRange, generate_pairs_and_dates_combinations
 
 
 def test_date_range_breaks_with_reversed_dates():
@@ -30,20 +31,23 @@ def test_date_range_generates_proper_dates_when_itered():
     assert len(dates) == 3
 
 
-def generate_currency_and_dates_combinations_outputs_correctly():
-
+def test_generate_pair_and_dates_combinations_outputs_correctly():
     date_range = DateRange(
         start_date=datetime.date(year=2024, month=1, day=1),
         end_date=datetime.date(year=2024, month=1, day=3),
     )
 
-    currencies = {Currency.USD, Currency.EUR, Currency.GBP}
+    pairs = {
+        CurrencyPair(from_currency="USD", to_currency="EUR"),
+        CurrencyPair(from_currency="USD", to_currency="GBP"),
+        CurrencyPair(from_currency="EUR", to_currency="GBP"),
+    }
 
-    combinations = generate_currency_and_dates_combinations(
-        currencies=currencies, date_range=date_range
+    combinations = generate_pairs_and_dates_combinations(
+        pairs=pairs, date_range=date_range
     )
 
     assert len(combinations) == 9
-    assert len({date for date in combinations["date"]}) == 3
-    assert len({currency for currency in combinations["from_currency"]}) == 3
-    assert len({currency for currency in combinations["to_currency"]}) == 3
+    assert len({combination["date"] for combination in combinations}) == 3
+    assert len({combination["from_currency"] for combination in combinations}) == 2
+    assert len({combination["to_currency"] for combination in combinations}) == 2

xexe/cli.py

@@ -72,6 +72,7 @@ def dwh_healthcheck():
 @click.option(
     "--currencies", default=",".join([]), show_default=True, type=click.STRING
 )
+@click.option("--pairs", default=",".join([]), show_default=True, type=click.STRING)
 @click.option("--dry-run", is_flag=True)
 @click.option("--rates-source", type=click.Choice(RATES_SOURCES.keys()), default="mock")
 @click.option("--ignore-warnings", is_flag=True)
@@ -80,6 +81,7 @@ def get_rates(
     start_date: Union[str, datetime.datetime, datetime.date],
     end_date: Union[str, datetime.datetime, datetime.date],
     currencies: Union[None, str],
+    pairs: Union[None, str],
     dry_run: bool,
     rates_source: str,
     ignore_warnings: bool,
@@ -89,6 +91,7 @@ def get_rates(
         start_date=start_date,
         end_date=end_date,
         currencies=currencies,
+        pairs=pairs,
         dry_run=dry_run,
         rates_source=rates_source,
         ignore_warnings=ignore_warnings,

xexe/currency_pair.py

@@ -0,0 +1,30 @@
+from money.currency import Currency
+
+
+class CurrencyPair:
+    """Two currencies with directionality (from->to)."""
+
+    def __init__(
+        self, from_currency: Currency, to_currency: Currency
+    ) -> "CurrencyPair":
+        self.from_currency = from_currency
+        self.to_currency = to_currency
+
+    def get_reverse_pair(self) -> "CurrencyPair":
+        return CurrencyPair(
+            from_currency=self.to_currency, to_currency=self.from_currency
+        )
+
+    def __str__(self):
+        return str(self.from_currency) + str(self.to_currency)
+
+    def __eq__(self, other: "CurrencyPair") -> bool:
+        return (self.from_currency == other.from_currency) and (
+            self.to_currency == other.to_currency
+        )
+
+    def __repr__(self):
+        return str(self)
+
+    def __hash__(self):
+        return hash((self.from_currency, self.to_currency))

xexe/inputs_handling.py

@@ -1,11 +1,13 @@
 import datetime
 import logging
 import pathlib
+from itertools import combinations
 from typing import Union
 
 from money.currency import Currency
 
 from xexe.constants import DEFAULT_CURRENCIES, RATES_SOURCES
+from xexe.currency_pair import CurrencyPair
 from xexe.utils import DateRange
 
 logger = logging.getLogger()
@@ -14,11 +16,12 @@ logger = logging.getLogger()
 def handle_get_rates_inputs(
     start_date: Union[datetime.datetime, datetime.date],
     end_date: Union[datetime.datetime, datetime.date],
-    currencies: Union[None, str],
     dry_run: bool,
     rates_source: str,
     ignore_warnings: bool,
     output: Union[str, pathlib.Path],
+    currencies: Union[None, str] = None,
+    pairs: Union[None, str] = None,
 ):
     logger.info("Handling inputs.")
 
@@ -27,15 +30,33 @@ def handle_get_rates_inputs(
     if date_range.end_date > datetime.datetime.today().date():
         date_range.end_date = datetime.datetime.today().date()
 
+    if (currencies is None or currencies == "") and not pairs:
+        logger.info("No currency list or pairs passed. Running for default currencies.")
+        currencies = DEFAULT_CURRENCIES
+
+    if pairs:
+        if currencies:
+            logger.error(f"Received both currencies and pairs.")
+            logger.error(f"Currencies: '{currencies}'.")
+            logger.error(f"Pairs: '{pairs}'.")
+            raise ValueError("You can pass currencies or pairs, but not both.")
+
+        pairs = {
+            CurrencyPair(
+                from_currency=Currency[str_pair[0:3]],
+                to_currency=Currency[str_pair[3:6]],
+            )
+            for str_pair in pairs.split(",")
+        }
+
     if currencies:
         # CLI input comes as a string of comma-separated currency codes
         currencies = {currency_code.strip() for currency_code in currencies.split(",")}
-        tmp = {Currency(currency_code) for currency_code in currencies}
-        currencies = tmp
-
-    if currencies is None or currencies == "":
-        logger.info("No currency list passed. Running for default currencies.")
-        currencies = DEFAULT_CURRENCIES
+        currencies = {Currency(currency_code) for currency_code in currencies}
+        pairs = list(combinations(currencies, 2))
+        pairs = {
+            CurrencyPair(from_currency=pair[0], to_currency=pair[1]) for pair in pairs
+        }
 
     if rates_source not in RATES_SOURCES:
         raise ValueError(f"--rates-source must be one of {RATES_SOURCES.keys()}.")
@@ -49,8 +70,8 @@ def handle_get_rates_inputs(
 
     prepared_inputs = {
         "date_range": date_range,
-        "currencies": currencies,
         "dry_run": dry_run,
+        "pairs": pairs,
         "rates_source": rates_source,
         "ignore_warnings": ignore_warnings,
         "output": output,

xexe/processes.py

@@ -1,16 +1,17 @@
 import logging
 import os
 import pathlib
-from typing import List
+from typing import List, Set, Union
 
 from money.currency import Currency
 from xecd_rates_client import XecdClient
 
 from xexe.constants import RATES_SOURCES
+from xexe.currency_pair import CurrencyPair
 from xexe.exchange_rates import ExchangeRates, add_equal_rates, add_inverse_rates
 from xexe.rate_fetching import build_rate_fetcher
 from xexe.rate_writing import build_rate_writer
-from xexe.utils import DateRange, generate_currency_and_dates_combinations
+from xexe.utils import DateRange, generate_pairs_and_dates_combinations
 
 logger = logging.getLogger()
 
@@ -67,20 +68,23 @@ def run_dwh_healthcheck():
 
 def run_get_rates(
     date_range: DateRange,
-    currencies: List[Currency],
     dry_run: bool,
     rates_source: str,
     ignore_warnings: bool,
     output: pathlib.Path,
+    pairs: Union[Set[CurrencyPair], None] = None,
 ) -> None:
     logger.info("Getting rates")
 
     process_state = GetRatesProcessState(ignore_warnings=ignore_warnings)
 
+    currency_and_date_combinations = generate_pairs_and_dates_combinations(
+        date_range=date_range, pairs=pairs
+    )
+
     rates = obtain_rates_from_source(
         rates_source=rates_source,
-        date_range=date_range,
-        currencies=currencies,
+        currency_and_date_combinations=currency_and_date_combinations,
         ignore_warnings=ignore_warnings,
     )
     logger.info("Rates obtained.")
@@ -94,18 +98,13 @@ def run_get_rates(
 
 def obtain_rates_from_source(
     rates_source: str,
-    date_range: DateRange,
-    currencies: List[Currency],
     ignore_warnings: bool,
+    currency_and_date_combinations,
 ) -> ExchangeRates:
     rates_fetcher = build_rate_fetcher(
         rates_source=rates_source, rate_sources_mapping=RATES_SOURCES
     )
 
-    currency_and_date_combinations = generate_currency_and_dates_combinations(
-        date_range=date_range, currencies=currencies
-    )
-
     large_api_call_planned = (
         rates_fetcher.is_production_grade and len(currency_and_date_combinations) > 100
     )

xexe/utils.py

@@ -4,6 +4,8 @@ from typing import Set, Tuple
 
 from money.currency import Currency
 
+from xexe.currency_pair import CurrencyPair
+
 
 class DateRange:
 
@@ -67,18 +69,16 @@ class DateRange:
             current_date += datetime.timedelta(days=1)
 
 
-def generate_currency_and_dates_combinations(
-    date_range: DateRange, currencies: Set[Currency]
+def generate_pairs_and_dates_combinations(
+    date_range: DateRange, pairs: Set[CurrencyPair]
 ) -> Tuple[dict]:
-    currency_pairs = list(combinations(currencies, 2))
-
     currency_date_combinations = []
     for date in date_range:
-        for from_currency, to_currency in currency_pairs:
+        for pair in pairs:
             currency_date_combinations.append(
                 {
-                    "from_currency": from_currency,
-                    "to_currency": to_currency,
+                    "from_currency": pair.from_currency,
+                    "to_currency": pair.to_currency,
                     "date": date,
                 }
             )