From 707ac6ad7cac3bd947d40d56681919b48094837c Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Fri, 13 Jan 2023 13:45:12 +0100 Subject: [PATCH 01/16] Add ENV_FILE_PATH and default value --- lolafect/defaults.py | 1 + lolafect/lolaconfig.py | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/lolafect/defaults.py b/lolafect/defaults.py index e8397bc..feb66e4 100644 --- a/lolafect/defaults.py +++ b/lolafect/defaults.py @@ -1,4 +1,5 @@ DEFAULT_ENV_S3_BUCKET="pdo-prefect-flows" +DEFAULT_ENV_FILE_PATH="env/env_prd.json" DEFAULT_PATH_TO_SLACK_WEBHOOKS_FILE = "env/slack_webhooks.json" DEFAULT_KUBERNETES_IMAGE = "373245262072.dkr.ecr.eu-central-1.amazonaws.com/pdo-data-prefect:production" DEFAULT_KUBERNETES_LABELS = ["k8s"] diff --git a/lolafect/lolaconfig.py b/lolafect/lolaconfig.py index e7eefda..1a71a92 100644 --- a/lolafect/lolaconfig.py +++ b/lolafect/lolaconfig.py @@ -9,6 +9,7 @@ from lolafect.defaults import ( DEFAULT_KUBERNETES_IMAGE, DEFAULT_KUBERNETES_LABELS, DEFAULT_FLOWS_PATH_IN_BUCKET, + DEFAULT_ENV_FILE_PATH ) from lolafect.utils import S3FileReader @@ -22,6 +23,7 @@ class LolaConfig: self, flow_name: str, env_s3_bucket: str = None, + env_file_path: str = None, kubernetes_labels: List = None, kubernetes_image: str = None, slack_webhooks_file: str = None, @@ -32,6 +34,7 @@ class LolaConfig: :param flow_name: the name of the flow. :param env_s3_bucket: the name of the S3 bucket where env vars should be searched. + :param env_file_path: the path to the environment file. :param kubernetes_labels: labels to be passed to the kubernetes agent. :param kubernetes_image: image to use when running through the kubernetes agent. :param slack_webhooks_file: path to the slack webhooks file within the env @@ -42,6 +45,9 @@ class LolaConfig: self.S3_BUCKET_NAME = ( DEFAULT_ENV_S3_BUCKET if env_s3_bucket is None else env_s3_bucket ) + self.ENV_FILE_PATH = ( + DEFAULT_ENV_FILE_PATH if env_file_path is None else env_file_path + ) self.SLACK_WEBHOOKS_FILE = ( DEFAULT_PATH_TO_SLACK_WEBHOOKS_FILE if slack_webhooks_file is None From 503c70fa02b445b0d56d7c354c27003cfaf24b2f Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Fri, 13 Jan 2023 14:56:36 +0100 Subject: [PATCH 02/16] Fetch trino credentials and test --- lolafect/lolaconfig.py | 26 +++++++++++++++++++++++++- tests/test_lolaconfig.py | 23 ++++++++++++++++++++--- 2 files changed, 45 insertions(+), 4 deletions(-) diff --git a/lolafect/lolaconfig.py b/lolafect/lolaconfig.py index 1a71a92..9ac66c0 100644 --- a/lolafect/lolaconfig.py +++ b/lolafect/lolaconfig.py @@ -9,7 +9,7 @@ from lolafect.defaults import ( DEFAULT_KUBERNETES_IMAGE, DEFAULT_KUBERNETES_LABELS, DEFAULT_FLOWS_PATH_IN_BUCKET, - DEFAULT_ENV_FILE_PATH + DEFAULT_ENV_FILE_PATH, ) from lolafect.utils import S3FileReader @@ -68,6 +68,8 @@ class LolaConfig: DEFAULT_KUBERNETES_IMAGE if kubernetes_image is None else kubernetes_image ) + self.TRINO_CREDENTIALS = None + self._s3_reader = S3FileReader(s3_client=boto3.client("s3")) def fetch_slack_webhooks(self, s3_reader=None) -> None: @@ -85,6 +87,28 @@ class LolaConfig: bucket=self.S3_BUCKET_NAME, key=self.SLACK_WEBHOOKS_FILE ) + def fetch_trino_credentials(self, s3_reader=None) -> None: + """ + Read the env file from S3 and store the trino credentials in memory. + + :param s3_reader: a client to fetch files from S3. + :return: None + """ + + if s3_reader is None: + s3_reader = self._s3_reader + + env_data = s3_reader.read_json_from_s3_file( + bucket=self.S3_BUCKET_NAME, key=self.ENV_FILE_PATH + ) + + self.TRINO_CREDENTIALS = { + "host": env_data["trino_host"], + "user": env_data["trino_user"], + "password": env_data["trino_pass"], + "port": env_data["trino_port"], + } + def build_lolaconfig( flow_name: str, diff --git a/tests/test_lolaconfig.py b/tests/test_lolaconfig.py index 742de77..5d7189a 100644 --- a/tests/test_lolaconfig.py +++ b/tests/test_lolaconfig.py @@ -4,12 +4,10 @@ from lolafect.lolaconfig import LolaConfig def test_lolaconfig_instantiates_with_defaults_properly(): - lolaconfig = LolaConfig(flow_name="some-flow") def test_lolaconfig_instantiates_without_defaults_proplery(): - lolaconfig = LolaConfig( flow_name="some-flow", env_s3_bucket="bucket", @@ -20,7 +18,6 @@ def test_lolaconfig_instantiates_without_defaults_proplery(): def test_lolaconfig_fetches_webhooks_properly(): - lolaconfig = LolaConfig(flow_name="some-flow") fake_s3_reader = SimpleNamespace() @@ -33,3 +30,23 @@ def test_lolaconfig_fetches_webhooks_properly(): lolaconfig.fetch_slack_webhooks(s3_reader=fake_s3_reader) assert type(lolaconfig.SLACK_WEBHOOKS) is dict + + +def test_lolaconfig_fetches_trino_creds_properly(): + lolaconfig = LolaConfig(flow_name="some-flow") + + fake_s3_reader = SimpleNamespace() + + def mock_read_json_from_s3_file(bucket, key): + return { + "trino_host": "some_host", + "trino_user": "some_user", + "trino_pass": "some_password", + "trino_port": "some_port", + } + + fake_s3_reader.read_json_from_s3_file = mock_read_json_from_s3_file + + lolaconfig.fetch_trino_credentials(s3_reader=fake_s3_reader) + + assert type(lolaconfig.TRINO_CREDENTIALS) is dict From 592dd6bb69e5faacc74b46771057a8d7ed44ac13 Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Fri, 13 Jan 2023 15:37:02 +0100 Subject: [PATCH 03/16] Fetch SSH tunnel creds and test --- lolafect/lolaconfig.py | 24 ++++++++++++++++++++++++ tests/test_lolaconfig.py | 21 +++++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/lolafect/lolaconfig.py b/lolafect/lolaconfig.py index 9ac66c0..786855a 100644 --- a/lolafect/lolaconfig.py +++ b/lolafect/lolaconfig.py @@ -69,6 +69,7 @@ class LolaConfig: ) self.TRINO_CREDENTIALS = None + self.SSH_TUNNEL_CREDENTIALS = None self._s3_reader = S3FileReader(s3_client=boto3.client("s3")) @@ -109,6 +110,29 @@ class LolaConfig: "port": env_data["trino_port"], } + def fetch_ssh_tunnel_credentials(self, s3_reader=None) -> None: + """ + Read the env file from S3 and store the SSH tunnel credentials. + + :param s3_reader: a client to fetch files from S3. + :return: None + """ + + if s3_reader is None: + s3_reader = self._s3_reader + + env_data = s3_reader.read_json_from_s3_file( + bucket=self.S3_BUCKET_NAME, key=self.ENV_FILE_PATH + ) + + self.SSH_TUNNEL_CREDENTIALS = { + "path_to_ssh_pkey": env_data["pt_ssh_pkey_path"], + "ssh_pkey_password": env_data["pt_ssh_pkey_passphrase"], + "ssh_username": env_data["pt_ssh_username"], + "ssh_port": env_data["pt_ssh_jumphost_port"], + "ssh_jumphost": env_data["pt_ssh_jumphost"], + } + def build_lolaconfig( flow_name: str, diff --git a/tests/test_lolaconfig.py b/tests/test_lolaconfig.py index 5d7189a..f42dd4b 100644 --- a/tests/test_lolaconfig.py +++ b/tests/test_lolaconfig.py @@ -50,3 +50,24 @@ def test_lolaconfig_fetches_trino_creds_properly(): lolaconfig.fetch_trino_credentials(s3_reader=fake_s3_reader) assert type(lolaconfig.TRINO_CREDENTIALS) is dict + + +def test_lolaconfig_fetches_ssh_tunnel_creds_properly(): + lolaconfig = LolaConfig(flow_name="some-flow") + + fake_s3_reader = SimpleNamespace() + + def mock_read_json_from_s3_file(bucket, key): + return { + "pt_ssh_pkey_path": "some-path", + "pt_ssh_pkey_passphrase": "some-password", + "pt_ssh_username": "some-username", + "pt_ssh_jumphost_port": "some-port", + "pt_ssh_jumphost": "some-jumphost", + } + + fake_s3_reader.read_json_from_s3_file = mock_read_json_from_s3_file + + lolaconfig.fetch_ssh_tunnel_credentials(s3_reader=fake_s3_reader) + + assert type(lolaconfig.SSH_TUNNEL_CREDENTIALS) is dict From 4f22ad017229cb3b956007ca8acc1ac45f997d19 Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Mon, 16 Jan 2023 13:29:44 +0100 Subject: [PATCH 04/16] Refactor the fetching of env data with a specific function and a decorator --- lolafect/lolaconfig.py | 48 ++++++++++++++++++++++++------------------ 1 file changed, 28 insertions(+), 20 deletions(-) diff --git a/lolafect/lolaconfig.py b/lolafect/lolaconfig.py index 786855a..a8265eb 100644 --- a/lolafect/lolaconfig.py +++ b/lolafect/lolaconfig.py @@ -14,6 +14,14 @@ from lolafect.defaults import ( from lolafect.utils import S3FileReader +def _needs_env_data(method): + def wrapper(self, *args, **kwargs): + self.fetch_env_data(kwargs.get("s3_reader", None)) + return method(self, *args, **kwargs) + + return wrapper + + class LolaConfig: """ A global-ish container for configurations required in pretty much all flows. @@ -68,6 +76,7 @@ class LolaConfig: DEFAULT_KUBERNETES_IMAGE if kubernetes_image is None else kubernetes_image ) + self.ENV_DATA = None self.TRINO_CREDENTIALS = None self.SSH_TUNNEL_CREDENTIALS = None @@ -88,6 +97,7 @@ class LolaConfig: bucket=self.S3_BUCKET_NAME, key=self.SLACK_WEBHOOKS_FILE ) + @_needs_env_data def fetch_trino_credentials(self, s3_reader=None) -> None: """ Read the env file from S3 and store the trino credentials in memory. @@ -96,20 +106,14 @@ class LolaConfig: :return: None """ - if s3_reader is None: - s3_reader = self._s3_reader - - env_data = s3_reader.read_json_from_s3_file( - bucket=self.S3_BUCKET_NAME, key=self.ENV_FILE_PATH - ) - self.TRINO_CREDENTIALS = { - "host": env_data["trino_host"], - "user": env_data["trino_user"], - "password": env_data["trino_pass"], - "port": env_data["trino_port"], + "host": self.ENV_DATA["trino_host"], + "user": self.ENV_DATA["trino_user"], + "password": self.ENV_DATA["trino_pass"], + "port": self.ENV_DATA["trino_port"], } + @_needs_env_data def fetch_ssh_tunnel_credentials(self, s3_reader=None) -> None: """ Read the env file from S3 and store the SSH tunnel credentials. @@ -118,20 +122,24 @@ class LolaConfig: :return: None """ + self.SSH_TUNNEL_CREDENTIALS = { + "path_to_ssh_pkey": self.ENV_DATA["pt_ssh_pkey_path"], + "ssh_pkey_password": self.ENV_DATA["pt_ssh_pkey_passphrase"], + "ssh_username": self.ENV_DATA["pt_ssh_username"], + "ssh_port": self.ENV_DATA["pt_ssh_jumphost_port"], + "ssh_jumphost": self.ENV_DATA["pt_ssh_jumphost"], + } + + def fetch_env_data(self, s3_reader=None): + if self.ENV_DATA is not None: + return + if s3_reader is None: s3_reader = self._s3_reader - env_data = s3_reader.read_json_from_s3_file( bucket=self.S3_BUCKET_NAME, key=self.ENV_FILE_PATH ) - - self.SSH_TUNNEL_CREDENTIALS = { - "path_to_ssh_pkey": env_data["pt_ssh_pkey_path"], - "ssh_pkey_password": env_data["pt_ssh_pkey_passphrase"], - "ssh_username": env_data["pt_ssh_username"], - "ssh_port": env_data["pt_ssh_jumphost_port"], - "ssh_jumphost": env_data["pt_ssh_jumphost"], - } + self.ENV_DATA = env_data def build_lolaconfig( From 610288aafeed44e7ed9c9028c39ef291687bf294 Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Mon, 16 Jan 2023 13:33:45 +0100 Subject: [PATCH 05/16] Some docstrings, typing and formatting. --- lolafect/lolaconfig.py | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/lolafect/lolaconfig.py b/lolafect/lolaconfig.py index a8265eb..5f45188 100644 --- a/lolafect/lolaconfig.py +++ b/lolafect/lolaconfig.py @@ -1,4 +1,4 @@ -from typing import List +from typing import List, Callable from prefect.storage import S3 import boto3 @@ -14,7 +14,15 @@ from lolafect.defaults import ( from lolafect.utils import S3FileReader -def _needs_env_data(method): +def _needs_env_data(method: Callable) -> Callable: + """ + Decorator to wrap methods from LolaConfig that depend on having the env + data file loaded in memory. + + :param method: the method that needs env data. + :return: the method, wrapped with a call to fetch the env data. + """ + def wrapper(self, *args, **kwargs): self.fetch_env_data(kwargs.get("s3_reader", None)) return method(self, *args, **kwargs) @@ -130,16 +138,23 @@ class LolaConfig: "ssh_jumphost": self.ENV_DATA["pt_ssh_jumphost"], } - def fetch_env_data(self, s3_reader=None): + def fetch_env_data(self, s3_reader=None) -> None: + """ + Read the env file from S3 with the default or a passed s3_reader and + store the contents in the object. + + :param s3_reader: an optional s3_reader to use instead of the default + one. + :return: None + """ if self.ENV_DATA is not None: return if s3_reader is None: s3_reader = self._s3_reader - env_data = s3_reader.read_json_from_s3_file( + self.ENV_DATA = s3_reader.read_json_from_s3_file( bucket=self.S3_BUCKET_NAME, key=self.ENV_FILE_PATH ) - self.ENV_DATA = env_data def build_lolaconfig( From 3590bedffd7d0787a1ddce2528187987889ade08 Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Mon, 16 Jan 2023 13:36:46 +0100 Subject: [PATCH 06/16] build_lolaconfig now also fetches credentials --- lolafect/lolaconfig.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lolafect/lolaconfig.py b/lolafect/lolaconfig.py index 5f45188..5d275f6 100644 --- a/lolafect/lolaconfig.py +++ b/lolafect/lolaconfig.py @@ -182,5 +182,7 @@ def build_lolaconfig( ) lolaconfig.fetch_slack_webhooks() + lolaconfig.fetch_trino_credentials() + lolaconfig.fetch_ssh_tunnel_credentials() return lolaconfig From 2d9300c18ea90a9628e197edfb4016819f3ef7ed Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Mon, 16 Jan 2023 13:57:34 +0100 Subject: [PATCH 07/16] Fetch DW credentials and test. --- lolafect/lolaconfig.py | 17 +++++++++++++++++ tests/test_lolaconfig.py | 20 ++++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/lolafect/lolaconfig.py b/lolafect/lolaconfig.py index 5d275f6..515d824 100644 --- a/lolafect/lolaconfig.py +++ b/lolafect/lolaconfig.py @@ -87,6 +87,7 @@ class LolaConfig: self.ENV_DATA = None self.TRINO_CREDENTIALS = None self.SSH_TUNNEL_CREDENTIALS = None + self.DW_CREDENTIALS = None self._s3_reader = S3FileReader(s3_client=boto3.client("s3")) @@ -138,6 +139,22 @@ class LolaConfig: "ssh_jumphost": self.ENV_DATA["pt_ssh_jumphost"], } + @_needs_env_data + def fetch_dw_credentials(self, s3_reader=None) -> None: + """ + Read the env file from S3 and store the DW credentials. + + :param s3_reader: a client to fetch files from S3. + :return: None + """ + + self.DW_CREDENTIALS = { + "host": self.ENV_DATA["datadw_host"], + "user": self.ENV_DATA["datadw_user"], + "password": self.ENV_DATA["datadw_pass"], + "port": self.ENV_DATA["datadw_port"], + } + def fetch_env_data(self, s3_reader=None) -> None: """ Read the env file from S3 with the default or a passed s3_reader and diff --git a/tests/test_lolaconfig.py b/tests/test_lolaconfig.py index f42dd4b..212bc69 100644 --- a/tests/test_lolaconfig.py +++ b/tests/test_lolaconfig.py @@ -71,3 +71,23 @@ def test_lolaconfig_fetches_ssh_tunnel_creds_properly(): lolaconfig.fetch_ssh_tunnel_credentials(s3_reader=fake_s3_reader) assert type(lolaconfig.SSH_TUNNEL_CREDENTIALS) is dict + + +def test_lolaconfig_fetches_dw_creds_properly(): + lolaconfig = LolaConfig(flow_name="some-flow") + + fake_s3_reader = SimpleNamespace() + + def mock_read_json_from_s3_file(bucket, key): + return { + "datadw_host": "some_host", + "datadw_user": "some_user", + "datadw_pass": "some_password", + "datadw_port": "some_port", + } + + fake_s3_reader.read_json_from_s3_file = mock_read_json_from_s3_file + + lolaconfig.fetch_dw_credentials(s3_reader=fake_s3_reader) + + assert type(lolaconfig.DW_CREDENTIALS) is dict From 9606e3e269e3b08c72a1acb3ae9f6635ab05bc99 Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Mon, 16 Jan 2023 14:03:12 +0100 Subject: [PATCH 08/16] Fetch prefect host and test --- lolafect/lolaconfig.py | 11 +++++++++++ tests/test_lolaconfig.py | 17 +++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/lolafect/lolaconfig.py b/lolafect/lolaconfig.py index 515d824..e615d66 100644 --- a/lolafect/lolaconfig.py +++ b/lolafect/lolaconfig.py @@ -155,6 +155,17 @@ class LolaConfig: "port": self.ENV_DATA["datadw_port"], } + @_needs_env_data + def fetch_prefect_host(self, s3_reader=None) -> None: + """ + Read the env file from S3 and store the prefect_host. + + :param s3_reader: a client to fetch files from S3. + :return: None + """ + + self.PREFECT_HOST = self.ENV_DATA["prefect_host"] + def fetch_env_data(self, s3_reader=None) -> None: """ Read the env file from S3 with the default or a passed s3_reader and diff --git a/tests/test_lolaconfig.py b/tests/test_lolaconfig.py index 212bc69..64b6094 100644 --- a/tests/test_lolaconfig.py +++ b/tests/test_lolaconfig.py @@ -91,3 +91,20 @@ def test_lolaconfig_fetches_dw_creds_properly(): lolaconfig.fetch_dw_credentials(s3_reader=fake_s3_reader) assert type(lolaconfig.DW_CREDENTIALS) is dict + + +def test_lolaconfig_fetches_prefect_host_properly(): + lolaconfig = LolaConfig(flow_name="some-flow") + + fake_s3_reader = SimpleNamespace() + + def mock_read_json_from_s3_file(bucket, key): + return { + "prefect_host": "some_host", + } + + fake_s3_reader.read_json_from_s3_file = mock_read_json_from_s3_file + + lolaconfig.fetch_prefect_host(s3_reader=fake_s3_reader) + + assert type(lolaconfig.PREFECT_HOST) is str From 601e3a667c3304a1a475c7ac02ea3b0a16db222c Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Mon, 16 Jan 2023 14:09:02 +0100 Subject: [PATCH 09/16] Added decorator. --- lolafect/lolaconfig.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lolafect/lolaconfig.py b/lolafect/lolaconfig.py index e615d66..039de78 100644 --- a/lolafect/lolaconfig.py +++ b/lolafect/lolaconfig.py @@ -91,6 +91,7 @@ class LolaConfig: self._s3_reader = S3FileReader(s3_client=boto3.client("s3")) + @_needs_env_data def fetch_slack_webhooks(self, s3_reader=None) -> None: """ Read the slack webhooks file from S3 and store the webhooks in memory. From 40be75e9999148d8d8410d0d1dcd115436e92061 Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Mon, 16 Jan 2023 14:09:15 +0100 Subject: [PATCH 10/16] build_lolaconfig also fetches prefect host --- lolafect/lolaconfig.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lolafect/lolaconfig.py b/lolafect/lolaconfig.py index 039de78..652729c 100644 --- a/lolafect/lolaconfig.py +++ b/lolafect/lolaconfig.py @@ -213,5 +213,6 @@ def build_lolaconfig( lolaconfig.fetch_slack_webhooks() lolaconfig.fetch_trino_credentials() lolaconfig.fetch_ssh_tunnel_credentials() + lolaconfig.fetch_slack_webhooks() return lolaconfig From 4dca4a370b18f9f9e5872115fec06470a2d544a0 Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Mon, 16 Jan 2023 14:59:03 +0100 Subject: [PATCH 11/16] build_lolaconfig also fetches dw credentials --- lolafect/lolaconfig.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lolafect/lolaconfig.py b/lolafect/lolaconfig.py index 652729c..ebb017d 100644 --- a/lolafect/lolaconfig.py +++ b/lolafect/lolaconfig.py @@ -213,6 +213,7 @@ def build_lolaconfig( lolaconfig.fetch_slack_webhooks() lolaconfig.fetch_trino_credentials() lolaconfig.fetch_ssh_tunnel_credentials() + lolaconfig.fetch_dw_credentials() lolaconfig.fetch_slack_webhooks() return lolaconfig From fae3c9f53270ba01b4c2a931c86cc9e7cb092dfe Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Mon, 16 Jan 2023 15:25:25 +0100 Subject: [PATCH 12/16] Fixed wrong call --- lolafect/lolaconfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lolafect/lolaconfig.py b/lolafect/lolaconfig.py index ebb017d..59b372b 100644 --- a/lolafect/lolaconfig.py +++ b/lolafect/lolaconfig.py @@ -214,6 +214,6 @@ def build_lolaconfig( lolaconfig.fetch_trino_credentials() lolaconfig.fetch_ssh_tunnel_credentials() lolaconfig.fetch_dw_credentials() - lolaconfig.fetch_slack_webhooks() + lolaconfig.fetch_prefect_host() return lolaconfig From fd568e8be6dd163c510f43b7e85bfe07549f2faa Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Mon, 16 Jan 2023 15:43:46 +0100 Subject: [PATCH 13/16] Fixed typo --- lolafect/lolaconfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lolafect/lolaconfig.py b/lolafect/lolaconfig.py index 59b372b..1a645c4 100644 --- a/lolafect/lolaconfig.py +++ b/lolafect/lolaconfig.py @@ -57,7 +57,7 @@ class LolaConfig: bucket. """ self.FLOW_NAME = flow_name - self.FLOW_NAME_UDCS = flow_name.replace("-", "_ ") + self.FLOW_NAME_UDCS = flow_name.replace("-", "_") self.S3_BUCKET_NAME = ( DEFAULT_ENV_S3_BUCKET if env_s3_bucket is None else env_s3_bucket ) From d550674a040e186aa9d52f69917b101a6d7422bc Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Wed, 18 Jan 2023 17:16:29 +0100 Subject: [PATCH 14/16] Updated changelog --- CHANGELOG.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..fecf8cc --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,13 @@ +# Change Log + +All notable changes to this project will be documented in this file. + +## [Unreleased] + +### Added + +- `LolaConfig` can now fetch the DW credentials from S3. +- `LolaConfig` can now fetch the SSH tunnel from S3. +- `LolaConfig` can now fetch the Trino credentials from S3. +- `LolaConfig` can now fetch the Prefect Host from S3. +- `build_lolaconfig` executes all of the above when called. From f9720efe30239e9c563403ce15bf62661a9c7d41 Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Wed, 18 Jan 2023 17:18:19 +0100 Subject: [PATCH 15/16] Added new interesting stuff to quickstart in the readme --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 2ec44ac..6785aa8 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,10 @@ lolaconfig.STORAGE lolaconfig.KUBERNETES_IMAGE lolaconfig.KUBERNETES_LABELS lolaconfig.SLACK_WEBHOOKS +lolaconfig.DW_CREDENTIALS +lolaconfig.TRINO_CREDENTIALS +lolaconfig.SSH_TUNNEL_CREDENTIALS +lolaconfig.PREFECT_HOST # etc ``` From 4b23354d446ca493c949d258cda3b6a62c02ccf9 Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Wed, 18 Jan 2023 17:20:54 +0100 Subject: [PATCH 16/16] Improved the quickstart a bit --- README.md | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 6785aa8..e7b7994 100644 --- a/README.md +++ b/README.md @@ -11,12 +11,7 @@ You can find below examples of how to leverage `lolafect` in your flows. ```python from lolafect.lolaconfig import build_lolaconfig -lolaconfig = build_lolaconfig( - flow_name="some-flow", - env_s3_bucket="bucket", - kubernetes_labels=["some_label"], - kubernetes_image="the-image:latest", -) +lolaconfig = build_lolaconfig(flow_name="some-flow") # Now you can access all the env stuff from here lolaconfig.FLOW_NAME @@ -30,6 +25,15 @@ lolaconfig.TRINO_CREDENTIALS lolaconfig.SSH_TUNNEL_CREDENTIALS lolaconfig.PREFECT_HOST # etc + +# Your flow is different from the typical one? +# You can customize the behaviour of LolaConfig +lolaconfig = build_lolaconfig( + flow_name="some-flow", + env_s3_bucket="my-odd-bucket", + kubernetes_labels=["some-unusual-label"], + kubernetes_image="the-image:not-the-production-one", +) ``` **Send a warning message to slack if your tasks fails**