From e81707f177c5fc50f07e2a3d705bb1d0bed88d8a Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Fri, 30 Dec 2022 15:25:43 +0100 Subject: [PATCH 1/5] Added boto3 dependency. --- requirements-dev.txt | 3 ++- setup.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index a4f2f46..369f5f8 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,2 +1,3 @@ prefect==1.2.2 -requests==2.28.1 \ No newline at end of file +requests==2.28.1 +boto3==1.26.40 \ No newline at end of file diff --git a/setup.py b/setup.py index d1fb455..1b6c89e 100644 --- a/setup.py +++ b/setup.py @@ -23,5 +23,5 @@ setup( package_dir={"lolafect": "lolafect"}, include_package_data=True, python_requires=">=3.7", - install_requires=["prefect==1.2.2", "requests==2.28.1"], + install_requires=["prefect==1.2.2", "requests==2.28.1", "boto3==1.26.40"], ) From f1ed3832e55dbce2deff005a692ea8e4fc7318fe Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Fri, 30 Dec 2022 15:25:56 +0100 Subject: [PATCH 2/5] LolaConfig and defaults --- lolafect/defaults.py | 5 ++ lolafect/lolaconfig.py | 107 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 112 insertions(+) create mode 100644 lolafect/defaults.py create mode 100644 lolafect/lolaconfig.py diff --git a/lolafect/defaults.py b/lolafect/defaults.py new file mode 100644 index 0000000..e8397bc --- /dev/null +++ b/lolafect/defaults.py @@ -0,0 +1,5 @@ +DEFAULT_ENV_S3_BUCKET="pdo-prefect-flows" +DEFAULT_PATH_TO_SLACK_WEBHOOKS_FILE = "env/slack_webhooks.json" +DEFAULT_KUBERNETES_IMAGE = "373245262072.dkr.ecr.eu-central-1.amazonaws.com/pdo-data-prefect:production" +DEFAULT_KUBERNETES_LABELS = ["k8s"] +DEFAULT_FLOWS_PATH_IN_BUCKET = "flows/" diff --git a/lolafect/lolaconfig.py b/lolafect/lolaconfig.py new file mode 100644 index 0000000..d0dab5a --- /dev/null +++ b/lolafect/lolaconfig.py @@ -0,0 +1,107 @@ +import json +from typing import List + +from prefect.storage import S3 +import boto3 + +from lolafect.defaults import ( + DEFAULT_ENV_S3_BUCKET, + DEFAULT_PATH_TO_SLACK_WEBHOOKS_FILE, + DEFAULT_KUBERNETES_IMAGE, + DEFAULT_KUBERNETES_LABELS, + DEFAULT_FLOWS_PATH_IN_BUCKET, +) + + +class LolaConfig: + """ + A global-ish container for configurations required in pretty much all flows. + """ + + def __init__( + self, + flow_name: str, + env_s3_bucket: str = None, + kubernetes_labels: List = None, + kubernetes_image: str = None, + slack_webhooks_file: str = None, + ): + """ + Init and set defaults where no value was passed. + + :param flow_name: the name of the flow. + :param env_s3_bucket: the name of the S3 bucket where env vars should be + searched. + :param kubernetes_labels: labels to be passed to the kubernetes agent. + :param kubernetes_image: image to use when running through the kubernetes agent. + :param slack_webhooks_file: path to the slack webhooks file within the env + bucket. + """ + self.FLOW_NAME = flow_name + self.FLOW_NAME_UDCS = flow_name.replace("-", "_ ") + self.S3_BUCKET_NAME = ( + DEFAULT_ENV_S3_BUCKET if env_s3_bucket is None else env_s3_bucket + ) + self.SLACK_WEBHOOKS_FILE = ( + DEFAULT_PATH_TO_SLACK_WEBHOOKS_FILE + if slack_webhooks_file is None + else slack_webhooks_file + ) + self.SLACK_WEBHOOKS = None + self.STORAGE = S3( + bucket=self.S3_BUCKET_NAME, + key=DEFAULT_FLOWS_PATH_IN_BUCKET + self.FLOW_NAME + ".py", + stored_as_script=True, + ) + self.KUBERNETES_LABELS = ( + DEFAULT_KUBERNETES_LABELS + if kubernetes_labels is None + else kubernetes_labels + ) + self.KUBERNETES_IMAGE = ( + DEFAULT_KUBERNETES_IMAGE if kubernetes_image is None else kubernetes_image + ) + + def fetch_slack_webhooks(self) -> None: + """ + Read the slack webhooks file from S3 and store the webhooks in memory. + + :return: None + """ + self.SLACK_WEBHOOKS = json.loads( + boto3.client("s3") + .get_object(Bucket=self.S3_BUCKET_NAME, Key=self.SLACK_WEBHOOKS_FILE)[ + "Body" + ] + .read() + .decode("utf-8") + ) + + +def build_lolaconfig( + flow_name: str, + env_s3_bucket: str = None, + kubernetes_labels: List = None, + kubernetes_image: str = None, +) -> LolaConfig: + """ + Build a LolaConfig instance from the passed params. + + :param flow_name: the name of the flow. + :param env_s3_bucket: the name of the S3 bucket where env vars should be + searched. + :param kubernetes_labels: labels to be passed to the kubernetes agent. + :param kubernetes_image: image to use when running through the kubernetes agent. + :return: a ready to use LolaConfig instance. + """ + + lolaconfig = LolaConfig( + flow_name=flow_name, + env_s3_bucket=env_s3_bucket, + kubernetes_labels=kubernetes_labels, + kubernetes_image=kubernetes_image, + ) + + lolaconfig.fetch_slack_webhooks() + + return lolaconfig From f684b2a043443c9b7496465f3c7a30a8b953741b Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Fri, 30 Dec 2022 15:47:13 +0100 Subject: [PATCH 3/5] Made S3 client injectable to ease mocking for testing --- lolafect/lolaconfig.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/lolafect/lolaconfig.py b/lolafect/lolaconfig.py index d0dab5a..519b128 100644 --- a/lolafect/lolaconfig.py +++ b/lolafect/lolaconfig.py @@ -62,17 +62,21 @@ class LolaConfig: DEFAULT_KUBERNETES_IMAGE if kubernetes_image is None else kubernetes_image ) - def fetch_slack_webhooks(self) -> None: + def fetch_slack_webhooks(self, s3_client=None) -> None: """ Read the slack webhooks file from S3 and store the webhooks in memory. + :param s3_client: a client to fetch files from S3. :return: None """ + + if s3_client is None: + s3_client = boto3.client("s3") + self.SLACK_WEBHOOKS = json.loads( - boto3.client("s3") - .get_object(Bucket=self.S3_BUCKET_NAME, Key=self.SLACK_WEBHOOKS_FILE)[ - "Body" - ] + s3_client.get_object( + Bucket=self.S3_BUCKET_NAME, Key=self.SLACK_WEBHOOKS_FILE + )["Body"] .read() .decode("utf-8") ) From 8534c727c41e1454d0bf47ab4e58ab17f2b6c4b8 Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Mon, 9 Jan 2023 13:38:49 +0100 Subject: [PATCH 4/5] Test and refactors. --- lolafect/lolaconfig.py | 20 +++++++++----------- lolafect/utils.py | 24 ++++++++++++++++++++++++ requirements-dev.txt | 3 ++- tests/test_lolaconfig.py | 35 +++++++++++++++++++++++++++++++++++ 4 files changed, 70 insertions(+), 12 deletions(-) create mode 100644 lolafect/utils.py create mode 100644 tests/test_lolaconfig.py diff --git a/lolafect/lolaconfig.py b/lolafect/lolaconfig.py index 519b128..e7eefda 100644 --- a/lolafect/lolaconfig.py +++ b/lolafect/lolaconfig.py @@ -1,4 +1,3 @@ -import json from typing import List from prefect.storage import S3 @@ -11,6 +10,7 @@ from lolafect.defaults import ( DEFAULT_KUBERNETES_LABELS, DEFAULT_FLOWS_PATH_IN_BUCKET, ) +from lolafect.utils import S3FileReader class LolaConfig: @@ -62,23 +62,21 @@ class LolaConfig: DEFAULT_KUBERNETES_IMAGE if kubernetes_image is None else kubernetes_image ) - def fetch_slack_webhooks(self, s3_client=None) -> None: + self._s3_reader = S3FileReader(s3_client=boto3.client("s3")) + + def fetch_slack_webhooks(self, s3_reader=None) -> None: """ Read the slack webhooks file from S3 and store the webhooks in memory. - :param s3_client: a client to fetch files from S3. + :param s3_reader: a client to fetch files from S3. :return: None """ - if s3_client is None: - s3_client = boto3.client("s3") + if s3_reader is None: + s3_reader = self._s3_reader - self.SLACK_WEBHOOKS = json.loads( - s3_client.get_object( - Bucket=self.S3_BUCKET_NAME, Key=self.SLACK_WEBHOOKS_FILE - )["Body"] - .read() - .decode("utf-8") + self.SLACK_WEBHOOKS = s3_reader.read_json_from_s3_file( + bucket=self.S3_BUCKET_NAME, key=self.SLACK_WEBHOOKS_FILE ) diff --git a/lolafect/utils.py b/lolafect/utils.py new file mode 100644 index 0000000..621e59d --- /dev/null +++ b/lolafect/utils.py @@ -0,0 +1,24 @@ +import json + + +class S3FileReader: + """ + An S3 client along with a few reading utils. + """ + + def __init__(self, s3_client): + self.s3_client = s3_client + + def read_json_from_s3_file(self, bucket: str, key: str) -> dict: + """ + Read a JSON file from an S3 location and return contents as a dict. + + :param bucket: the name of the bucket where the file is stored. + :param key: the path to the file within the bucket. + :return: the file contents. + """ + return json.loads( + self.s3_client.get_object(Bucket=bucket, Key=key)["Body"] + .read() + .decode("utf-8") + ) diff --git a/requirements-dev.txt b/requirements-dev.txt index 369f5f8..86204fb 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,3 +1,4 @@ prefect==1.2.2 requests==2.28.1 -boto3==1.26.40 \ No newline at end of file +boto3==1.26.40 +pytest==7.2.0 \ No newline at end of file diff --git a/tests/test_lolaconfig.py b/tests/test_lolaconfig.py new file mode 100644 index 0000000..742de77 --- /dev/null +++ b/tests/test_lolaconfig.py @@ -0,0 +1,35 @@ +from types import SimpleNamespace + +from lolafect.lolaconfig import LolaConfig + + +def test_lolaconfig_instantiates_with_defaults_properly(): + + lolaconfig = LolaConfig(flow_name="some-flow") + + +def test_lolaconfig_instantiates_without_defaults_proplery(): + + lolaconfig = LolaConfig( + flow_name="some-flow", + env_s3_bucket="bucket", + kubernetes_labels=["some_label"], + kubernetes_image="loliloli:latest", + slack_webhooks_file="the_file/is/here.json", + ) + + +def test_lolaconfig_fetches_webhooks_properly(): + + lolaconfig = LolaConfig(flow_name="some-flow") + + fake_s3_reader = SimpleNamespace() + + def mock_read_json_from_s3_file(bucket, key): + return {"a-channel-name": "a-channel-url.com"} + + fake_s3_reader.read_json_from_s3_file = mock_read_json_from_s3_file + + lolaconfig.fetch_slack_webhooks(s3_reader=fake_s3_reader) + + assert type(lolaconfig.SLACK_WEBHOOKS) is dict From 1d7423c265fd3d63f9016dca0998a9237fcc98b7 Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Mon, 9 Jan 2023 13:51:16 +0100 Subject: [PATCH 5/5] Instructions on how to run tests --- README.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index a2e40b0..5af6c0a 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,13 @@ # Lolafect -Lolafect is a collection of Python bits that help us build our Prefect flows. \ No newline at end of file +Lolafect is a collection of Python bits that help us build our Prefect flows. + + +## How to test + +IDE-agnostic: +1. Set up a virtual environment which contains both `lolafect` and the dependencies listed in `requirements-dev.txt`. +2. Run: `pytests tests` + +In Pycharm: if you configure `pytest` as the project test runner, Pycharm will most probably autodetect the test +folder and allow you to run the test suite within the IDE. \ No newline at end of file