From 4b66cefe2bc244f76eebf8f5ba361ec6ca6c905c Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Thu, 29 Dec 2022 18:08:34 +0100 Subject: [PATCH 01/16] Created readme --- README.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..a2e40b0 --- /dev/null +++ b/README.md @@ -0,0 +1,3 @@ +# Lolafect + +Lolafect is a collection of Python bits that help us build our Prefect flows. \ No newline at end of file From ee31ec817c66dc1ed803205c600dc84f1a466937 Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Fri, 30 Dec 2022 10:26:28 +0100 Subject: [PATCH 02/16] Dumb skeleton --- lolafect/__init__.py | 0 lolafect/__version__.py | 1 + lolafect/slack.py | 2 ++ setup.py | 27 +++++++++++++++++++++++++++ 4 files changed, 30 insertions(+) create mode 100644 lolafect/__init__.py create mode 100644 lolafect/__version__.py create mode 100644 lolafect/slack.py create mode 100644 setup.py diff --git a/lolafect/__init__.py b/lolafect/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/lolafect/__version__.py b/lolafect/__version__.py new file mode 100644 index 0000000..7e876b1 --- /dev/null +++ b/lolafect/__version__.py @@ -0,0 +1 @@ +__version__="dev" \ No newline at end of file diff --git a/lolafect/slack.py b/lolafect/slack.py new file mode 100644 index 0000000..299adec --- /dev/null +++ b/lolafect/slack.py @@ -0,0 +1,2 @@ +def say_hi(): + return "Slack says hi" \ No newline at end of file diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..aec9d05 --- /dev/null +++ b/setup.py @@ -0,0 +1,27 @@ +import pathlib + +from setuptools import setup + +about = {} +here = pathlib.Path(__file__).absolute() +with open("lolafect/__version__.py", "r") as f: + exec(f.read(), about) + +with open("README.md", "r") as f: + readme = f.read() + +setup( + name="lolafect", + version=about["__version__"], + description="Lolafect is a collection of Python bits that help us build our Prefect flows.", + long_description=readme, + long_description_content_type="text/markdown", + author="data-team", + author_email="data@lolamarket.com", + url="https://github.com/lolamarket/data-lolafect", + packages=["lolafect"], + package_dir={"lolafect": "lolafect"}, + include_package_data=True, + python_requires=">=3.7", + install_requires=[], +) From 630701e0daa4f7c49773d4bad8abf971d6e87d8e Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Fri, 30 Dec 2022 11:13:17 +0100 Subject: [PATCH 03/16] Adding prefect --- requirements-dev.txt | 1 + setup.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 requirements-dev.txt diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..0f7c174 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1 @@ +prefect==1.2.2 \ No newline at end of file diff --git a/setup.py b/setup.py index aec9d05..0984bc5 100644 --- a/setup.py +++ b/setup.py @@ -23,5 +23,5 @@ setup( package_dir={"lolafect": "lolafect"}, include_package_data=True, python_requires=">=3.7", - install_requires=[], + install_requires=["prefect==1.2.2"], ) From 028f181ba869a9fbfd24408283288dde18d5f1da Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Fri, 30 Dec 2022 11:11:49 +0100 Subject: [PATCH 04/16] Copied over code from flow --- lolafect/slack.py | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/lolafect/slack.py b/lolafect/slack.py index 299adec..edf1dcc 100644 --- a/lolafect/slack.py +++ b/lolafect/slack.py @@ -1,2 +1,30 @@ -def say_hi(): - return "Slack says hi" \ No newline at end of file +import json + +from prefect.core import Task +from prefect.triggers import any_failed +import requests +class SendSlackMessageTask(Task): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def run(self, webhook_url:, text_to_send) -> None: + send_message_to_slack_channel(webhook_url, text_to_send) + + +def send_message_to_slack_channel(webhook_url, text_to_send): + + slack_data = {"text": text_to_send} + response = requests.post( + webhook_url, + data=json.dumps(slack_data), + headers={"Content-Type": "application/json"}, + ) + if response.status_code != 200: + raise ValueError( + "Request to slack returned an error %s, the response is:\n%s" + % (response.status_code, response.text) + ) + + +send_message_to_slack_channel_on_upstream_all_successful = SendSlackMessageTask() +send_error_warning_to_slack = SendSlackMessageTask(trigger=any_failed) From 5e13cac9d726cee7f983ff782ce80a01b29e15c6 Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Fri, 30 Dec 2022 11:35:31 +0100 Subject: [PATCH 05/16] Docstrings and typing --- lolafect/slack.py | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/lolafect/slack.py b/lolafect/slack.py index edf1dcc..71141f9 100644 --- a/lolafect/slack.py +++ b/lolafect/slack.py @@ -1,17 +1,35 @@ import json from prefect.core import Task -from prefect.triggers import any_failed import requests + + class SendSlackMessageTask(Task): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - def run(self, webhook_url:, text_to_send) -> None: + def run(self, webhook_url: str, text_to_send: str) -> None: + """ + Pass the details from the task to the actual function. + + :param webhook_url: the URL of the Slack webhook that should receive the + message. + :param text_to_send: the text to send to the Slack channel. + :return: None + """ send_message_to_slack_channel(webhook_url, text_to_send) -def send_message_to_slack_channel(webhook_url, text_to_send): +def send_message_to_slack_channel(webhook_url: str, text_to_send: str) -> None: + """ + Send an HTTP POST to a Slack webhook to deliver a message in a channel. Raise + an error if Slack does not reply with a 200 OK status. + + :param webhook_url: the URL of the Slack webhook that should receive the + message. + :param text_to_send: the text to send to the Slack channel. + :return: None + """ slack_data = {"text": text_to_send} response = requests.post( @@ -24,7 +42,3 @@ def send_message_to_slack_channel(webhook_url, text_to_send): "Request to slack returned an error %s, the response is:\n%s" % (response.status_code, response.text) ) - - -send_message_to_slack_channel_on_upstream_all_successful = SendSlackMessageTask() -send_error_warning_to_slack = SendSlackMessageTask(trigger=any_failed) From 861a167336cc4ab9a8f26dd7242d6d2fd763720d Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Fri, 30 Dec 2022 11:36:24 +0100 Subject: [PATCH 06/16] Added requests dependency --- requirements-dev.txt | 3 ++- setup.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 0f7c174..a4f2f46 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1 +1,2 @@ -prefect==1.2.2 \ No newline at end of file +prefect==1.2.2 +requests==2.28.1 \ No newline at end of file diff --git a/setup.py b/setup.py index 0984bc5..d1fb455 100644 --- a/setup.py +++ b/setup.py @@ -23,5 +23,5 @@ setup( package_dir={"lolafect": "lolafect"}, include_package_data=True, python_requires=">=3.7", - install_requires=["prefect==1.2.2"], + install_requires=["prefect==1.2.2", "requests==2.28.1"], ) From e81707f177c5fc50f07e2a3d705bb1d0bed88d8a Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Fri, 30 Dec 2022 15:25:43 +0100 Subject: [PATCH 07/16] Added boto3 dependency. --- requirements-dev.txt | 3 ++- setup.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index a4f2f46..369f5f8 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,2 +1,3 @@ prefect==1.2.2 -requests==2.28.1 \ No newline at end of file +requests==2.28.1 +boto3==1.26.40 \ No newline at end of file diff --git a/setup.py b/setup.py index d1fb455..1b6c89e 100644 --- a/setup.py +++ b/setup.py @@ -23,5 +23,5 @@ setup( package_dir={"lolafect": "lolafect"}, include_package_data=True, python_requires=">=3.7", - install_requires=["prefect==1.2.2", "requests==2.28.1"], + install_requires=["prefect==1.2.2", "requests==2.28.1", "boto3==1.26.40"], ) From f1ed3832e55dbce2deff005a692ea8e4fc7318fe Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Fri, 30 Dec 2022 15:25:56 +0100 Subject: [PATCH 08/16] LolaConfig and defaults --- lolafect/defaults.py | 5 ++ lolafect/lolaconfig.py | 107 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 112 insertions(+) create mode 100644 lolafect/defaults.py create mode 100644 lolafect/lolaconfig.py diff --git a/lolafect/defaults.py b/lolafect/defaults.py new file mode 100644 index 0000000..e8397bc --- /dev/null +++ b/lolafect/defaults.py @@ -0,0 +1,5 @@ +DEFAULT_ENV_S3_BUCKET="pdo-prefect-flows" +DEFAULT_PATH_TO_SLACK_WEBHOOKS_FILE = "env/slack_webhooks.json" +DEFAULT_KUBERNETES_IMAGE = "373245262072.dkr.ecr.eu-central-1.amazonaws.com/pdo-data-prefect:production" +DEFAULT_KUBERNETES_LABELS = ["k8s"] +DEFAULT_FLOWS_PATH_IN_BUCKET = "flows/" diff --git a/lolafect/lolaconfig.py b/lolafect/lolaconfig.py new file mode 100644 index 0000000..d0dab5a --- /dev/null +++ b/lolafect/lolaconfig.py @@ -0,0 +1,107 @@ +import json +from typing import List + +from prefect.storage import S3 +import boto3 + +from lolafect.defaults import ( + DEFAULT_ENV_S3_BUCKET, + DEFAULT_PATH_TO_SLACK_WEBHOOKS_FILE, + DEFAULT_KUBERNETES_IMAGE, + DEFAULT_KUBERNETES_LABELS, + DEFAULT_FLOWS_PATH_IN_BUCKET, +) + + +class LolaConfig: + """ + A global-ish container for configurations required in pretty much all flows. + """ + + def __init__( + self, + flow_name: str, + env_s3_bucket: str = None, + kubernetes_labels: List = None, + kubernetes_image: str = None, + slack_webhooks_file: str = None, + ): + """ + Init and set defaults where no value was passed. + + :param flow_name: the name of the flow. + :param env_s3_bucket: the name of the S3 bucket where env vars should be + searched. + :param kubernetes_labels: labels to be passed to the kubernetes agent. + :param kubernetes_image: image to use when running through the kubernetes agent. + :param slack_webhooks_file: path to the slack webhooks file within the env + bucket. + """ + self.FLOW_NAME = flow_name + self.FLOW_NAME_UDCS = flow_name.replace("-", "_ ") + self.S3_BUCKET_NAME = ( + DEFAULT_ENV_S3_BUCKET if env_s3_bucket is None else env_s3_bucket + ) + self.SLACK_WEBHOOKS_FILE = ( + DEFAULT_PATH_TO_SLACK_WEBHOOKS_FILE + if slack_webhooks_file is None + else slack_webhooks_file + ) + self.SLACK_WEBHOOKS = None + self.STORAGE = S3( + bucket=self.S3_BUCKET_NAME, + key=DEFAULT_FLOWS_PATH_IN_BUCKET + self.FLOW_NAME + ".py", + stored_as_script=True, + ) + self.KUBERNETES_LABELS = ( + DEFAULT_KUBERNETES_LABELS + if kubernetes_labels is None + else kubernetes_labels + ) + self.KUBERNETES_IMAGE = ( + DEFAULT_KUBERNETES_IMAGE if kubernetes_image is None else kubernetes_image + ) + + def fetch_slack_webhooks(self) -> None: + """ + Read the slack webhooks file from S3 and store the webhooks in memory. + + :return: None + """ + self.SLACK_WEBHOOKS = json.loads( + boto3.client("s3") + .get_object(Bucket=self.S3_BUCKET_NAME, Key=self.SLACK_WEBHOOKS_FILE)[ + "Body" + ] + .read() + .decode("utf-8") + ) + + +def build_lolaconfig( + flow_name: str, + env_s3_bucket: str = None, + kubernetes_labels: List = None, + kubernetes_image: str = None, +) -> LolaConfig: + """ + Build a LolaConfig instance from the passed params. + + :param flow_name: the name of the flow. + :param env_s3_bucket: the name of the S3 bucket where env vars should be + searched. + :param kubernetes_labels: labels to be passed to the kubernetes agent. + :param kubernetes_image: image to use when running through the kubernetes agent. + :return: a ready to use LolaConfig instance. + """ + + lolaconfig = LolaConfig( + flow_name=flow_name, + env_s3_bucket=env_s3_bucket, + kubernetes_labels=kubernetes_labels, + kubernetes_image=kubernetes_image, + ) + + lolaconfig.fetch_slack_webhooks() + + return lolaconfig From f684b2a043443c9b7496465f3c7a30a8b953741b Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Fri, 30 Dec 2022 15:47:13 +0100 Subject: [PATCH 09/16] Made S3 client injectable to ease mocking for testing --- lolafect/lolaconfig.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/lolafect/lolaconfig.py b/lolafect/lolaconfig.py index d0dab5a..519b128 100644 --- a/lolafect/lolaconfig.py +++ b/lolafect/lolaconfig.py @@ -62,17 +62,21 @@ class LolaConfig: DEFAULT_KUBERNETES_IMAGE if kubernetes_image is None else kubernetes_image ) - def fetch_slack_webhooks(self) -> None: + def fetch_slack_webhooks(self, s3_client=None) -> None: """ Read the slack webhooks file from S3 and store the webhooks in memory. + :param s3_client: a client to fetch files from S3. :return: None """ + + if s3_client is None: + s3_client = boto3.client("s3") + self.SLACK_WEBHOOKS = json.loads( - boto3.client("s3") - .get_object(Bucket=self.S3_BUCKET_NAME, Key=self.SLACK_WEBHOOKS_FILE)[ - "Body" - ] + s3_client.get_object( + Bucket=self.S3_BUCKET_NAME, Key=self.SLACK_WEBHOOKS_FILE + )["Body"] .read() .decode("utf-8") ) From 8534c727c41e1454d0bf47ab4e58ab17f2b6c4b8 Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Mon, 9 Jan 2023 13:38:49 +0100 Subject: [PATCH 10/16] Test and refactors. --- lolafect/lolaconfig.py | 20 +++++++++----------- lolafect/utils.py | 24 ++++++++++++++++++++++++ requirements-dev.txt | 3 ++- tests/test_lolaconfig.py | 35 +++++++++++++++++++++++++++++++++++ 4 files changed, 70 insertions(+), 12 deletions(-) create mode 100644 lolafect/utils.py create mode 100644 tests/test_lolaconfig.py diff --git a/lolafect/lolaconfig.py b/lolafect/lolaconfig.py index 519b128..e7eefda 100644 --- a/lolafect/lolaconfig.py +++ b/lolafect/lolaconfig.py @@ -1,4 +1,3 @@ -import json from typing import List from prefect.storage import S3 @@ -11,6 +10,7 @@ from lolafect.defaults import ( DEFAULT_KUBERNETES_LABELS, DEFAULT_FLOWS_PATH_IN_BUCKET, ) +from lolafect.utils import S3FileReader class LolaConfig: @@ -62,23 +62,21 @@ class LolaConfig: DEFAULT_KUBERNETES_IMAGE if kubernetes_image is None else kubernetes_image ) - def fetch_slack_webhooks(self, s3_client=None) -> None: + self._s3_reader = S3FileReader(s3_client=boto3.client("s3")) + + def fetch_slack_webhooks(self, s3_reader=None) -> None: """ Read the slack webhooks file from S3 and store the webhooks in memory. - :param s3_client: a client to fetch files from S3. + :param s3_reader: a client to fetch files from S3. :return: None """ - if s3_client is None: - s3_client = boto3.client("s3") + if s3_reader is None: + s3_reader = self._s3_reader - self.SLACK_WEBHOOKS = json.loads( - s3_client.get_object( - Bucket=self.S3_BUCKET_NAME, Key=self.SLACK_WEBHOOKS_FILE - )["Body"] - .read() - .decode("utf-8") + self.SLACK_WEBHOOKS = s3_reader.read_json_from_s3_file( + bucket=self.S3_BUCKET_NAME, key=self.SLACK_WEBHOOKS_FILE ) diff --git a/lolafect/utils.py b/lolafect/utils.py new file mode 100644 index 0000000..621e59d --- /dev/null +++ b/lolafect/utils.py @@ -0,0 +1,24 @@ +import json + + +class S3FileReader: + """ + An S3 client along with a few reading utils. + """ + + def __init__(self, s3_client): + self.s3_client = s3_client + + def read_json_from_s3_file(self, bucket: str, key: str) -> dict: + """ + Read a JSON file from an S3 location and return contents as a dict. + + :param bucket: the name of the bucket where the file is stored. + :param key: the path to the file within the bucket. + :return: the file contents. + """ + return json.loads( + self.s3_client.get_object(Bucket=bucket, Key=key)["Body"] + .read() + .decode("utf-8") + ) diff --git a/requirements-dev.txt b/requirements-dev.txt index 369f5f8..86204fb 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,3 +1,4 @@ prefect==1.2.2 requests==2.28.1 -boto3==1.26.40 \ No newline at end of file +boto3==1.26.40 +pytest==7.2.0 \ No newline at end of file diff --git a/tests/test_lolaconfig.py b/tests/test_lolaconfig.py new file mode 100644 index 0000000..742de77 --- /dev/null +++ b/tests/test_lolaconfig.py @@ -0,0 +1,35 @@ +from types import SimpleNamespace + +from lolafect.lolaconfig import LolaConfig + + +def test_lolaconfig_instantiates_with_defaults_properly(): + + lolaconfig = LolaConfig(flow_name="some-flow") + + +def test_lolaconfig_instantiates_without_defaults_proplery(): + + lolaconfig = LolaConfig( + flow_name="some-flow", + env_s3_bucket="bucket", + kubernetes_labels=["some_label"], + kubernetes_image="loliloli:latest", + slack_webhooks_file="the_file/is/here.json", + ) + + +def test_lolaconfig_fetches_webhooks_properly(): + + lolaconfig = LolaConfig(flow_name="some-flow") + + fake_s3_reader = SimpleNamespace() + + def mock_read_json_from_s3_file(bucket, key): + return {"a-channel-name": "a-channel-url.com"} + + fake_s3_reader.read_json_from_s3_file = mock_read_json_from_s3_file + + lolaconfig.fetch_slack_webhooks(s3_reader=fake_s3_reader) + + assert type(lolaconfig.SLACK_WEBHOOKS) is dict From 1d7423c265fd3d63f9016dca0998a9237fcc98b7 Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Mon, 9 Jan 2023 13:51:16 +0100 Subject: [PATCH 11/16] Instructions on how to run tests --- README.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index a2e40b0..5af6c0a 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,13 @@ # Lolafect -Lolafect is a collection of Python bits that help us build our Prefect flows. \ No newline at end of file +Lolafect is a collection of Python bits that help us build our Prefect flows. + + +## How to test + +IDE-agnostic: +1. Set up a virtual environment which contains both `lolafect` and the dependencies listed in `requirements-dev.txt`. +2. Run: `pytests tests` + +In Pycharm: if you configure `pytest` as the project test runner, Pycharm will most probably autodetect the test +folder and allow you to run the test suite within the IDE. \ No newline at end of file From 547cb4d059f7d124edb8de3577c5f33ca80e6825 Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Mon, 9 Jan 2023 14:28:22 +0100 Subject: [PATCH 12/16] Listed tests --- tests/test_slack.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 tests/test_slack.py diff --git a/tests/test_slack.py b/tests/test_slack.py new file mode 100644 index 0000000..f98d381 --- /dev/null +++ b/tests/test_slack.py @@ -0,0 +1,17 @@ + +def test_send_message_to_slack_channel_works_properly(): + # Build flow + + # Check that server got the thingy + pass + +def test_send_message_to_slack_channel_raises_error(): + assert 1 == 0 + + +def test_flow_with_positive_outcome(): + assert 1 == 0 + + +def test_flow_with_negative_outcome(): + assert 1 == 0 \ No newline at end of file From fd854dd8d4e75bc0828092dbda719a5dae10a85b Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Mon, 9 Jan 2023 14:53:54 +0100 Subject: [PATCH 13/16] First couple of tests. Added httpretty --- requirements-dev.txt | 3 ++- tests/test_slack.py | 39 +++++++++++++++++++++++++++++++++------ 2 files changed, 35 insertions(+), 7 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 86204fb..40f6a2a 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,4 +1,5 @@ prefect==1.2.2 requests==2.28.1 boto3==1.26.40 -pytest==7.2.0 \ No newline at end of file +pytest==7.2.0 +httpretty==1.1.4 \ No newline at end of file diff --git a/tests/test_slack.py b/tests/test_slack.py index f98d381..78d54b3 100644 --- a/tests/test_slack.py +++ b/tests/test_slack.py @@ -1,12 +1,39 @@ +import json +import httpretty +import pytest + +from lolafect.slack import send_message_to_slack_channel + + +@httpretty.activate(allow_net_connect=False, verbose=True) def test_send_message_to_slack_channel_works_properly(): - # Build flow + mock_webhook_url = "http://the-webhook-url.com" + mock_message_to_channel = "Hi there!" + httpretty.register_uri(method=httpretty.POST, uri=mock_webhook_url, status=200) - # Check that server got the thingy - pass + send_message_to_slack_channel( + webhook_url=mock_webhook_url, text_to_send=mock_message_to_channel + ) -def test_send_message_to_slack_channel_raises_error(): - assert 1 == 0 + request_observed_by_server = httpretty.last_request() + + assert (request_observed_by_server.method == "POST") and ( + json.loads(request_observed_by_server.body.decode("UTF-8")) + == {"text": mock_message_to_channel} + ) + + +@httpretty.activate(allow_net_connect=False, verbose=True) +def test_send_message_to_slack_channel_response_400_raises_exception(): + mock_webhook_url = "http://the-webhook-url.com" + mock_message_to_channel = "Hi there!" + httpretty.register_uri(method=httpretty.POST, uri=mock_webhook_url, status=400) + + with pytest.raises(ValueError): + send_message_to_slack_channel( + webhook_url=mock_webhook_url, text_to_send=mock_message_to_channel + ) def test_flow_with_positive_outcome(): @@ -14,4 +41,4 @@ def test_flow_with_positive_outcome(): def test_flow_with_negative_outcome(): - assert 1 == 0 \ No newline at end of file + assert 1 == 0 From 4eabcdf6e7d45661f7d436ece69d633d9114b85f Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Mon, 9 Jan 2023 16:03:32 +0100 Subject: [PATCH 14/16] Added tests for slack messaging --- tests/test_slack.py | 31 ++++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/tests/test_slack.py b/tests/test_slack.py index 78d54b3..ac294fd 100644 --- a/tests/test_slack.py +++ b/tests/test_slack.py @@ -3,7 +3,7 @@ import json import httpretty import pytest -from lolafect.slack import send_message_to_slack_channel +from lolafect.slack import send_message_to_slack_channel, SendSlackMessageTask @httpretty.activate(allow_net_connect=False, verbose=True) @@ -36,9 +36,30 @@ def test_send_message_to_slack_channel_response_400_raises_exception(): ) -def test_flow_with_positive_outcome(): - assert 1 == 0 +@httpretty.activate(allow_net_connect=False, verbose=True) +def test_slack_task_reaches_server_successfully(): + mock_webhook_url = "http://the-webhook-url.com" + mock_message_to_channel = "Hi there!" + httpretty.register_uri(method=httpretty.POST, uri=mock_webhook_url, status=200) + + slack_task = SendSlackMessageTask() + slack_task.run(webhook_url=mock_webhook_url, text_to_send=mock_message_to_channel) + + request_observed_by_server = httpretty.last_request() + assert (request_observed_by_server.method == "POST") and ( + json.loads(request_observed_by_server.body.decode("UTF-8")) + == {"text": mock_message_to_channel} + ) -def test_flow_with_negative_outcome(): - assert 1 == 0 +@httpretty.activate(allow_net_connect=False, verbose=True) +def test_slack_task_raises_value_error(): + mock_webhook_url = "http://the-webhook-url.com" + mock_message_to_channel = "Hi there!" + httpretty.register_uri(method=httpretty.POST, uri=mock_webhook_url, status=400) + + with pytest.raises(ValueError): + slack_task = SendSlackMessageTask() + slack_task.run( + webhook_url=mock_webhook_url, text_to_send=mock_message_to_channel + ) From 7d6fabe206de9b96f219ee1b6cd249d7b0603915 Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Mon, 9 Jan 2023 16:29:38 +0100 Subject: [PATCH 15/16] Added a couple of usage examples --- README.md | 51 +++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 49 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5af6c0a..2ec44ac 100644 --- a/README.md +++ b/README.md @@ -2,12 +2,59 @@ Lolafect is a collection of Python bits that help us build our Prefect flows. +## Quickstart + +You can find below examples of how to leverage `lolafect` in your flows. + +**Let the `LolaConfig` object do the boilerplate env stuff for you** + +```python +from lolafect.lolaconfig import build_lolaconfig + +lolaconfig = build_lolaconfig( + flow_name="some-flow", + env_s3_bucket="bucket", + kubernetes_labels=["some_label"], + kubernetes_image="the-image:latest", +) + +# Now you can access all the env stuff from here +lolaconfig.FLOW_NAME +lolaconfig.FLOW_NAME_UDCS +lolaconfig.STORAGE +lolaconfig.KUBERNETES_IMAGE +lolaconfig.KUBERNETES_LABELS +lolaconfig.SLACK_WEBHOOKS +# etc +``` + +**Send a warning message to slack if your tasks fails** + +```python +from prefect.triggers import any_failed +from lolafect.slack import SendSlackMessageTask + +send_warning_message_on_any_failure = SendSlackMessageTask(trigger=any_failed) # You can generate other tasks with +#different triggers. For example, you can send a message when all tasks fail, or all tasks succeed + +with Flow(...) as flow: + crucial_task_result = some_crucial_task() + + send_warning_message_on_any_failure( + webhook_url="the-channel-webhook", # You can probably try to fetch this from lolaconfig.SLACK_WEBHOOKS + text_to_send="Watchout, the flow failed!", + upstream_tasks=[crucial_task_result] + ) +``` ## How to test IDE-agnostic: + 1. Set up a virtual environment which contains both `lolafect` and the dependencies listed in `requirements-dev.txt`. 2. Run: `pytests tests` -In Pycharm: if you configure `pytest` as the project test runner, Pycharm will most probably autodetect the test -folder and allow you to run the test suite within the IDE. \ No newline at end of file +In Pycharm: + +- If you configure `pytest` as the project test runner, Pycharm will most probably autodetect the test + folder and allow you to run the test suite within the IDE. \ No newline at end of file From cd510c91c4ae195c4e825e83a9d84fa4f0ea5daf Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Mon, 9 Jan 2023 19:05:32 +0100 Subject: [PATCH 16/16] Bump version --- lolafect/__version__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lolafect/__version__.py b/lolafect/__version__.py index 7e876b1..d82ddf9 100644 --- a/lolafect/__version__.py +++ b/lolafect/__version__.py @@ -1 +1 @@ -__version__="dev" \ No newline at end of file +__version__="0.1.0" \ No newline at end of file