From 28c066ad6fdfe96068d0ae7392855b5e98433ca5 Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Fri, 27 Jan 2023 13:24:28 +0100 Subject: [PATCH 1/6] Bump version --- lolafect/__version__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lolafect/__version__.py b/lolafect/__version__.py index 7e876b1..4e3f3d3 100644 --- a/lolafect/__version__.py +++ b/lolafect/__version__.py @@ -1 +1 @@ -__version__="dev" \ No newline at end of file +__version__="0.3.0" \ No newline at end of file From b3123c24702e534c791c4088cb91fb93ea621fa0 Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Fri, 27 Jan 2023 13:24:52 +0100 Subject: [PATCH 2/6] Update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f44d7a9..008da48 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ All notable changes to this project will be documented in this file. -## [Unreleased] +## [0.3.0] - 2023-01-27 ### Added From 6c8dc4f2074c3d1f34db6a3550b03fc5adee5188 Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Fri, 27 Jan 2023 13:36:51 +0100 Subject: [PATCH 3/6] Minor improvements in readme --- README.md | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 5d4b6f4..8258e82 100644 --- a/README.md +++ b/README.md @@ -73,8 +73,9 @@ with Flow(...) as flow: remote_target_host="some-host-probably-mysql", remote_target_port=12345, ) + task_result = some_task_that_needs_the_tunnel(tunnel=tunnel) # Tunnel is now alive. tunnel.is_active == True - close_ssh_tunnel(tunnel=tunnel) + close_ssh_tunnel(tunnel=tunnel, upstream_tasks=[task_result]) ``` **Connect to a MySQL instance** @@ -82,11 +83,11 @@ with Flow(...) as flow: from lolafect.connections import connect_to_mysql, close_mysql_connection with Flow(...) as flow: - connection = connect_to_mysql.run( + connection = connect_to_mysql( mysql_credentials={...}, # You probably want to get this from TEST_LOLACONFIG.DW_CREDENTIALS ) - connection.cursor().execute("SELECT 1") - close_mysql_connection.run(connection=connection) + task_result = some_task_that_needs_mysql(connection=connection) + close_mysql_connection(connection=connection, upstream_tasks=[task_result]) # Want to connect through an SSH tunnel? Open the tunnel normally and then # override the host and port when connecting to MySQL. @@ -106,17 +107,17 @@ with Flow(...) as flow: remote_target_port=3306, ) - connection = connect_to_mysql.run( + connection = connect_to_mysql( mysql_credentials={...}, # You probably want to get this from TEST_LOLACONFIG.DW_CREDENTIALS overriding_host_and_port=get_local_bind_address_from_ssh_tunnel.run( tunnel=tunnel # This will open the connection through the SSH tunnel instead of straight to MySQL ), ) - connection.cursor().execute("SELECT 1") - - close_mysql_connection.run(connection=connection) - close_ssh_tunnel.run(tunnel=tunnel) + task_result = some_task_that_needs_mysql(connection=connection) + + mysql_closed = close_mysql_connection(connection=connection, upstream_tasks=[task_result]) + close_ssh_tunnel.run(tunnel=tunnel, upstream_tasks=[mysql_closed]) ``` ### Slack From 737bdcab7f3a151cc462c18711afab9f7cb0d57c Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Fri, 27 Jan 2023 15:21:49 +0100 Subject: [PATCH 4/6] Last minute fix, remove hardcoded database from mysql connection. --- lolafect/connections.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lolafect/connections.py b/lolafect/connections.py index 223c9f4..a333653 100644 --- a/lolafect/connections.py +++ b/lolafect/connections.py @@ -257,8 +257,7 @@ def connect_to_mysql( host=mysql_host, port=mysql_port, user=mysql_credentials["user"], - password=mysql_credentials["password"], - database="dw_xl", + password=mysql_credentials["password"] ) # Customizing this attributes to retrieve them later in the flow From d5bc0fb28b52203ea2c9648f1b6378b60c67b210 Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Fri, 27 Jan 2023 15:22:47 +0100 Subject: [PATCH 5/6] Formatting. --- lolafect/__version__.py | 2 +- lolafect/connections.py | 2 +- lolafect/defaults.py | 10 ++++++---- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/lolafect/__version__.py b/lolafect/__version__.py index 4e3f3d3..493f741 100644 --- a/lolafect/__version__.py +++ b/lolafect/__version__.py @@ -1 +1 @@ -__version__="0.3.0" \ No newline at end of file +__version__ = "0.3.0" diff --git a/lolafect/connections.py b/lolafect/connections.py index a333653..12699ca 100644 --- a/lolafect/connections.py +++ b/lolafect/connections.py @@ -257,7 +257,7 @@ def connect_to_mysql( host=mysql_host, port=mysql_port, user=mysql_credentials["user"], - password=mysql_credentials["password"] + password=mysql_credentials["password"], ) # Customizing this attributes to retrieve them later in the flow diff --git a/lolafect/defaults.py b/lolafect/defaults.py index 1d4af84..5c6c15f 100644 --- a/lolafect/defaults.py +++ b/lolafect/defaults.py @@ -1,7 +1,9 @@ -DEFAULT_ENV_S3_BUCKET="pdo-prefect-flows" -DEFAULT_ENV_FILE_PATH="env/env_prd.json" +DEFAULT_ENV_S3_BUCKET = "pdo-prefect-flows" +DEFAULT_ENV_FILE_PATH = "env/env_prd.json" DEFAULT_PATH_TO_SLACK_WEBHOOKS_FILE = "env/slack_webhooks.json" -DEFAULT_KUBERNETES_IMAGE = "373245262072.dkr.ecr.eu-central-1.amazonaws.com/pdo-data-prefect:production" -DEFAULT_KUBERNETES_LABELS = ["k8s"] +DEFAULT_KUBERNETES_IMAGE = ( + "373245262072.dkr.ecr.eu-central-1.amazonaws.com/pdo-data-prefect:production" +) +DEFAULT_KUBERNETES_LABELS = ["k8s"] DEFAULT_FLOWS_PATH_IN_BUCKET = "flows/" DEFAULT_TRINO_HTTP_SCHEME = "https" From 143158a3fbbf67e331f3afb6f3ddba3b69475611 Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Fri, 27 Jan 2023 15:29:32 +0100 Subject: [PATCH 6/6] Remove unnecessary trick for Great Expec. --- lolafect/connections.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lolafect/connections.py b/lolafect/connections.py index 12699ca..33e2e5f 100644 --- a/lolafect/connections.py +++ b/lolafect/connections.py @@ -260,10 +260,6 @@ def connect_to_mysql( password=mysql_credentials["password"], ) - # Customizing this attributes to retrieve them later in the flow - db_connection.raw_user = mysql_credentials["user"] - db_connection.raw_password = mysql_credentials["password"] - logger.info( f"Connected to MySQL at {mysql_credentials['host']}:{mysql_credentials['port']}." )