diff --git a/CHANGELOG.md b/CHANGELOG.md index f44d7a9..008da48 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ All notable changes to this project will be documented in this file. -## [Unreleased] +## [0.3.0] - 2023-01-27 ### Added diff --git a/README.md b/README.md index 0486671..f5b5876 100644 --- a/README.md +++ b/README.md @@ -73,8 +73,9 @@ with Flow(...) as flow: remote_target_host="some-host-probably-mysql", remote_target_port=12345, ) + task_result = some_task_that_needs_the_tunnel(tunnel=tunnel) # Tunnel is now alive. tunnel.is_active == True - close_ssh_tunnel(tunnel=tunnel) + close_ssh_tunnel(tunnel=tunnel, upstream_tasks=[task_result]) ``` **Connect to a MySQL instance** @@ -82,11 +83,11 @@ with Flow(...) as flow: from lolafect.connections import connect_to_mysql, close_mysql_connection with Flow(...) as flow: - connection = connect_to_mysql.run( + connection = connect_to_mysql( mysql_credentials={...}, # You probably want to get this from TEST_LOLACONFIG.DW_CREDENTIALS ) - connection.cursor().execute("SELECT 1") - close_mysql_connection.run(connection=connection) + task_result = some_task_that_needs_mysql(connection=connection) + close_mysql_connection(connection=connection, upstream_tasks=[task_result]) # Want to connect through an SSH tunnel? Open the tunnel normally and then # override the host and port when connecting to MySQL. @@ -106,17 +107,17 @@ with Flow(...) as flow: remote_target_port=3306, ) - connection = connect_to_mysql.run( + connection = connect_to_mysql( mysql_credentials={...}, # You probably want to get this from TEST_LOLACONFIG.DW_CREDENTIALS overriding_host_and_port=get_local_bind_address_from_ssh_tunnel.run( tunnel=tunnel # This will open the connection through the SSH tunnel instead of straight to MySQL ), ) - connection.cursor().execute("SELECT 1") - - close_mysql_connection.run(connection=connection) - close_ssh_tunnel.run(tunnel=tunnel) + task_result = some_task_that_needs_mysql(connection=connection) + + mysql_closed = close_mysql_connection(connection=connection, upstream_tasks=[task_result]) + close_ssh_tunnel.run(tunnel=tunnel, upstream_tasks=[mysql_closed]) ``` ### Use Great Expectations diff --git a/lolafect/__version__.py b/lolafect/__version__.py index 7e876b1..493f741 100644 --- a/lolafect/__version__.py +++ b/lolafect/__version__.py @@ -1 +1 @@ -__version__="dev" \ No newline at end of file +__version__ = "0.3.0" diff --git a/lolafect/connections.py b/lolafect/connections.py index 223c9f4..33e2e5f 100644 --- a/lolafect/connections.py +++ b/lolafect/connections.py @@ -258,13 +258,8 @@ def connect_to_mysql( port=mysql_port, user=mysql_credentials["user"], password=mysql_credentials["password"], - database="dw_xl", ) - # Customizing this attributes to retrieve them later in the flow - db_connection.raw_user = mysql_credentials["user"] - db_connection.raw_password = mysql_credentials["password"] - logger.info( f"Connected to MySQL at {mysql_credentials['host']}:{mysql_credentials['port']}." ) diff --git a/lolafect/defaults.py b/lolafect/defaults.py index 439a075..7a793c6 100644 --- a/lolafect/defaults.py +++ b/lolafect/defaults.py @@ -1,8 +1,10 @@ -DEFAULT_ENV_S3_BUCKET="pdo-prefect-flows" -DEFAULT_ENV_FILE_PATH="env/env_prd.json" +DEFAULT_ENV_S3_BUCKET = "pdo-prefect-flows" +DEFAULT_ENV_FILE_PATH = "env/env_prd.json" DEFAULT_PATH_TO_SLACK_WEBHOOKS_FILE = "env/slack_webhooks.json" -DEFAULT_KUBERNETES_IMAGE = "373245262072.dkr.ecr.eu-central-1.amazonaws.com/pdo-data-prefect:production" -DEFAULT_KUBERNETES_LABELS = ["k8s"] +DEFAULT_KUBERNETES_IMAGE = ( + "373245262072.dkr.ecr.eu-central-1.amazonaws.com/pdo-data-prefect:production" +) +DEFAULT_KUBERNETES_LABELS = ["k8s"] DEFAULT_FLOWS_PATH_IN_BUCKET = "flows/" DEFAULT_TRINO_HTTP_SCHEME = "https" DEFAULT_GREAT_EXPECTATIONS_S3_BUCKET = "pdo-prod-great-expectations"