Merge remote-tracking branch 'origin/development' into feature/run-ge

CHANGELOG.md

@@ -2,7 +2,7 @@
 
 All notable changes to this project will be documented in this file.
 
-## [Unreleased]
+## [0.3.0] - 2023-01-27
 
 ### Added
 

README.md

@@ -73,8 +73,9 @@ with Flow(...) as flow:
         remote_target_host="some-host-probably-mysql",
         remote_target_port=12345,
     )
+    task_result = some_task_that_needs_the_tunnel(tunnel=tunnel)
     # Tunnel is now alive. tunnel.is_active == True
-    close_ssh_tunnel(tunnel=tunnel)  
+    close_ssh_tunnel(tunnel=tunnel, upstream_tasks=[task_result])  
 ```
 
 **Connect to a MySQL instance**
@@ -82,11 +83,11 @@ with Flow(...) as flow:
 from lolafect.connections import connect_to_mysql, close_mysql_connection
 
 with Flow(...) as flow:
-    connection = connect_to_mysql.run(
+    connection = connect_to_mysql(
         mysql_credentials={...}, # You probably want to get this from TEST_LOLACONFIG.DW_CREDENTIALS
     )
-    connection.cursor().execute("SELECT 1")
+    task_result = some_task_that_needs_mysql(connection=connection)
-    close_mysql_connection.run(connection=connection)
+    close_mysql_connection(connection=connection, upstream_tasks=[task_result])
 
 # Want to connect through an SSH tunnel? Open the tunnel normally and then
 # override the host and port when connecting to MySQL.
@@ -106,17 +107,17 @@ with Flow(...) as flow:
         remote_target_port=3306,
     )
 
-    connection = connect_to_mysql.run(
+    connection = connect_to_mysql(
         mysql_credentials={...}, # You probably want to get this from TEST_LOLACONFIG.DW_CREDENTIALS
         overriding_host_and_port=get_local_bind_address_from_ssh_tunnel.run(
             tunnel=tunnel # This will open the connection through the SSH tunnel instead of straight to MySQL
         ),
     )
 
-    connection.cursor().execute("SELECT 1")
+    task_result = some_task_that_needs_mysql(connection=connection)
     
-    close_mysql_connection.run(connection=connection)
+    mysql_closed = close_mysql_connection(connection=connection, upstream_tasks=[task_result])
-    close_ssh_tunnel.run(tunnel=tunnel)
+    close_ssh_tunnel.run(tunnel=tunnel, upstream_tasks=[mysql_closed])
 ```
 
 ### Use Great Expectations

lolafect/__version__.py

@@ -1 +1 @@
-__version__="dev"
+__version__ = "0.3.0"

lolafect/connections.py

@@ -258,13 +258,8 @@ def connect_to_mysql(
         port=mysql_port,
         user=mysql_credentials["user"],
         password=mysql_credentials["password"],
-        database="dw_xl",
     )
 
-    # Customizing this attributes to retrieve them later in the flow
-    db_connection.raw_user = mysql_credentials["user"]
-    db_connection.raw_password = mysql_credentials["password"]
-
     logger.info(
         f"Connected to MySQL at {mysql_credentials['host']}:{mysql_credentials['port']}."
     )

lolafect/defaults.py

@@ -1,7 +1,9 @@
 DEFAULT_ENV_S3_BUCKET = "pdo-prefect-flows"
 DEFAULT_ENV_FILE_PATH = "env/env_prd.json"
 DEFAULT_PATH_TO_SLACK_WEBHOOKS_FILE = "env/slack_webhooks.json"
-DEFAULT_KUBERNETES_IMAGE = "373245262072.dkr.ecr.eu-central-1.amazonaws.com/pdo-data-prefect:production"
+DEFAULT_KUBERNETES_IMAGE = (
+    "373245262072.dkr.ecr.eu-central-1.amazonaws.com/pdo-data-prefect:production"
+)
 DEFAULT_KUBERNETES_LABELS = ["k8s"]
 DEFAULT_FLOWS_PATH_IN_BUCKET = "flows/"
 DEFAULT_TRINO_HTTP_SCHEME = "https"