counterweight/lolafect
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge remote-tracking branch 'origin/development' into feature/run-ge

Browse source
This commit is contained in:
Pablo Martin 2023-02-02 17:24:16 +01:00
commit 18c5a3dca8
5 changed files with 18 additions and 20 deletions
Download patch file Download diff file Expand all files Collapse all files

2
CHANGELOG.md
View file

@ -2,7 +2,7 @@
All notable changes to this project will be documented in this file.
## [Unreleased]
## [0.3.0] - 2023-01-27
### Added

17
README.md
View file

@ -73,8 +73,9 @@ with Flow(...) as flow:
remote_target_host="some-host-probably-mysql",
remote_target_port=12345,
)
task_result = some_task_that_needs_the_tunnel(tunnel=tunnel)
# Tunnel is now alive. tunnel.is_active == True
close_ssh_tunnel(tunnel=tunnel)
close_ssh_tunnel(tunnel=tunnel, upstream_tasks=[task_result])
```
**Connect to a MySQL instance**
@ -82,11 +83,11 @@ with Flow(...) as flow:
from lolafect.connections import connect_to_mysql, close_mysql_connection
with Flow(...) as flow:
connection = connect_to_mysql.run(
connection = connect_to_mysql(
mysql_credentials={...}, # You probably want to get this from TEST_LOLACONFIG.DW_CREDENTIALS
)
connection.cursor().execute("SELECT 1")
close_mysql_connection.run(connection=connection)
task_result = some_task_that_needs_mysql(connection=connection)
close_mysql_connection(connection=connection, upstream_tasks=[task_result])
# Want to connect through an SSH tunnel? Open the tunnel normally and then
# override the host and port when connecting to MySQL.
@ -106,17 +107,17 @@ with Flow(...) as flow:
remote_target_port=3306,
)
connection = connect_to_mysql.run(
connection = connect_to_mysql(
mysql_credentials={...}, # You probably want to get this from TEST_LOLACONFIG.DW_CREDENTIALS
overriding_host_and_port=get_local_bind_address_from_ssh_tunnel.run(
tunnel=tunnel # This will open the connection through the SSH tunnel instead of straight to MySQL
),
)
connection.cursor().execute("SELECT 1")
task_result = some_task_that_needs_mysql(connection=connection)
close_mysql_connection.run(connection=connection)
close_ssh_tunnel.run(tunnel=tunnel)
mysql_closed = close_mysql_connection(connection=connection, upstream_tasks=[task_result])
close_ssh_tunnel.run(tunnel=tunnel, upstream_tasks=[mysql_closed])
```
### Use Great Expectations

2
lolafect/__version__.py
View file

@ -1 +1 @@
__version__="dev"
__version__ = "0.3.0"

5
lolafect/connections.py
View file

@ -258,13 +258,8 @@ def connect_to_mysql(
port=mysql_port,
user=mysql_credentials["user"],
password=mysql_credentials["password"],
database="dw_xl",
)
# Customizing this attributes to retrieve them later in the flow
db_connection.raw_user = mysql_credentials["user"]
db_connection.raw_password = mysql_credentials["password"]
logger.info(
f"Connected to MySQL at {mysql_credentials['host']}:{mysql_credentials['port']}."
)

10
lolafect/defaults.py
View file

@ -1,8 +1,10 @@
DEFAULT_ENV_S3_BUCKET="pdo-prefect-flows"
DEFAULT_ENV_FILE_PATH="env/env_prd.json"
DEFAULT_ENV_S3_BUCKET = "pdo-prefect-flows"
DEFAULT_ENV_FILE_PATH = "env/env_prd.json"
DEFAULT_PATH_TO_SLACK_WEBHOOKS_FILE = "env/slack_webhooks.json"
DEFAULT_KUBERNETES_IMAGE = "373245262072.dkr.ecr.eu-central-1.amazonaws.com/pdo-data-prefect:production"
DEFAULT_KUBERNETES_LABELS = ["k8s"]
DEFAULT_KUBERNETES_IMAGE = (
"373245262072.dkr.ecr.eu-central-1.amazonaws.com/pdo-data-prefect:production"
)
DEFAULT_KUBERNETES_LABELS = ["k8s"]
DEFAULT_FLOWS_PATH_IN_BUCKET = "flows/"
DEFAULT_TRINO_HTTP_SCHEME = "https"
DEFAULT_GREAT_EXPECTATIONS_S3_BUCKET = "pdo-prod-great-expectations"