Updated the readme
This commit is contained in:
parent
025f2b7a5c
commit
3797615b39
1 changed files with 69 additions and 1 deletions
68
README.md
68
README.md
|
|
@ -6,6 +6,12 @@ Lolafect is a collection of Python bits that help us build our Prefect flows.
|
||||||
|
|
||||||
You can find below examples of how to leverage `lolafect` in your flows.
|
You can find below examples of how to leverage `lolafect` in your flows.
|
||||||
|
|
||||||
|
**_Note: the code excerpts below are simplified for brevity and won't run
|
||||||
|
as-is. If you want to see perfect examples, you might want to check the tests
|
||||||
|
in this repository._**
|
||||||
|
|
||||||
|
### Config
|
||||||
|
|
||||||
**Let the `LolaConfig` object do the boilerplate env stuff for you**
|
**Let the `LolaConfig` object do the boilerplate env stuff for you**
|
||||||
|
|
||||||
```python
|
```python
|
||||||
|
|
@ -36,6 +42,8 @@ lolaconfig = build_lolaconfig(
|
||||||
)
|
)
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### Connections
|
||||||
|
|
||||||
**Connect to a Trino server**
|
**Connect to a Trino server**
|
||||||
|
|
||||||
```python
|
```python
|
||||||
|
|
@ -52,6 +60,66 @@ with Flow(...) as flow:
|
||||||
)
|
)
|
||||||
```
|
```
|
||||||
|
|
||||||
|
**Open an SSH tunnel**
|
||||||
|
|
||||||
|
```python
|
||||||
|
from lolafect.connections import open_ssh_tunnel_with_s3_pkey, close_ssh_tunnel
|
||||||
|
|
||||||
|
with Flow(...) as flow:
|
||||||
|
# You probably want to fetch these args from lolaconfig.SSH_CREDENTIALS and lolaconfig.DW_CREDENTIALS
|
||||||
|
tunnel = open_ssh_tunnel_with_s3_pkey(
|
||||||
|
s3_bucket_name="some-bucket",
|
||||||
|
ssh_tunnel_credentials={...},
|
||||||
|
remote_target_host="some-host-probably-mysql",
|
||||||
|
remote_target_port=12345,
|
||||||
|
)
|
||||||
|
# Tunnel is now alive. tunnel.is_active == True
|
||||||
|
close_ssh_tunnel(tunnel=tunnel)
|
||||||
|
```
|
||||||
|
|
||||||
|
**Connect to a MySQL instance**
|
||||||
|
```python
|
||||||
|
from lolafect.connections import connect_to_mysql, close_mysql_connection
|
||||||
|
|
||||||
|
with Flow(...) as flow:
|
||||||
|
connection = connect_to_mysql.run(
|
||||||
|
mysql_credentials={...}, # You probably want to get this from TEST_LOLACONFIG.DW_CREDENTIALS
|
||||||
|
)
|
||||||
|
connection.cursor().execute("SELECT 1")
|
||||||
|
close_mysql_connection.run(connection=connection)
|
||||||
|
|
||||||
|
# Want to connect through an SSH tunnel? Open the tunnel normally and then
|
||||||
|
# override the host and port when connecting to MySQL.
|
||||||
|
|
||||||
|
from lolafect.connections import (
|
||||||
|
open_ssh_tunnel_with_s3_pkey,
|
||||||
|
get_local_bind_address_from_ssh_tunnel,
|
||||||
|
close_ssh_tunnel
|
||||||
|
)
|
||||||
|
|
||||||
|
with Flow(...) as flow:
|
||||||
|
# You probably want to fetch these args from lolaconfig.SSH_CREDENTIALS and lolaconfig.DW_CREDENTIALS
|
||||||
|
tunnel = open_ssh_tunnel_with_s3_pkey(
|
||||||
|
s3_bucket_name="some-bucket",
|
||||||
|
ssh_tunnel_credentials={...},
|
||||||
|
remote_target_host="the-mysql-host",
|
||||||
|
remote_target_port=3306,
|
||||||
|
)
|
||||||
|
|
||||||
|
connection = connect_to_mysql.run(
|
||||||
|
mysql_credentials={...}, # You probably want to get this from TEST_LOLACONFIG.DW_CREDENTIALS
|
||||||
|
overriding_host_and_port=get_local_bind_address_from_ssh_tunnel.run(
|
||||||
|
tunnel=tunnel # This will open the connection through the SSH tunnel instead of straight to MySQL
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
connection.cursor().execute("SELECT 1")
|
||||||
|
|
||||||
|
close_mysql_connection.run(connection=connection)
|
||||||
|
close_ssh_tunnel.run(tunnel=tunnel)
|
||||||
|
```
|
||||||
|
|
||||||
|
### Slack
|
||||||
|
|
||||||
**Send a warning message to slack if your tasks fails**
|
**Send a warning message to slack if your tasks fails**
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue