Updated the readme
This commit is contained in:
parent
025f2b7a5c
commit
3797615b39
1 changed files with 69 additions and 1 deletions
68
README.md
68
README.md
|
|
@ -6,6 +6,12 @@ Lolafect is a collection of Python bits that help us build our Prefect flows.
|
|||
|
||||
You can find below examples of how to leverage `lolafect` in your flows.
|
||||
|
||||
**_Note: the code excerpts below are simplified for brevity and won't run
|
||||
as-is. If you want to see perfect examples, you might want to check the tests
|
||||
in this repository._**
|
||||
|
||||
### Config
|
||||
|
||||
**Let the `LolaConfig` object do the boilerplate env stuff for you**
|
||||
|
||||
```python
|
||||
|
|
@ -36,6 +42,8 @@ lolaconfig = build_lolaconfig(
|
|||
)
|
||||
```
|
||||
|
||||
### Connections
|
||||
|
||||
**Connect to a Trino server**
|
||||
|
||||
```python
|
||||
|
|
@ -52,6 +60,66 @@ with Flow(...) as flow:
|
|||
)
|
||||
```
|
||||
|
||||
**Open an SSH tunnel**
|
||||
|
||||
```python
|
||||
from lolafect.connections import open_ssh_tunnel_with_s3_pkey, close_ssh_tunnel
|
||||
|
||||
with Flow(...) as flow:
|
||||
# You probably want to fetch these args from lolaconfig.SSH_CREDENTIALS and lolaconfig.DW_CREDENTIALS
|
||||
tunnel = open_ssh_tunnel_with_s3_pkey(
|
||||
s3_bucket_name="some-bucket",
|
||||
ssh_tunnel_credentials={...},
|
||||
remote_target_host="some-host-probably-mysql",
|
||||
remote_target_port=12345,
|
||||
)
|
||||
# Tunnel is now alive. tunnel.is_active == True
|
||||
close_ssh_tunnel(tunnel=tunnel)
|
||||
```
|
||||
|
||||
**Connect to a MySQL instance**
|
||||
```python
|
||||
from lolafect.connections import connect_to_mysql, close_mysql_connection
|
||||
|
||||
with Flow(...) as flow:
|
||||
connection = connect_to_mysql.run(
|
||||
mysql_credentials={...}, # You probably want to get this from TEST_LOLACONFIG.DW_CREDENTIALS
|
||||
)
|
||||
connection.cursor().execute("SELECT 1")
|
||||
close_mysql_connection.run(connection=connection)
|
||||
|
||||
# Want to connect through an SSH tunnel? Open the tunnel normally and then
|
||||
# override the host and port when connecting to MySQL.
|
||||
|
||||
from lolafect.connections import (
|
||||
open_ssh_tunnel_with_s3_pkey,
|
||||
get_local_bind_address_from_ssh_tunnel,
|
||||
close_ssh_tunnel
|
||||
)
|
||||
|
||||
with Flow(...) as flow:
|
||||
# You probably want to fetch these args from lolaconfig.SSH_CREDENTIALS and lolaconfig.DW_CREDENTIALS
|
||||
tunnel = open_ssh_tunnel_with_s3_pkey(
|
||||
s3_bucket_name="some-bucket",
|
||||
ssh_tunnel_credentials={...},
|
||||
remote_target_host="the-mysql-host",
|
||||
remote_target_port=3306,
|
||||
)
|
||||
|
||||
connection = connect_to_mysql.run(
|
||||
mysql_credentials={...}, # You probably want to get this from TEST_LOLACONFIG.DW_CREDENTIALS
|
||||
overriding_host_and_port=get_local_bind_address_from_ssh_tunnel.run(
|
||||
tunnel=tunnel # This will open the connection through the SSH tunnel instead of straight to MySQL
|
||||
),
|
||||
)
|
||||
|
||||
connection.cursor().execute("SELECT 1")
|
||||
|
||||
close_mysql_connection.run(connection=connection)
|
||||
close_ssh_tunnel.run(tunnel=tunnel)
|
||||
```
|
||||
|
||||
### Slack
|
||||
|
||||
**Send a warning message to slack if your tasks fails**
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue