counterweight/lolafect
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fixed the flow, now it runs.

Browse source
This commit is contained in:
Pablo Martin 2023-03-06 17:28:00 +01:00
parent 19b1e447ff
commit 0c3243c9ce
2 changed files with 7 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

7
docs/gallery/data_testing/data_testing_flow.py
View file

@ -110,7 +110,7 @@ with Flow(
# ↓↓↓ This is where we actually run the data test. The result of the test # ↓↓↓ This is where we actually run the data test. The result of the test
# gets stored in data_test_result. # gets stored in data_test_result.
data_test_result = run_data_test_on_mysql.run( data_test_result = run_data_test_on_mysql(
name=DATA_TEST_NAME, # ←←← The name we set earlier name=DATA_TEST_NAME, # ←←← The name we set earlier
# ↓↓↓ The credentials to the MySQL where the data lives. We pass the # ↓↓↓ The credentials to the MySQL where the data lives. We pass the
# ssh tunnel host and port instead of the true MySQL because we want # ssh tunnel host and port instead of the true MySQL because we want
@ -126,7 +126,7 @@ with Flow(
}, # in the queries regardless. }, # in the queries regardless.
query=DATA_TEST_QUERY, # ←←← The query we set earlier query=DATA_TEST_QUERY, # ←←← The query we set earlier
expectation_configurations=DATA_TEST_EXPECTATIONS, # ←←← Idem expectation_configurations=DATA_TEST_EXPECTATIONS, # ←←← Idem
upstream_tasks=[ssh_tunnel] # ←←← We must wait for the tunnel to be ready upstream_tasks=[ssh_tunnel], # ←←← We must wait for the tunnel to be ready
) )
# ↑↑↑ will take care of everything: connecting to S3 and DW, generate all # ↑↑↑ will take care of everything: connecting to S3 and DW, generate all
@ -141,8 +141,7 @@ with Flow(
# The following snippets are optional. You should judge if you want to do # The following snippets are optional. You should judge if you want to do
# something similar or not in your flow based on your needs. # something similar or not in your flow based on your needs.
tunnel_closed = close_ssh_tunnel(ssh_tunnel) tunnel_closed = close_ssh_tunnel(ssh_tunnel, upstream_tasks=[data_test_result])
# TODO # TODO

4
docs/gallery/data_testing/requirements.txt Normal file
View file

@ -0,0 +1,4 @@
prefect==1.2.2
great_expectations==0.15.45
SQLAlchemy==1.4.46
lolafect==0.4.0