counterweight/lolafect
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Refactor shared elements to follow DRY

Browse source
This commit is contained in:
Pablo Martin 2023-03-29 17:17:44 +02:00
parent 885c30184f
commit c3507cccec
1 changed files with 40 additions and 45 deletions
Download patch file Download diff file Expand all files Collapse all files

85
tests/test_integration/test_data_testing.py
View file

@ -19,29 +19,44 @@ from lolafect.connections import open_ssh_tunnel_with_s3_pkey, close_ssh_tunnel
TEST_LOLACONFIG = build_lolaconfig(flow_name="testing-suite")
TEST_QUERY = """
SELECT 1 AS a_one,
"lol" AS a_string,
NULL AS a_null
"""
TEST_EXPECTATIONS_THAT_FIT_DATA = [
ExpectationConfiguration(
expectation_type="expect_column_values_to_be_between",
kwargs={"column": "a_one", "min_value": 1, "max_value": 1},
),
ExpectationConfiguration(
expectation_type="expect_column_values_to_match_like_pattern",
kwargs={"column": "a_string", "like_pattern": "%lol%"},
),
ExpectationConfiguration(
expectation_type="expect_column_values_to_be_null",
kwargs={"column": "a_null"},
),
]
TEST_EXPECTATIONS_THAT_DONT_FIT_DATA = [
ExpectationConfiguration(
expectation_type="expect_column_values_to_be_between",
kwargs={"column": "a_one", "min_value": 2, "max_value": 2},
),
ExpectationConfiguration(
expectation_type="expect_column_values_to_match_like_pattern",
kwargs={"column": "a_string", "like_pattern": "%xD%"},
),
ExpectationConfiguration(
expectation_type="expect_column_values_to_not_be_null",
kwargs={"column": "a_null"},
),
]
def test_validation_on_mysql_succeeds():
test_query = """
SELECT 1 AS a_one,
"lol" AS a_string,
NULL AS a_null
"""
test_expectations = [
ExpectationConfiguration(
expectation_type="expect_column_values_to_be_between",
kwargs={"column": "a_one", "min_value": 1, "max_value": 1},
),
ExpectationConfiguration(
expectation_type="expect_column_values_to_match_like_pattern",
kwargs={"column": "a_string", "like_pattern": "%lol%"},
),
ExpectationConfiguration(
expectation_type="expect_column_values_to_be_null",
kwargs={"column": "a_null"},
),
]
ssh_tunnel = open_ssh_tunnel_with_s3_pkey.run(
s3_bucket_name=TEST_LOLACONFIG.S3_BUCKET_NAME,
ssh_tunnel_credentials=TEST_LOLACONFIG.SSH_TUNNEL_CREDENTIALS,
@ -58,8 +73,8 @@ def test_validation_on_mysql_succeeds():
"password": TEST_LOLACONFIG.DW_CREDENTIALS["password"],
"db": TEST_LOLACONFIG.DW_CREDENTIALS["default_db"],
},
query=test_query,
expectation_configurations=test_expectations,
query=TEST_QUERY,
expectation_configurations=TEST_EXPECTATIONS_THAT_FIT_DATA,
)
closed_tunnel = close_ssh_tunnel.run(ssh_tunnel)
@ -70,26 +85,6 @@ def test_validation_on_mysql_succeeds():
def test_validation_on_mysql_fails():
test_query = """
SELECT 1 AS a_one,
"lol" AS a_string,
NULL AS a_null
"""
test_expectations = [
ExpectationConfiguration(
expectation_type="expect_column_values_to_be_between",
kwargs={"column": "a_one", "min_value": 2, "max_value": 2},
),
ExpectationConfiguration(
expectation_type="expect_column_values_to_match_like_pattern",
kwargs={"column": "a_string", "like_pattern": "%xD%"},
),
ExpectationConfiguration(
expectation_type="expect_column_values_to_not_be_null",
kwargs={"column": "a_null"},
),
]
ssh_tunnel = open_ssh_tunnel_with_s3_pkey.run(
s3_bucket_name=TEST_LOLACONFIG.S3_BUCKET_NAME,
ssh_tunnel_credentials=TEST_LOLACONFIG.SSH_TUNNEL_CREDENTIALS,
@ -106,8 +101,8 @@ def test_validation_on_mysql_fails():
"password": TEST_LOLACONFIG.DW_CREDENTIALS["password"],
"db": TEST_LOLACONFIG.DW_CREDENTIALS["default_db"],
},
query=test_query,
expectation_configurations=test_expectations,
query=TEST_QUERY,
expectation_configurations=TEST_EXPECTATIONS_THAT_DONT_FIT_DATA,
)
closed_tunnel = close_ssh_tunnel.run(ssh_tunnel)