counterweight/lolafect
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

DB now comes from config

Browse source
This commit is contained in:
Pablo Martin 2023-02-02 15:05:53 +01:00
parent 3dfac07cb5
commit 2f62b34543
3 changed files with 12 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

8
lolafect/data_testing.py
View file

@ -25,8 +25,8 @@ def run_data_test_on_mysql(
def create_in_memory_data_context(
mysql_credentials,
database_name="staging",
mysql_credentials: dict,
database_name="information_schema",
great_expectations_bucket_name="pdo-prod-great-expectations",
):
@ -34,11 +34,11 @@ def create_in_memory_data_context(
data_context = BaseDataContext(
project_config=DataContextConfig(
datasources={
"HARCODED": DatasourceConfig(#f"{mysql_credentials['host']}:{mysql_credentials['port']}": DatasourceConfig(
"HARCODED": DatasourceConfig( # f"{mysql_credentials['host']}:{mysql_credentials['port']}": DatasourceConfig(
class_name="Datasource",
execution_engine={
"class_name": "SqlAlchemyExecutionEngine",
"connection_string": f"mysql+pymysql://%s:%s@%s:%s/{database_name}"
"connection_string": f"mysql+pymysql://%s:%s@%s:%s/{mysql_credentials['db']}"
% (
mysql_credentials["user"],
urlquote(mysql_credentials["password"]),

1
lolafect/lolaconfig.py
View file

@ -157,6 +157,7 @@ class LolaConfig:
"user": self.ENV_DATA["datadw_user"],
"password": self.ENV_DATA["datadw_pass"],
"port": self.ENV_DATA["datadw_port"],
"default_db": self.ENV_DATA["datadw_default_db"]
}
@_needs_env_data

12
tests/test_integration/test_data_testing.py
View file

@ -19,6 +19,7 @@ from lolafect.connections import open_ssh_tunnel_with_s3_pkey, close_ssh_tunnel
TEST_LOLACONFIG = build_lolaconfig(flow_name="testing-suite")
def test_validation_on_mysql_succeeds():
test_query = """
@ -54,10 +55,11 @@ def test_validation_on_mysql_succeeds():
"host": ssh_tunnel.local_bind_address[0],
"port": ssh_tunnel.local_bind_address[1],
"user": TEST_LOLACONFIG.DW_CREDENTIALS["user"],
"password": TEST_LOLACONFIG.DW_CREDENTIALS["password"]
"password": TEST_LOLACONFIG.DW_CREDENTIALS["password"],
"db": TEST_LOLACONFIG.DW_CREDENTIALS["default_db"]
},
query=test_query,
expectations=test_expectations
expectations=test_expectations,
)
closed_tunnel = close_ssh_tunnel.run(ssh_tunnel)
@ -95,17 +97,17 @@ def test_validation_on_mysql_fails():
remote_target_port=TEST_LOLACONFIG.DW_CREDENTIALS["port"],
)
validation_result = run_data_test_on_mysql.run(
name="lolafect-testing-test_validation_on_mysql_fails",
mysql_credentials={
"host": ssh_tunnel.local_bind_address[0],
"port": ssh_tunnel.local_bind_address[1],
"user": TEST_LOLACONFIG.DW_CREDENTIALS["user"],
"password": TEST_LOLACONFIG.DW_CREDENTIALS["password"]
"password": TEST_LOLACONFIG.DW_CREDENTIALS["password"],
"db": TEST_LOLACONFIG.DW_CREDENTIALS["default_db"]
},
query=test_query,
expectations=test_expectations
expectations=test_expectations,
)
closed_tunnel = close_ssh_tunnel.run(ssh_tunnel)