From 2f62b34543914540a46b338bacb8d9865647c065 Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Thu, 2 Feb 2023 15:05:53 +0100 Subject: [PATCH] DB now comes from config --- lolafect/data_testing.py | 8 ++++---- lolafect/lolaconfig.py | 1 + tests/test_integration/test_data_testing.py | 12 +++++++----- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/lolafect/data_testing.py b/lolafect/data_testing.py index 107ceb0..4a2e1d5 100644 --- a/lolafect/data_testing.py +++ b/lolafect/data_testing.py @@ -25,8 +25,8 @@ def run_data_test_on_mysql( def create_in_memory_data_context( - mysql_credentials, - database_name="staging", + mysql_credentials: dict, + database_name="information_schema", great_expectations_bucket_name="pdo-prod-great-expectations", ): @@ -34,11 +34,11 @@ def create_in_memory_data_context( data_context = BaseDataContext( project_config=DataContextConfig( datasources={ - "HARCODED": DatasourceConfig(#f"{mysql_credentials['host']}:{mysql_credentials['port']}": DatasourceConfig( + "HARCODED": DatasourceConfig( # f"{mysql_credentials['host']}:{mysql_credentials['port']}": DatasourceConfig( class_name="Datasource", execution_engine={ "class_name": "SqlAlchemyExecutionEngine", - "connection_string": f"mysql+pymysql://%s:%s@%s:%s/{database_name}" + "connection_string": f"mysql+pymysql://%s:%s@%s:%s/{mysql_credentials['db']}" % ( mysql_credentials["user"], urlquote(mysql_credentials["password"]), diff --git a/lolafect/lolaconfig.py b/lolafect/lolaconfig.py index 3cb05fe..36fef09 100644 --- a/lolafect/lolaconfig.py +++ b/lolafect/lolaconfig.py @@ -157,6 +157,7 @@ class LolaConfig: "user": self.ENV_DATA["datadw_user"], "password": self.ENV_DATA["datadw_pass"], "port": self.ENV_DATA["datadw_port"], + "default_db": self.ENV_DATA["datadw_default_db"] } @_needs_env_data diff --git a/tests/test_integration/test_data_testing.py b/tests/test_integration/test_data_testing.py index 3e8a6f3..b8219d0 100644 --- a/tests/test_integration/test_data_testing.py +++ b/tests/test_integration/test_data_testing.py @@ -19,6 +19,7 @@ from lolafect.connections import open_ssh_tunnel_with_s3_pkey, close_ssh_tunnel TEST_LOLACONFIG = build_lolaconfig(flow_name="testing-suite") + def test_validation_on_mysql_succeeds(): test_query = """ @@ -54,10 +55,11 @@ def test_validation_on_mysql_succeeds(): "host": ssh_tunnel.local_bind_address[0], "port": ssh_tunnel.local_bind_address[1], "user": TEST_LOLACONFIG.DW_CREDENTIALS["user"], - "password": TEST_LOLACONFIG.DW_CREDENTIALS["password"] + "password": TEST_LOLACONFIG.DW_CREDENTIALS["password"], + "db": TEST_LOLACONFIG.DW_CREDENTIALS["default_db"] }, query=test_query, - expectations=test_expectations + expectations=test_expectations, ) closed_tunnel = close_ssh_tunnel.run(ssh_tunnel) @@ -95,17 +97,17 @@ def test_validation_on_mysql_fails(): remote_target_port=TEST_LOLACONFIG.DW_CREDENTIALS["port"], ) - validation_result = run_data_test_on_mysql.run( name="lolafect-testing-test_validation_on_mysql_fails", mysql_credentials={ "host": ssh_tunnel.local_bind_address[0], "port": ssh_tunnel.local_bind_address[1], "user": TEST_LOLACONFIG.DW_CREDENTIALS["user"], - "password": TEST_LOLACONFIG.DW_CREDENTIALS["password"] + "password": TEST_LOLACONFIG.DW_CREDENTIALS["password"], + "db": TEST_LOLACONFIG.DW_CREDENTIALS["default_db"] }, query=test_query, - expectations=test_expectations + expectations=test_expectations, ) closed_tunnel = close_ssh_tunnel.run(ssh_tunnel)