counterweight/lolafect
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

DB now comes from config

Browse source
This commit is contained in:
Pablo Martin 2023-02-02 15:05:53 +01:00
parent 3dfac07cb5
commit 2f62b34543
3 changed files with 12 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

8
lolafect/data_testing.py
View file

@ -25,8 +25,8 @@ def run_data_test_on_mysql(
def create_in_memory_data_context( def create_in_memory_data_context(
mysql_credentials, mysql_credentials: dict,
database_name="staging", database_name="information_schema",
great_expectations_bucket_name="pdo-prod-great-expectations", great_expectations_bucket_name="pdo-prod-great-expectations",
): ):
@ -34,11 +34,11 @@ def create_in_memory_data_context(
data_context = BaseDataContext( data_context = BaseDataContext(
project_config=DataContextConfig( project_config=DataContextConfig(
datasources={ datasources={
"HARCODED": DatasourceConfig(#f"{mysql_credentials['host']}:{mysql_credentials['port']}": DatasourceConfig( "HARCODED": DatasourceConfig( # f"{mysql_credentials['host']}:{mysql_credentials['port']}": DatasourceConfig(
class_name="Datasource", class_name="Datasource",
execution_engine={ execution_engine={
"class_name": "SqlAlchemyExecutionEngine", "class_name": "SqlAlchemyExecutionEngine",
"connection_string": f"mysql+pymysql://%s:%s@%s:%s/{database_name}" "connection_string": f"mysql+pymysql://%s:%s@%s:%s/{mysql_credentials['db']}"
% ( % (
mysql_credentials["user"], mysql_credentials["user"],
urlquote(mysql_credentials["password"]), urlquote(mysql_credentials["password"]),

1
lolafect/lolaconfig.py
View file

@ -157,6 +157,7 @@ class LolaConfig:
"user": self.ENV_DATA["datadw_user"], "user": self.ENV_DATA["datadw_user"],
"password": self.ENV_DATA["datadw_pass"], "password": self.ENV_DATA["datadw_pass"],
"port": self.ENV_DATA["datadw_port"], "port": self.ENV_DATA["datadw_port"],
"default_db": self.ENV_DATA["datadw_default_db"]
} }
@_needs_env_data @_needs_env_data

12
tests/test_integration/test_data_testing.py
View file

@ -19,6 +19,7 @@ from lolafect.connections import open_ssh_tunnel_with_s3_pkey, close_ssh_tunnel
TEST_LOLACONFIG = build_lolaconfig(flow_name="testing-suite") TEST_LOLACONFIG = build_lolaconfig(flow_name="testing-suite")
def test_validation_on_mysql_succeeds(): def test_validation_on_mysql_succeeds():
test_query = """ test_query = """
@ -54,10 +55,11 @@ def test_validation_on_mysql_succeeds():
"host": ssh_tunnel.local_bind_address[0], "host": ssh_tunnel.local_bind_address[0],
"port": ssh_tunnel.local_bind_address[1], "port": ssh_tunnel.local_bind_address[1],
"user": TEST_LOLACONFIG.DW_CREDENTIALS["user"], "user": TEST_LOLACONFIG.DW_CREDENTIALS["user"],
"password": TEST_LOLACONFIG.DW_CREDENTIALS["password"] "password": TEST_LOLACONFIG.DW_CREDENTIALS["password"],
"db": TEST_LOLACONFIG.DW_CREDENTIALS["default_db"]
}, },
query=test_query, query=test_query,
expectations=test_expectations expectations=test_expectations,
) )
closed_tunnel = close_ssh_tunnel.run(ssh_tunnel) closed_tunnel = close_ssh_tunnel.run(ssh_tunnel)
@ -95,17 +97,17 @@ def test_validation_on_mysql_fails():
remote_target_port=TEST_LOLACONFIG.DW_CREDENTIALS["port"], remote_target_port=TEST_LOLACONFIG.DW_CREDENTIALS["port"],
) )
validation_result = run_data_test_on_mysql.run( validation_result = run_data_test_on_mysql.run(
name="lolafect-testing-test_validation_on_mysql_fails", name="lolafect-testing-test_validation_on_mysql_fails",
mysql_credentials={ mysql_credentials={
"host": ssh_tunnel.local_bind_address[0], "host": ssh_tunnel.local_bind_address[0],
"port": ssh_tunnel.local_bind_address[1], "port": ssh_tunnel.local_bind_address[1],
"user": TEST_LOLACONFIG.DW_CREDENTIALS["user"], "user": TEST_LOLACONFIG.DW_CREDENTIALS["user"],
"password": TEST_LOLACONFIG.DW_CREDENTIALS["password"] "password": TEST_LOLACONFIG.DW_CREDENTIALS["password"],
"db": TEST_LOLACONFIG.DW_CREDENTIALS["default_db"]
}, },
query=test_query, query=test_query,
expectations=test_expectations expectations=test_expectations,
) )
closed_tunnel = close_ssh_tunnel.run(ssh_tunnel) closed_tunnel = close_ssh_tunnel.run(ssh_tunnel)