diff --git a/CHANGELOG.MD b/CHANGELOG.MD index 6c74205..76ed3b8 100644 --- a/CHANGELOG.MD +++ b/CHANGELOG.MD @@ -1,5 +1,15 @@ # Changelog +## [unreleased] + +### Added +- Users can now specify a password for the SSH key used in the SSH tunnel. This is done by entering the password in + the config file, under the entry `connection_details > ssh_tunneling > ssh_private_key_password`. + +### Changed +- The `ssh_tunneling` section of the config file is now optional and the application will work even if the config + file does not contain it. + ## [0.2.0] - 2022-07-26 ### Added diff --git a/_version.py b/_version.py index d3ec452..ca70210 100644 --- a/_version.py +++ b/_version.py @@ -1 +1 @@ -__version__ = "0.2.0" +__version__ = "latest" diff --git a/config_examples/mysql_without_tunnel_config.json b/config_examples/mysql_config.json similarity index 86% rename from config_examples/mysql_without_tunnel_config.json rename to config_examples/mysql_config.json index a849e48..e6859df 100644 --- a/config_examples/mysql_without_tunnel_config.json +++ b/config_examples/mysql_config.json @@ -5,9 +5,7 @@ "port": 3306, "user": "your_user", "password": "your_password", - "schema": "comprea", - "ssh_tunneling": { - "use_tunnel": false + "schema": "comprea" }, "queries_to_measure": [ { diff --git a/config_examples/mysql_with_tunnel_config.json b/config_examples/mysql_with_tunnel_config.json index 63fd975..8510894 100644 --- a/config_examples/mysql_with_tunnel_config.json +++ b/config_examples/mysql_with_tunnel_config.json @@ -11,7 +11,8 @@ "ssh_host": "the_ssh_tunnel_host", "ssh_username": "the_ssh_tunnel_user", "ssh_port": 22, - "path_to_key": "G:\\path\\to\\ssh\\key.pem" + "path_to_key": "G:\\path\\to\\ssh\\key.pem", + "ssh_private_key_password": "my_keys_password" } }, "queries_to_measure": [ diff --git a/config_examples/trino_config.json b/config_examples/trino_config.json index eaac0c0..8cd8806 100644 --- a/config_examples/trino_config.json +++ b/config_examples/trino_config.json @@ -7,10 +7,7 @@ "password": "your_password", "http_scheme": "https", "catalog": "app_lm_mysql", - "schema": "comprea", - "ssh_tunneling": { - "use_tunnel": false - } + "schema": "comprea" }, "queries_to_measure": [ { diff --git a/connections.py b/connections.py index 67edb91..b05810f 100644 --- a/connections.py +++ b/connections.py @@ -21,15 +21,22 @@ def singleton(class_): @singleton class MySSHTunnel: def __init__( - self, ssh_host, ssh_port, ssh_username, ssh_pkey, remote_host, remote_port + self, + ssh_host, + ssh_port, + ssh_username, + ssh_pkey, + remote_host, + remote_port, + ssh_private_key_password=None, ): - self.tunnel = SSHTunnelForwarder( ssh_host=(ssh_host, ssh_port), ssh_username=ssh_username, ssh_pkey=ssh_pkey, remote_bind_address=(remote_host, remote_port), local_bind_address=("127.0.0.1", remote_port), + ssh_private_key_password=ssh_private_key_password, ) def start(self): @@ -39,9 +46,9 @@ class MySSHTunnel: self.tunnel.stop() -def get_connection(connection_config: dict) -> Union[trino.dbapi.Connection]: +def get_connection(connection_config: dict) -> Union[trino.dbapi.Connection, mysql.connector.MySQLConnection]: """ - Pick the right way to build a connection and pass it the connection details. + Pick the right way to build a connection and connect. :param connection_config: confi :return: @@ -115,12 +122,13 @@ def get_connection_to_mysql( :param connection_config: specifies host, port, etc. :return: the connection object """ - if connection_config["ssh_tunneling"]["use_tunnel"]: + mysql_connection_host = connection_config["host"] + + if connection_config.get("ssh_tunneling", {}).get("use_tunnel", None): open_ssh_tunnel(connection_config) mysql_connection_host = "127.0.0.1" - - if not connection_config["ssh_tunneling"]["use_tunnel"]: - mysql_connection_host = connection_config["host"] + # If we open an SSH tunnel, we reference the local bind instead of the + # actual host connection = mysql.connector.connect( host=mysql_connection_host, @@ -162,6 +170,10 @@ def open_ssh_tunnel(connection_config: dict) -> None: ssh_pkey=connection_config["ssh_tunneling"]["path_to_key"], remote_host=connection_config["host"], remote_port=connection_config["port"], + ssh_private_key_password=connection_config["ssh_tunneling"].get( + "ssh_private_key_password", + None, # Since password is optional, we need a safe default + ), ).start() print("SSH tunnel is now open.") diff --git a/query_performance_gauge.py b/query_performance_gauge.py index 5d00fdf..353fc45 100644 --- a/query_performance_gauge.py +++ b/query_performance_gauge.py @@ -21,7 +21,25 @@ def run_measuring_session(config: dict) -> None: connection = get_connection(config["connection_details"]) - for query_config in config["queries_to_measure"]: + measure_queries(config["queries_to_measure"], connection) + + print("Finished the measuring session.") + + clean_up_connection(config["connection_details"]) + + +def measure_queries( + queries_config: dict, + connection: Union[trino.dbapi.Connection, mysql.connector.MySQLConnection], +) -> None: + """ + Measure several queries through a connection. + + :param queries_config: the configuration for the queries to measure. + :param connection: the connection to the queriable server. + :return: None + """ + for query_config in queries_config: try: query = TestableQuery( name=query_config["name"], query_string=query_config["query_string"] @@ -31,10 +49,6 @@ def run_measuring_session(config: dict) -> None: print(f"""Something went wrong with query {query_config["name"]}.""") print(f"{traceback.format_exc()}") - print("Finished the measuring session.") - - clean_up_connection(config["connection_details"]) - class TestableQuery: """ diff --git a/readme.md b/readme.md index e9b5652..5522b7e 100644 --- a/readme.md +++ b/readme.md @@ -9,10 +9,12 @@ several queries and measure how long it takes for results to come back to your l Drive Shared Drive replicated locally, you can do it like this: ```commandline -pip install "git+file:///g:\shared drives\data drive\90 useful\10 query_performance_gauge@master"``` +pip install "file:///g:\shared drives\data drive\90 useful\10 query_performance_gauge" ``` -If not, you simply need to clone the repo somewhere in your machine and replace the path in the previous command. +You will install whatever version is in the shared drive at that point. Depending on what you want to achieve, you +might want to instead make a copy of the repository in your own local machine and install from there. That way, you +won't be affected by someone making `git checkout` in the shared drive. 2. Afterwards, you need to make a config file. See below details on how to compose one. @@ -26,7 +28,7 @@ measure_query_performance --config my_config_file.json quick and easy hack is to redirect output in Powershell to a file. You can do it like this: ```commandline -measure_query_performance --config my_config_file.json | Out-File - FilePath my_results.txt +measure_query_performance --config my_config_file.json | Out-File -FilePath my_results.txt ``` ## Composing a config file @@ -41,9 +43,15 @@ A few notes: - I advice you to make the first query a silly, fast query such as `SELECT 1` to validate your connection and quickly confirm that everything is set up properly. +## Other features +- The connection to the database can be made through an SSH tunnel. See the examples in `config_examples` to + understand how to configure it. + ## A few more details - Queries are run sequentially, as in the second query will only start after the first query is finished. +- The script will run all queries, even if there is an exception when running one or more of them. If one query + fails, the error traceback will be printed so you can debug and the script will move on to the next query. - For this to work, your local machine must have access and permission to the connection you are targeting, so remember to set up VPNs and other necessary configs properly. - A peculiarity: when using MySQL through an SSH tunnel, the port number used by the remote MySQL should be