Compare commits

..

No commits in common. "develop" and "master" have entirely different histories.

8 changed files with 25 additions and 65 deletions

View file

@ -1,15 +1,5 @@
# Changelog # Changelog
## [unreleased]
### Added
- Users can now specify a password for the SSH key used in the SSH tunnel. This is done by entering the password in
the config file, under the entry `connection_details > ssh_tunneling > ssh_private_key_password`.
### Changed
- The `ssh_tunneling` section of the config file is now optional and the application will work even if the config
file does not contain it.
## [0.2.0] - 2022-07-26 ## [0.2.0] - 2022-07-26
### Added ### Added

View file

@ -1 +1 @@
__version__ = "latest" __version__ = "0.2.0"

View file

@ -11,8 +11,7 @@
"ssh_host": "the_ssh_tunnel_host", "ssh_host": "the_ssh_tunnel_host",
"ssh_username": "the_ssh_tunnel_user", "ssh_username": "the_ssh_tunnel_user",
"ssh_port": 22, "ssh_port": 22,
"path_to_key": "G:\\path\\to\\ssh\\key.pem", "path_to_key": "G:\\path\\to\\ssh\\key.pem"
"ssh_private_key_password": "my_keys_password"
} }
}, },
"queries_to_measure": [ "queries_to_measure": [

View file

@ -5,7 +5,9 @@
"port": 3306, "port": 3306,
"user": "your_user", "user": "your_user",
"password": "your_password", "password": "your_password",
"schema": "comprea" "schema": "comprea",
"ssh_tunneling": {
"use_tunnel": false
}, },
"queries_to_measure": [ "queries_to_measure": [
{ {

View file

@ -7,7 +7,10 @@
"password": "your_password", "password": "your_password",
"http_scheme": "https", "http_scheme": "https",
"catalog": "app_lm_mysql", "catalog": "app_lm_mysql",
"schema": "comprea" "schema": "comprea",
"ssh_tunneling": {
"use_tunnel": false
}
}, },
"queries_to_measure": [ "queries_to_measure": [
{ {

View file

@ -21,22 +21,15 @@ def singleton(class_):
@singleton @singleton
class MySSHTunnel: class MySSHTunnel:
def __init__( def __init__(
self, self, ssh_host, ssh_port, ssh_username, ssh_pkey, remote_host, remote_port
ssh_host,
ssh_port,
ssh_username,
ssh_pkey,
remote_host,
remote_port,
ssh_private_key_password=None,
): ):
self.tunnel = SSHTunnelForwarder( self.tunnel = SSHTunnelForwarder(
ssh_host=(ssh_host, ssh_port), ssh_host=(ssh_host, ssh_port),
ssh_username=ssh_username, ssh_username=ssh_username,
ssh_pkey=ssh_pkey, ssh_pkey=ssh_pkey,
remote_bind_address=(remote_host, remote_port), remote_bind_address=(remote_host, remote_port),
local_bind_address=("127.0.0.1", remote_port), local_bind_address=("127.0.0.1", remote_port),
ssh_private_key_password=ssh_private_key_password,
) )
def start(self): def start(self):
@ -46,9 +39,9 @@ class MySSHTunnel:
self.tunnel.stop() self.tunnel.stop()
def get_connection(connection_config: dict) -> Union[trino.dbapi.Connection, mysql.connector.MySQLConnection]: def get_connection(connection_config: dict) -> Union[trino.dbapi.Connection]:
""" """
Pick the right way to build a connection and connect. Pick the right way to build a connection and pass it the connection details.
:param connection_config: confi :param connection_config: confi
:return: :return:
@ -122,13 +115,12 @@ def get_connection_to_mysql(
:param connection_config: specifies host, port, etc. :param connection_config: specifies host, port, etc.
:return: the connection object :return: the connection object
""" """
mysql_connection_host = connection_config["host"] if connection_config["ssh_tunneling"]["use_tunnel"]:
if connection_config.get("ssh_tunneling", {}).get("use_tunnel", None):
open_ssh_tunnel(connection_config) open_ssh_tunnel(connection_config)
mysql_connection_host = "127.0.0.1" mysql_connection_host = "127.0.0.1"
# If we open an SSH tunnel, we reference the local bind instead of the
# actual host if not connection_config["ssh_tunneling"]["use_tunnel"]:
mysql_connection_host = connection_config["host"]
connection = mysql.connector.connect( connection = mysql.connector.connect(
host=mysql_connection_host, host=mysql_connection_host,
@ -170,10 +162,6 @@ def open_ssh_tunnel(connection_config: dict) -> None:
ssh_pkey=connection_config["ssh_tunneling"]["path_to_key"], ssh_pkey=connection_config["ssh_tunneling"]["path_to_key"],
remote_host=connection_config["host"], remote_host=connection_config["host"],
remote_port=connection_config["port"], remote_port=connection_config["port"],
ssh_private_key_password=connection_config["ssh_tunneling"].get(
"ssh_private_key_password",
None, # Since password is optional, we need a safe default
),
).start() ).start()
print("SSH tunnel is now open.") print("SSH tunnel is now open.")

View file

@ -21,25 +21,7 @@ def run_measuring_session(config: dict) -> None:
connection = get_connection(config["connection_details"]) connection = get_connection(config["connection_details"])
measure_queries(config["queries_to_measure"], connection) for query_config in config["queries_to_measure"]:
print("Finished the measuring session.")
clean_up_connection(config["connection_details"])
def measure_queries(
queries_config: dict,
connection: Union[trino.dbapi.Connection, mysql.connector.MySQLConnection],
) -> None:
"""
Measure several queries through a connection.
:param queries_config: the configuration for the queries to measure.
:param connection: the connection to the queriable server.
:return: None
"""
for query_config in queries_config:
try: try:
query = TestableQuery( query = TestableQuery(
name=query_config["name"], query_string=query_config["query_string"] name=query_config["name"], query_string=query_config["query_string"]
@ -49,6 +31,10 @@ def measure_queries(
print(f"""Something went wrong with query {query_config["name"]}.""") print(f"""Something went wrong with query {query_config["name"]}.""")
print(f"{traceback.format_exc()}") print(f"{traceback.format_exc()}")
print("Finished the measuring session.")
clean_up_connection(config["connection_details"])
class TestableQuery: class TestableQuery:
""" """

View file

@ -9,12 +9,10 @@ several queries and measure how long it takes for results to come back to your l
Drive Shared Drive replicated locally, you can do it like this: Drive Shared Drive replicated locally, you can do it like this:
```commandline ```commandline
pip install "file:///g:\shared drives\data drive\90 useful\10 query_performance_gauge" pip install "git+file:///g:\shared drives\data drive\90 useful\10 query_performance_gauge@master"```
``` ```
You will install whatever version is in the shared drive at that point. Depending on what you want to achieve, you If not, you simply need to clone the repo somewhere in your machine and replace the path in the previous command.
might want to instead make a copy of the repository in your own local machine and install from there. That way, you
won't be affected by someone making `git checkout` in the shared drive.
2. Afterwards, you need to make a config file. See below details on how to compose one. 2. Afterwards, you need to make a config file. See below details on how to compose one.
@ -28,7 +26,7 @@ measure_query_performance --config my_config_file.json
quick and easy hack is to redirect output in Powershell to a file. You can do it like this: quick and easy hack is to redirect output in Powershell to a file. You can do it like this:
```commandline ```commandline
measure_query_performance --config my_config_file.json | Out-File -FilePath my_results.txt measure_query_performance --config my_config_file.json | Out-File - FilePath my_results.txt
``` ```
## Composing a config file ## Composing a config file
@ -43,15 +41,9 @@ A few notes:
- I advice you to make the first query a silly, fast query such as `SELECT 1` to validate your connection and - I advice you to make the first query a silly, fast query such as `SELECT 1` to validate your connection and
quickly confirm that everything is set up properly. quickly confirm that everything is set up properly.
## Other features
- The connection to the database can be made through an SSH tunnel. See the examples in `config_examples` to
understand how to configure it.
## A few more details ## A few more details
- Queries are run sequentially, as in the second query will only start after the first query is finished. - Queries are run sequentially, as in the second query will only start after the first query is finished.
- The script will run all queries, even if there is an exception when running one or more of them. If one query
fails, the error traceback will be printed so you can debug and the script will move on to the next query.
- For this to work, your local machine must have access and permission to the connection you are targeting, so - For this to work, your local machine must have access and permission to the connection you are targeting, so
remember to set up VPNs and other necessary configs properly. remember to set up VPNs and other necessary configs properly.
- A peculiarity: when using MySQL through an SSH tunnel, the port number used by the remote MySQL should be - A peculiarity: when using MySQL through an SSH tunnel, the port number used by the remote MySQL should be