Compare commits

...
Sign in to create a new pull request.

8 commits

Author SHA1 Message Date
Pablo Martin
d7f2e59f94 Small change in docstring. 2022-08-22 13:47:18 +02:00
Pablo Martin
3c8d2281ca Typing, docstrings, refactors. 2022-08-22 13:45:46 +02:00
Pablo Martin
85c4de0d69 A few improvements in the readme 2022-08-19 17:32:19 +02:00
Pablo Martin
8e37e41d82 Added info about SSH tunneling in the readme. 2022-08-19 17:27:07 +02:00
Pablo Martin
0effe7f3c8 Updated config examples. 2022-08-19 17:25:34 +02:00
Pablo Martin
02f412a42e Updated the changelog. 2022-08-19 17:23:41 +02:00
Pablo Martin
9db02c154b SSHTunnel can now optionally take the password of the SSH key file. 2022-08-19 17:20:20 +02:00
Pablo Martin
c9f81f1c07 Put version back to latest. 2022-07-26 14:32:17 +02:00
8 changed files with 65 additions and 25 deletions

View file

@ -1,5 +1,15 @@
# Changelog
## [unreleased]
### Added
- Users can now specify a password for the SSH key used in the SSH tunnel. This is done by entering the password in
the config file, under the entry `connection_details > ssh_tunneling > ssh_private_key_password`.
### Changed
- The `ssh_tunneling` section of the config file is now optional and the application will work even if the config
file does not contain it.
## [0.2.0] - 2022-07-26
### Added

View file

@ -1 +1 @@
__version__ = "0.2.0"
__version__ = "latest"

View file

@ -5,9 +5,7 @@
"port": 3306,
"user": "your_user",
"password": "your_password",
"schema": "comprea",
"ssh_tunneling": {
"use_tunnel": false
"schema": "comprea"
},
"queries_to_measure": [
{

View file

@ -11,7 +11,8 @@
"ssh_host": "the_ssh_tunnel_host",
"ssh_username": "the_ssh_tunnel_user",
"ssh_port": 22,
"path_to_key": "G:\\path\\to\\ssh\\key.pem"
"path_to_key": "G:\\path\\to\\ssh\\key.pem",
"ssh_private_key_password": "my_keys_password"
}
},
"queries_to_measure": [

View file

@ -7,10 +7,7 @@
"password": "your_password",
"http_scheme": "https",
"catalog": "app_lm_mysql",
"schema": "comprea",
"ssh_tunneling": {
"use_tunnel": false
}
"schema": "comprea"
},
"queries_to_measure": [
{

View file

@ -21,15 +21,22 @@ def singleton(class_):
@singleton
class MySSHTunnel:
def __init__(
self, ssh_host, ssh_port, ssh_username, ssh_pkey, remote_host, remote_port
self,
ssh_host,
ssh_port,
ssh_username,
ssh_pkey,
remote_host,
remote_port,
ssh_private_key_password=None,
):
self.tunnel = SSHTunnelForwarder(
ssh_host=(ssh_host, ssh_port),
ssh_username=ssh_username,
ssh_pkey=ssh_pkey,
remote_bind_address=(remote_host, remote_port),
local_bind_address=("127.0.0.1", remote_port),
ssh_private_key_password=ssh_private_key_password,
)
def start(self):
@ -39,9 +46,9 @@ class MySSHTunnel:
self.tunnel.stop()
def get_connection(connection_config: dict) -> Union[trino.dbapi.Connection]:
def get_connection(connection_config: dict) -> Union[trino.dbapi.Connection, mysql.connector.MySQLConnection]:
"""
Pick the right way to build a connection and pass it the connection details.
Pick the right way to build a connection and connect.
:param connection_config: confi
:return:
@ -115,12 +122,13 @@ def get_connection_to_mysql(
:param connection_config: specifies host, port, etc.
:return: the connection object
"""
if connection_config["ssh_tunneling"]["use_tunnel"]:
mysql_connection_host = connection_config["host"]
if connection_config.get("ssh_tunneling", {}).get("use_tunnel", None):
open_ssh_tunnel(connection_config)
mysql_connection_host = "127.0.0.1"
if not connection_config["ssh_tunneling"]["use_tunnel"]:
mysql_connection_host = connection_config["host"]
# If we open an SSH tunnel, we reference the local bind instead of the
# actual host
connection = mysql.connector.connect(
host=mysql_connection_host,
@ -162,6 +170,10 @@ def open_ssh_tunnel(connection_config: dict) -> None:
ssh_pkey=connection_config["ssh_tunneling"]["path_to_key"],
remote_host=connection_config["host"],
remote_port=connection_config["port"],
ssh_private_key_password=connection_config["ssh_tunneling"].get(
"ssh_private_key_password",
None, # Since password is optional, we need a safe default
),
).start()
print("SSH tunnel is now open.")

View file

@ -21,7 +21,25 @@ def run_measuring_session(config: dict) -> None:
connection = get_connection(config["connection_details"])
for query_config in config["queries_to_measure"]:
measure_queries(config["queries_to_measure"], connection)
print("Finished the measuring session.")
clean_up_connection(config["connection_details"])
def measure_queries(
queries_config: dict,
connection: Union[trino.dbapi.Connection, mysql.connector.MySQLConnection],
) -> None:
"""
Measure several queries through a connection.
:param queries_config: the configuration for the queries to measure.
:param connection: the connection to the queriable server.
:return: None
"""
for query_config in queries_config:
try:
query = TestableQuery(
name=query_config["name"], query_string=query_config["query_string"]
@ -31,10 +49,6 @@ def run_measuring_session(config: dict) -> None:
print(f"""Something went wrong with query {query_config["name"]}.""")
print(f"{traceback.format_exc()}")
print("Finished the measuring session.")
clean_up_connection(config["connection_details"])
class TestableQuery:
"""

View file

@ -9,10 +9,12 @@ several queries and measure how long it takes for results to come back to your l
Drive Shared Drive replicated locally, you can do it like this:
```commandline
pip install "git+file:///g:\shared drives\data drive\90 useful\10 query_performance_gauge@master"```
pip install "file:///g:\shared drives\data drive\90 useful\10 query_performance_gauge"
```
If not, you simply need to clone the repo somewhere in your machine and replace the path in the previous command.
You will install whatever version is in the shared drive at that point. Depending on what you want to achieve, you
might want to instead make a copy of the repository in your own local machine and install from there. That way, you
won't be affected by someone making `git checkout` in the shared drive.
2. Afterwards, you need to make a config file. See below details on how to compose one.
@ -26,7 +28,7 @@ measure_query_performance --config my_config_file.json
quick and easy hack is to redirect output in Powershell to a file. You can do it like this:
```commandline
measure_query_performance --config my_config_file.json | Out-File - FilePath my_results.txt
measure_query_performance --config my_config_file.json | Out-File -FilePath my_results.txt
```
## Composing a config file
@ -41,9 +43,15 @@ A few notes:
- I advice you to make the first query a silly, fast query such as `SELECT 1` to validate your connection and
quickly confirm that everything is set up properly.
## Other features
- The connection to the database can be made through an SSH tunnel. See the examples in `config_examples` to
understand how to configure it.
## A few more details
- Queries are run sequentially, as in the second query will only start after the first query is finished.
- The script will run all queries, even if there is an exception when running one or more of them. If one query
fails, the error traceback will be printed so you can debug and the script will move on to the next query.
- For this to work, your local machine must have access and permission to the connection you are targeting, so
remember to set up VPNs and other necessary configs properly.
- A peculiarity: when using MySQL through an SSH tunnel, the port number used by the remote MySQL should be