From 91087cff7b8d9f83d5b76db42251f688c669fe6d Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Mon, 22 Aug 2022 14:36:00 +0200 Subject: [PATCH 1/5] A session can be configured with different files for credentials and queries. --- cli.py | 11 +++++++---- utils.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 4 deletions(-) create mode 100644 utils.py diff --git a/cli.py b/cli.py index c21acfa..64addf5 100644 --- a/cli.py +++ b/cli.py @@ -3,9 +3,12 @@ import json import click from query_performance_gauge import run_measuring_session - +from utils import compose_config @click.command() -@click.option("--config", required=True, type=click.File()) -def measure_performance(config): - run_measuring_session(json.load(config)) +@click.option("--config", type=click.File()) +@click.option("--credentials", type=click.File()) +@click.option("--queries", type=click.File()) +def measure_performance(config, credentials, queries): + config = compose_config(config, credentials, queries) + run_measuring_session(config) diff --git a/utils.py b/utils.py new file mode 100644 index 0000000..89cb246 --- /dev/null +++ b/utils.py @@ -0,0 +1,29 @@ +from typing import Union +from io import TextIOWrapper +import json + + +def compose_config( + config: Union[TextIOWrapper, None], + credentials: Union[TextIOWrapper, None], + queries: Union[TextIOWrapper, None], +) -> dict: + """ + Receive the CLI arguments and compose a session config. + + :param config: file pointer to a full config file. + :param credentials: file pointer to a credentials file. + :param queries: file pointer to a queries file. + :return: a dict with the composed configuration. + """ + + if config is not None: + return json.load(config) + + if credentials is None or queries is None: + raise ValueError( + "You need to provide both --credentials and --queries arguments." + ) + + # Merge both into one dict so it follows the same structure as a full config file + return {**json.load(credentials), **json.load(queries)} From 560381f57b2852811cf2da131a4816e876c9192c Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Mon, 22 Aug 2022 15:02:09 +0200 Subject: [PATCH 2/5] Changed order to reflect importance. --- cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cli.py b/cli.py index 64addf5..1fa1ba3 100644 --- a/cli.py +++ b/cli.py @@ -6,9 +6,9 @@ from query_performance_gauge import run_measuring_session from utils import compose_config @click.command() -@click.option("--config", type=click.File()) @click.option("--credentials", type=click.File()) @click.option("--queries", type=click.File()) -def measure_performance(config, credentials, queries): +@click.option("--config", type=click.File()) +def measure_performance(credentials, queries, config): config = compose_config(config, credentials, queries) run_measuring_session(config) From c8cc49a6784073ad34a053aaf41481d19c44fe85 Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Mon, 22 Aug 2022 15:02:21 +0200 Subject: [PATCH 3/5] Small util to decide which config to use. --- utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/utils.py b/utils.py index 89cb246..9d38aa3 100644 --- a/utils.py +++ b/utils.py @@ -18,6 +18,9 @@ def compose_config( """ if config is not None: + DeprecationWarning( + "Usage of a full config file will be deprecated. Instead, use the credentials and queries arguments." + ) return json.load(config) if credentials is None or queries is None: From 10a7d379f2f86715d7c09973f8882f0e21002f17 Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Mon, 22 Aug 2022 15:05:54 +0200 Subject: [PATCH 4/5] Updated config examples. --- ...nfig.json => legacy_full_config_file.json} | 0 config_examples/mysql_credentials_config.json | 10 +++++++++ ...mysql_credentials_with_tunnel_config.json} | 12 +--------- config_examples/query_config_example.json | 12 ++++++++++ config_examples/trino_config.json | 22 ------------------- config_examples/trino_credentials_config.json | 12 ++++++++++ 6 files changed, 35 insertions(+), 33 deletions(-) rename config_examples/{mysql_config.json => legacy_full_config_file.json} (100%) create mode 100644 config_examples/mysql_credentials_config.json rename config_examples/{mysql_with_tunnel_config.json => mysql_credentials_with_tunnel_config.json} (59%) create mode 100644 config_examples/query_config_example.json delete mode 100644 config_examples/trino_config.json create mode 100644 config_examples/trino_credentials_config.json diff --git a/config_examples/mysql_config.json b/config_examples/legacy_full_config_file.json similarity index 100% rename from config_examples/mysql_config.json rename to config_examples/legacy_full_config_file.json diff --git a/config_examples/mysql_credentials_config.json b/config_examples/mysql_credentials_config.json new file mode 100644 index 0000000..31990dd --- /dev/null +++ b/config_examples/mysql_credentials_config.json @@ -0,0 +1,10 @@ +{ + "connection_details": { + "engine": "mysql", + "host": "the-sql-host", + "port": 3306, + "user": "your_user", + "password": "your_password", + "schema": "comprea" + } +} \ No newline at end of file diff --git a/config_examples/mysql_with_tunnel_config.json b/config_examples/mysql_credentials_with_tunnel_config.json similarity index 59% rename from config_examples/mysql_with_tunnel_config.json rename to config_examples/mysql_credentials_with_tunnel_config.json index 8510894..5c3a911 100644 --- a/config_examples/mysql_with_tunnel_config.json +++ b/config_examples/mysql_credentials_with_tunnel_config.json @@ -14,15 +14,5 @@ "path_to_key": "G:\\path\\to\\ssh\\key.pem", "ssh_private_key_password": "my_keys_password" } - }, - "queries_to_measure": [ - { - "name": "Fast Smoke Test", - "query_string": "SELECT 1" - }, - { - "name": "Delivered carts on a day", - "query_string": "select * from comprea.cart c where c.status = 'delivered' and c.date_delivered >= UNIX_TIMESTAMP(date('2022-05-24'))" - } - ] + } } \ No newline at end of file diff --git a/config_examples/query_config_example.json b/config_examples/query_config_example.json new file mode 100644 index 0000000..bd256cf --- /dev/null +++ b/config_examples/query_config_example.json @@ -0,0 +1,12 @@ +{ + "queries_to_measure": [ + { + "name": "Fast Smoke Test", + "query_string": "SELECT 1" + }, + { + "name": "Delivered carts on a day", + "query_string": "select * from comprea.cart c where c.status = 'delivered' and c.date_delivered >= UNIX_TIMESTAMP(date('2022-05-24'))" + } + ] +} \ No newline at end of file diff --git a/config_examples/trino_config.json b/config_examples/trino_config.json deleted file mode 100644 index 8cd8806..0000000 --- a/config_examples/trino_config.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "connection_details": { - "engine": "trino", - "host": "the_trino_host", - "port": "443", - "user": "your_user", - "password": "your_password", - "http_scheme": "https", - "catalog": "app_lm_mysql", - "schema": "comprea" - }, - "queries_to_measure": [ - { - "name": "Fast Smoke Test", - "query_string": "SELECT * FROM system.runtime.nodes" - }, - { - "name": "Delivered carts on a day", - "query_string": "select * from app_lm_mysql.comprea.cart c where c.status = 'delivered' and c.date_delivered >= to_unixtime(date('2022-05-24'))" - } - ] -} \ No newline at end of file diff --git a/config_examples/trino_credentials_config.json b/config_examples/trino_credentials_config.json new file mode 100644 index 0000000..68d1ab7 --- /dev/null +++ b/config_examples/trino_credentials_config.json @@ -0,0 +1,12 @@ +{ + "connection_details": { + "engine": "trino", + "host": "the_trino_host", + "port": "443", + "user": "your_user", + "password": "your_password", + "http_scheme": "https", + "catalog": "app_lm_mysql", + "schema": "comprea" + } +} \ No newline at end of file From ee51c82145ba3e835a4cf304286c8fc3af127d45 Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Mon, 22 Aug 2022 15:05:59 +0200 Subject: [PATCH 5/5] Updated readme. --- readme.md | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/readme.md b/readme.md index 5522b7e..714d9b1 100644 --- a/readme.md +++ b/readme.md @@ -16,28 +16,32 @@ You will install whatever version is in the shared drive at that point. Dependin might want to instead make a copy of the repository in your own local machine and install from there. That way, you won't be affected by someone making `git checkout` in the shared drive. -2. Afterwards, you need to make a config file. See below details on how to compose one. +2. Afterwards, you need to make a credentials and a query config file. See below details on how to compose one. 3. Once you have your config file ready, run the following command from the terminal. ```commandline -measure_query_performance --config my_config_file.json +measure_query_performance --credentials my_credentials.json --queries my_queries.json ``` 4. Results will be printed in your console as they are available. If instead you would like to store them in a file, a quick and easy hack is to redirect output in Powershell to a file. You can do it like this: ```commandline -measure_query_performance --config my_config_file.json | Out-File -FilePath my_results.txt +measure_query_performance --credentials my_credentials.json --queries my_queries.json | Out-File -FilePath my_results.txt ``` -## Composing a config file +## Composing config files + +The application takes two config files: one for the credentials and connection details, one for the queries to run. +You can mix and match several of both (as in, you can run the same query set at different dbs, or you can have +multiple query sets run on the same database). You can take a look at examples for different setups in `config_examples`. If you want to make a new config file, it -will probably be easier for you to start from one of those templates. +will probably be easier for you to start from one of those templates. The legacy examples should be ignored unless +you want to understand outdated config files. A few notes: - - The valid engines are `"trino"` and `"mysql"`. - You can place as many queries as you would like in the `queries_to_measure` list. - I advice you to make the first query a silly, fast query such as `SELECT 1` to validate your connection and