Fetch trino credentials and test

This commit is contained in:
Pablo Martin 2023-01-13 14:56:36 +01:00
parent 707ac6ad7c
commit 503c70fa02
2 changed files with 45 additions and 4 deletions

View file

@ -9,7 +9,7 @@ from lolafect.defaults import (
DEFAULT_KUBERNETES_IMAGE,
DEFAULT_KUBERNETES_LABELS,
DEFAULT_FLOWS_PATH_IN_BUCKET,
DEFAULT_ENV_FILE_PATH
DEFAULT_ENV_FILE_PATH,
)
from lolafect.utils import S3FileReader
@ -68,6 +68,8 @@ class LolaConfig:
DEFAULT_KUBERNETES_IMAGE if kubernetes_image is None else kubernetes_image
)
self.TRINO_CREDENTIALS = None
self._s3_reader = S3FileReader(s3_client=boto3.client("s3"))
def fetch_slack_webhooks(self, s3_reader=None) -> None:
@ -85,6 +87,28 @@ class LolaConfig:
bucket=self.S3_BUCKET_NAME, key=self.SLACK_WEBHOOKS_FILE
)
def fetch_trino_credentials(self, s3_reader=None) -> None:
"""
Read the env file from S3 and store the trino credentials in memory.
:param s3_reader: a client to fetch files from S3.
:return: None
"""
if s3_reader is None:
s3_reader = self._s3_reader
env_data = s3_reader.read_json_from_s3_file(
bucket=self.S3_BUCKET_NAME, key=self.ENV_FILE_PATH
)
self.TRINO_CREDENTIALS = {
"host": env_data["trino_host"],
"user": env_data["trino_user"],
"password": env_data["trino_pass"],
"port": env_data["trino_port"],
}
def build_lolaconfig(
flow_name: str,