counterweight/data-anaxi
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

stuff

Browse source
This commit is contained in:
Pablo Martin 2024-08-13 17:38:29 +02:00
parent 76c2dedea3
commit 44703c5601
3 changed files with 27 additions and 20 deletions
Download patch file Download diff file Expand all files Collapse all files

4
anaxi/checkpoint.py
View file

@ -36,7 +36,9 @@ class CheckpointManager:
write_yaml( write_yaml(
path=self.file_path, path=self.file_path,
data={ data={
CheckpointManager.highest_synced_timestamp_field_key: self.highest_synced_timestamp.isoformat() CheckpointManager.highest_synced_timestamp_field_key: self.highest_synced_timestamp.astimezone(
tz=datetime.timezone.utc
).isoformat()
}, },
) )

23
anaxi/constants.py
View file

@ -2,7 +2,9 @@ import datetime
import pathlib import pathlib
from dataclasses import dataclass from dataclasses import dataclass
ASSUMED_PREHISTORICAL_DATETIME = datetime.datetime.fromtimestamp(0) ASSUMED_PREHISTORICAL_DATETIME = datetime.datetime.fromtimestamp(
0, tz=datetime.timezone.utc
)
DESTINATION_COLUMN_NAME = "documents" DESTINATION_COLUMN_NAME = "documents"
DEFAULT_BATCH_SIZE = 100 DEFAULT_BATCH_SIZE = 100
COSMOS_DB_TIMESTAMP_FIELD_KEY = "_ts" COSMOS_DB_TIMESTAMP_FIELD_KEY = "_ts"
@ -10,17 +12,12 @@ COSMOS_DB_TIMESTAMP_FIELD_KEY = "_ts"
@dataclass @dataclass
class PATHS: class PATHS:
logging_file: pathlib.Path = pathlib.Path("anaxi.log")
# Expand user is important. It will replace the user and give you the full # Expand user is important. It will replace the user and give you the full
# path. Stuff breaks without it. # path. Stuff breaks without it.
config_home_path: pathlib.Path = pathlib.Path("~/.anaxi/").expanduser() home_path: pathlib.Path = pathlib.Path("~/.anaxi/").expanduser()
cosmos_db_config_file_path: pathlib.Path = config_home_path / pathlib.Path( logging_file: pathlib.Path = home_path / pathlib.Path("anaxi.log")
"cosmos-db.yml" cosmos_db_config_file_path: pathlib.Path = home_path / pathlib.Path("cosmos-db.yml")
) postgres_config_file_path: pathlib.Path = home_path / pathlib.Path("postgres.yml")
postgres_config_file_path: pathlib.Path = config_home_path / pathlib.Path( streams_config_file_path: pathlib.Path = home_path / pathlib.Path("streams.yml")
"postgres.yml" checkpoints_folder_path: pathlib.Path = home_path / "checkpoints/"
)
streams_config_file_path: pathlib.Path = config_home_path / pathlib.Path(
"streams.yml"
)
checkpoints_folder_path: pathlib.Path = config_home_path / "checkpoints/"

20
anaxi/logging.py
View file

@ -1,15 +1,23 @@
import logging import logging
from anaxi.constants import PATHS
def get_anaxi_logger(name: str) -> None: def get_anaxi_logger(name: str) -> None:
logger = logging.getLogger(name) logger = logging.getLogger(name)
handler = logging.StreamHandler()
handler.setFormatter( anaxi_formatter = logging.Formatter(
logging.Formatter( fmt="%(asctime)s - [%(levelname)s] - %(filename)s - L%(lineno)d - %(message)s"
fmt="%(asctime)s - [%(levelname)s] - %(filename)s - L%(lineno)d - %(message)s"
)
) )
logger.addHandler(handler)
console_handler = logging.StreamHandler()
console_handler.setFormatter(anaxi_formatter)
logger.addHandler(console_handler)
file_handler = logging.FileHandler(filename=PATHS.logging_file)
file_handler.setFormatter(anaxi_formatter)
logger.addHandler(file_handler)
logger.setLevel(logging.INFO) logger.setLevel(logging.INFO)
return logger return logger