From 76c2dedea38a780731bcf69f98d72768529012c7 Mon Sep 17 00:00:00 2001 From: Pablo Martin Date: Tue, 13 Aug 2024 17:28:52 +0200 Subject: [PATCH] more refactor --- anaxi/constants.py | 1 + anaxi/sync.py | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/anaxi/constants.py b/anaxi/constants.py index 872dd91..80d701d 100644 --- a/anaxi/constants.py +++ b/anaxi/constants.py @@ -5,6 +5,7 @@ from dataclasses import dataclass ASSUMED_PREHISTORICAL_DATETIME = datetime.datetime.fromtimestamp(0) DESTINATION_COLUMN_NAME = "documents" DEFAULT_BATCH_SIZE = 100 +COSMOS_DB_TIMESTAMP_FIELD_KEY = "_ts" @dataclass diff --git a/anaxi/sync.py b/anaxi/sync.py index f1d95df..907b6c7 100644 --- a/anaxi/sync.py +++ b/anaxi/sync.py @@ -11,6 +11,7 @@ from anaxi.config import ( ) from anaxi.constants import ( ASSUMED_PREHISTORICAL_DATETIME, + COSMOS_DB_TIMESTAMP_FIELD_KEY, DEFAULT_BATCH_SIZE, DESTINATION_COLUMN_NAME, PATHS, @@ -94,7 +95,9 @@ class SyncJob: def commit_batch(self, docs_batch, cursor): self.checkpoint_manager.set_new_goal( - goal_timestamp=datetime.datetime.fromtimestamp(docs_batch[-1]["_ts"]) + goal_timestamp=datetime.datetime.fromtimestamp( + docs_batch[-1][COSMOS_DB_TIMESTAMP_FIELD_KEY] + ) ) json_data = [(json.dumps(d),) for d in docs_batch]