counterweight/data-anaxi
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

more refactor

Browse source
This commit is contained in:
Pablo Martin 2024-08-13 17:28:52 +02:00
parent 58e99a1869
commit 76c2dedea3
2 changed files with 5 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

1
anaxi/constants.py
View file

@ -5,6 +5,7 @@ from dataclasses import dataclass
ASSUMED_PREHISTORICAL_DATETIME = datetime.datetime.fromtimestamp(0) ASSUMED_PREHISTORICAL_DATETIME = datetime.datetime.fromtimestamp(0)
DESTINATION_COLUMN_NAME = "documents" DESTINATION_COLUMN_NAME = "documents"
DEFAULT_BATCH_SIZE = 100 DEFAULT_BATCH_SIZE = 100
COSMOS_DB_TIMESTAMP_FIELD_KEY = "_ts"
@dataclass @dataclass

5
anaxi/sync.py
View file

@ -11,6 +11,7 @@ from anaxi.config import (
) )
from anaxi.constants import ( from anaxi.constants import (
ASSUMED_PREHISTORICAL_DATETIME, ASSUMED_PREHISTORICAL_DATETIME,
COSMOS_DB_TIMESTAMP_FIELD_KEY,
DEFAULT_BATCH_SIZE, DEFAULT_BATCH_SIZE,
DESTINATION_COLUMN_NAME, DESTINATION_COLUMN_NAME,
PATHS, PATHS,
@ -94,7 +95,9 @@ class SyncJob:
def commit_batch(self, docs_batch, cursor): def commit_batch(self, docs_batch, cursor):
self.checkpoint_manager.set_new_goal( self.checkpoint_manager.set_new_goal(
goal_timestamp=datetime.datetime.fromtimestamp(docs_batch[-1]["_ts"]) goal_timestamp=datetime.datetime.fromtimestamp(
docs_batch[-1][COSMOS_DB_TIMESTAMP_FIELD_KEY]
)
) )
json_data = [(json.dumps(d),) for d in docs_batch] json_data = [(json.dumps(d),) for d in docs_batch]