Improve the traffic fetching system

This commit is contained in:
Urtzi Alfaro
2025-08-10 17:31:38 +02:00
parent 312fdc8ef3
commit 3c2acc934a
16 changed files with 3866 additions and 1981 deletions

View File

@@ -300,8 +300,14 @@ class BaseRepository(Generic[Model, CreateSchema, UpdateSchema], ABC):
self.session.add_all(db_objects)
await self.session.flush()
for db_obj in db_objects:
await self.session.refresh(db_obj)
# Skip expensive individual refresh operations for large datasets
# Only refresh if we have a small number of objects
if len(db_objects) <= 100:
for db_obj in db_objects:
await self.session.refresh(db_obj)
else:
# For large datasets, just log without refresh to prevent memory issues
logger.debug(f"Skipped individual refresh for large bulk operation ({len(db_objects)} records)")
logger.debug(f"Bulk created {len(db_objects)} {self.model.__name__} records")
return db_objects