Add ci/cd and fix multiple pods issues

services/procurement/app/services/delivery_tracking_service.py

@@ -1,12 +1,12 @@
 """
-Delivery Tracking Service - Simplified
+Delivery Tracking Service - With Leader Election
 
 Tracks purchase order deliveries and generates appropriate alerts using EventPublisher:
 - DELIVERY_ARRIVING_SOON: 2 hours before delivery window
 - DELIVERY_OVERDUE: 30 minutes after expected delivery time
 - STOCK_RECEIPT_INCOMPLETE: If delivery not marked as received
 
-Runs as internal scheduler with leader election.
+Runs as internal scheduler with leader election for horizontal scaling.
 Domain ownership: Procurement service owns all PO and delivery tracking.
 """
 
@@ -30,7 +30,7 @@ class DeliveryTrackingService:
     Monitors PO deliveries and generates time-based alerts using EventPublisher.
 
     Uses APScheduler with leader election to run hourly checks.
-    Only one pod executes checks (others skip if not leader).
+    Only one pod executes checks - leader election ensures no duplicate alerts.
     """
 
     def __init__(self, event_publisher: UnifiedEventPublisher, config, database_manager=None):
@@ -38,46 +38,121 @@ class DeliveryTrackingService:
         self.config = config
         self.database_manager = database_manager
         self.scheduler = AsyncIOScheduler()
-        self.is_leader = False
+        self._leader_election = None
+        self._redis_client = None
+        self._scheduler_started = False
         self.instance_id = str(uuid4())[:8]  # Short instance ID for logging
 
     async def start(self):
-        """Start the delivery tracking scheduler"""
-        # Initialize and start scheduler if not already running
+        """Start the delivery tracking scheduler with leader election"""
+        try:
+            # Initialize leader election
+            await self._setup_leader_election()
+        except Exception as e:
+            logger.error("Failed to setup leader election, starting in standalone mode",
+                        error=str(e))
+            # Fallback: start scheduler without leader election
+            await self._start_scheduler()
+
+    async def _setup_leader_election(self):
+        """Setup Redis-based leader election for horizontal scaling"""
+        from shared.leader_election import LeaderElectionService
+        import redis.asyncio as redis
+
+        # Build Redis URL from config
+        redis_url = getattr(self.config, 'REDIS_URL', None)
+        if not redis_url:
+            redis_password = getattr(self.config, 'REDIS_PASSWORD', '')
+            redis_host = getattr(self.config, 'REDIS_HOST', 'localhost')
+            redis_port = getattr(self.config, 'REDIS_PORT', 6379)
+            redis_db = getattr(self.config, 'REDIS_DB', 0)
+            redis_url = f"redis://:{redis_password}@{redis_host}:{redis_port}/{redis_db}"
+
+        self._redis_client = redis.from_url(redis_url, decode_responses=False)
+        await self._redis_client.ping()
+
+        # Create leader election service
+        self._leader_election = LeaderElectionService(
+            self._redis_client,
+            service_name="procurement-delivery-tracking"
+        )
+
+        # Start leader election with callbacks
+        await self._leader_election.start(
+            on_become_leader=self._on_become_leader,
+            on_lose_leader=self._on_lose_leader
+        )
+
+        logger.info("Leader election initialized for delivery tracking",
+                   is_leader=self._leader_election.is_leader,
+                   instance_id=self.instance_id)
+
+    async def _on_become_leader(self):
+        """Called when this instance becomes the leader"""
+        logger.info("Became leader for delivery tracking - starting scheduler",
+                   instance_id=self.instance_id)
+        await self._start_scheduler()
+
+    async def _on_lose_leader(self):
+        """Called when this instance loses leadership"""
+        logger.warning("Lost leadership for delivery tracking - stopping scheduler",
+                      instance_id=self.instance_id)
+        await self._stop_scheduler()
+
+    async def _start_scheduler(self):
+        """Start the APScheduler with delivery tracking jobs"""
+        if self._scheduler_started:
+            logger.debug("Scheduler already started", instance_id=self.instance_id)
+            return
+
         if not self.scheduler.running:
             # Add hourly job to check deliveries
             self.scheduler.add_job(
                 self._check_all_tenants,
-                trigger=CronTrigger(minute=30),  # Run every hour at :30 (00:30, 01:30, 02:30, etc.)
+                trigger=CronTrigger(minute=30),  # Run every hour at :30
                 id='hourly_delivery_check',
                 name='Hourly Delivery Tracking',
                 replace_existing=True,
-                max_instances=1,  # Ensure no overlapping runs
-                coalesce=True  # Combine missed runs
+                max_instances=1,
+                coalesce=True
             )
 
             self.scheduler.start()
+            self._scheduler_started = True
 
-            # Log next run time
             next_run = self.scheduler.get_job('hourly_delivery_check').next_run_time
-            logger.info(
-                "Delivery tracking scheduler started with hourly checks",
-                instance_id=self.instance_id,
-                next_run=next_run.isoformat() if next_run else None
-            )
-        else:
-            logger.info(
-                "Delivery tracking scheduler already running",
-                instance_id=self.instance_id
-            )
+            logger.info("Delivery tracking scheduler started",
+                       instance_id=self.instance_id,
+                       next_run=next_run.isoformat() if next_run else None)
+
+    async def _stop_scheduler(self):
+        """Stop the APScheduler"""
+        if not self._scheduler_started:
+            return
+
+        if self.scheduler.running:
+            self.scheduler.shutdown(wait=False)
+            self._scheduler_started = False
+            logger.info("Delivery tracking scheduler stopped", instance_id=self.instance_id)
 
     async def stop(self):
-        """Stop the scheduler and release leader lock"""
-        if self.scheduler.running:
-            self.scheduler.shutdown(wait=True)  # Graceful shutdown
-            logger.info("Delivery tracking scheduler stopped", instance_id=self.instance_id)
-        else:
-            logger.info("Delivery tracking scheduler already stopped", instance_id=self.instance_id)
+        """Stop the scheduler and leader election"""
+        # Stop leader election first
+        if self._leader_election:
+            await self._leader_election.stop()
+            logger.info("Leader election stopped", instance_id=self.instance_id)
+
+        # Stop scheduler
+        await self._stop_scheduler()
+
+        # Close Redis
+        if self._redis_client:
+            await self._redis_client.close()
+
+    @property
+    def is_leader(self) -> bool:
+        """Check if this instance is the leader"""
+        return self._leader_election.is_leader if self._leader_election else True
 
     async def _check_all_tenants(self):
         """