feat(safety): enhance rate limiting and cost control with alert deduplication and usage tracking

- Added `record_action` in `RateLimiter` for precise tracking of slot consumption post-validation.
- Introduced deduplication mechanism for warning alerts in `CostController` to prevent spamming.
- Refactored `CostController`'s session and daily budget alert handling for improved clarity.
- Implemented test suites for `CostController` and `SafetyGuardian` to validate changes.
- Expanded integration testing to cover deduplication, validation, and loop detection edge cases.

backend/app/services/safety/costs/controller.py

@@ -185,6 +185,9 @@ class CostController:
         # Alert handlers
         self._alert_handlers: list[Any] = []
 
+        # Track which budgets have had warning alerts sent (to avoid spam)
+        self._warned_budgets: set[str] = set()
+
     async def get_or_create_tracker(
         self,
         scope: BudgetScope,
@@ -343,32 +346,44 @@ class CostController:
         """
         # Update session budget
         if session_id:
+            session_key = f"session:{session_id}"
             session_tracker = await self.get_or_create_tracker(
                 BudgetScope.SESSION, session_id
             )
             await session_tracker.add_usage(tokens, cost_usd)
 
-            # Check for warning
+            # Check for warning (only alert once per budget to avoid spam)
             status = await session_tracker.get_status()
             if status.is_warning and not status.is_exceeded:
-                await self._send_alert(
-                    "warning",
-                    f"Session {session_id} at {status.tokens_used}/{status.tokens_limit} tokens",
-                    status,
-                )
+                if session_key not in self._warned_budgets:
+                    self._warned_budgets.add(session_key)
+                    await self._send_alert(
+                        "warning",
+                        f"Session {session_id} at {status.tokens_used}/{status.tokens_limit} tokens",
+                        status,
+                    )
+            elif not status.is_warning:
+                # Clear warning flag if usage dropped below threshold (e.g., after reset)
+                self._warned_budgets.discard(session_key)
 
         # Update agent daily budget
+        daily_key = f"daily:{agent_id}"
         agent_tracker = await self.get_or_create_tracker(BudgetScope.DAILY, agent_id)
         await agent_tracker.add_usage(tokens, cost_usd)
 
-        # Check for warning
+        # Check for warning (only alert once per budget to avoid spam)
         status = await agent_tracker.get_status()
         if status.is_warning and not status.is_exceeded:
-            await self._send_alert(
-                "warning",
-                f"Agent {agent_id} at {status.tokens_used}/{status.tokens_limit} daily tokens",
-                status,
-            )
+            if daily_key not in self._warned_budgets:
+                self._warned_budgets.add(daily_key)
+                await self._send_alert(
+                    "warning",
+                    f"Agent {agent_id} at {status.tokens_used}/{status.tokens_limit} daily tokens",
+                    status,
+                )
+        elif not status.is_warning:
+            # Clear warning flag if usage dropped below threshold (e.g., after reset)
+            self._warned_budgets.discard(daily_key)
 
     async def get_status(
         self,
@@ -388,20 +403,18 @@ class CostController:
         key = f"{scope.value}:{scope_id}"
         async with self._lock:
             tracker = self._trackers.get(key)
-
-        if tracker:
-            return await tracker.get_status()
-        return None
+            # Get status while holding lock to prevent TOCTOU race
+            if tracker:
+                return await tracker.get_status()
+            return None
 
     async def get_all_statuses(self) -> list[BudgetStatus]:
         """Get status of all tracked budgets."""
         statuses = []
         async with self._lock:
-            trackers = list(self._trackers.values())
-
-        for tracker in trackers:
-            statuses.append(await tracker.get_status())
-
+            # Get all statuses while holding lock to prevent TOCTOU race
+            for tracker in self._trackers.values():
+                statuses.append(await tracker.get_status())
         return statuses
 
     async def set_budget(
@@ -453,11 +466,11 @@ class CostController:
         key = f"{scope.value}:{scope_id}"
         async with self._lock:
             tracker = self._trackers.get(key)
-
-        if tracker:
-            await tracker.reset()
-            return True
-        return False
+            # Reset while holding lock to prevent TOCTOU race
+            if tracker:
+                await tracker.reset()
+                return True
+            return False
 
     def add_alert_handler(self, handler: Any) -> None:
         """Add an alert handler."""