« prev ^ index » next coverage.py v7.15.4, created at 2026-08-25 07:19 +0800
1"""Reactive failover tracking for the relay gateway.
3``RelayFailoverTracker`` turns consecutive upstream failures into runtime
4selection state: a channel that fails ``threshold`` times in a row is
5taken out of service through the registry's runtime overrides, and the
6next successful dispatch on that channel restores it. The tracker only
7restores channels it disabled itself — an operator drain through the
8permissioned controls surface is never silently reversed (mirroring the
9auto-tester's journal semantics).
11Failures are recorded per channel and reset on success; the tracker never
12reads request payloads and holds no upstream details.
14Deliberate divergence from ``lexigram-resilience``: this tracker does not
15reuse ``CircuitBreaker``. Gateway failures are ``Result``-based
16``RelayGatewayError`` values, not exceptions, and the breaker's state is
17not shared with the runtime overrides that selection, the operator
18controls surface, and the auto-tester all read. Recovery also differs:
19one successful dispatch restores, mirroring new-api. If distributed ban
20state is ever needed, add a storage protocol like
21``CircuitBreakerBackend`` instead of growing in-memory semantics.
22"""
24from __future__ import annotations
26from lexigram.ai.relay.gateway.channels import RelayChannelRegistry
27from lexigram.logging import get_logger
29__all__ = ["RelayFailoverTracker"]
31logger = get_logger(__name__)
34class RelayFailoverTracker:
35 """Track consecutive upstream failures and ban failing channels.
37 The tracker mutates only the registry's runtime enabled overrides,
38 the same surface the operator controls and the auto-tester use. The
39 threshold is compared with ``>=`` so the ban happens on the attempt
40 that reaches it.
42 Args:
43 registry: The channel registry whose runtime enable flags this
44 tracker mutates.
45 threshold: Consecutive failures that disable a channel. Must be
46 positive.
47 """
49 def __init__(
50 self,
51 registry: RelayChannelRegistry,
52 threshold: int,
53 ) -> None:
54 """Bind the tracker to the registry and threshold.
56 Args:
57 registry: The channel registry receiving runtime transitions.
58 threshold: Consecutive failures that disable a channel.
59 """
60 if threshold < 1:
61 raise ValueError("threshold must be a positive integer")
62 self._registry = registry
63 self._threshold = threshold
64 self._failures: dict[str, int] = {}
65 self._banned: set[str] = set()
67 @property
68 def threshold(self) -> int:
69 """Return the consecutive-failure threshold.
71 Returns:
72 The threshold configured at construction.
73 """
74 return self._threshold
76 def failure_count(self, channel: str) -> int:
77 """Return the recorded consecutive failures for *channel*.
79 Args:
80 channel: The channel name to inspect.
82 Returns:
83 The consecutive failure count, ``0`` when none recorded.
84 """
85 return self._failures.get(channel, 0)
87 def banned(self) -> frozenset[str]:
88 """Return the channels this tracker disabled.
90 Returns:
91 The immutable set of channel names banned by this tracker.
92 """
93 return frozenset(self._banned)
95 def record_failure(self, channel: str) -> None:
96 """Count one upstream failure for *channel* and ban at threshold.
98 When the count reaches the threshold and the channel was not
99 already banned, the channel is drained through the registry's
100 runtime overrides and journaled as banned by this tracker.
102 Args:
103 channel: The channel name that failed upstream.
104 """
105 count = self._failures.get(channel, 0) + 1
106 self._failures[channel] = count
107 if count >= self._threshold and channel not in self._banned:
108 self._registry.set_runtime_enabled(channel, False)
109 self._banned.add(channel)
110 logger.info(
111 "relay_gateway_channel_banned",
112 channel=channel,
113 failures=count,
114 threshold=self._threshold,
115 )
117 def record_success(self, channel: str) -> None:
118 """Reset *channel*'s failures and restore it when banned here.
120 A successful dispatch clears the consecutive-failure count; when
121 this tracker had banned the channel, it is restored at runtime
122 and removed from the ban journal.
124 Args:
125 channel: The channel name that succeeded upstream.
126 """
127 if channel in self._failures:
128 del self._failures[channel]
129 if channel in self._banned:
130 self._registry.set_runtime_enabled(channel, True)
131 self._banned.discard(channel)
132 logger.info(
133 "relay_gateway_channel_restored",
134 channel=channel,
135 reason="dispatch_recovered",
136 )