Coverage for adrs/oms/rate_limit/error_policy.py: 98%
66 statements
« prev ^ index » next coverage.py v7.15.2, created at 2026-08-29 05:07 +0800
1import re
2from abc import ABC, abstractmethod
3from enum import Enum, auto
5from cybotrade.binance import BinanceError
6from cybotrade.bybit import BybitError
7from cybotrade.hyperliquid import HyperliquidError
9# Binance reports rate limiting through the JSON body code, never the HTTP
10# status: -1003 arrives with both 429 (request budget exhausted) and 418 (IP
11# banned), and -1015 with an order-rate breach. Matching 418/429 against
12# BinanceError.code -- which only ever holds the body code -- can therefore
13# never fire, which is how an OMS came to poll straight through an hour-long IP
14# ban, renewing it with every request.
15BINANCE_RATE_LIMIT_CODES = frozenset({-1003, -1015})
16# Scoped to the account's order budget, so these only need to stop order
17# placement; reads can carry on.
18BINANCE_ORDER_RATE_LIMIT_CODES = frozenset({-1015})
19# Checked as well as the body code, because a 429/418 raised by Binance's edge
20# rather than its matching engine can carry a body with no code field at all.
21BINANCE_RATE_LIMIT_HTTP_STATUSES = frozenset({418, 429})
23# Binance embeds the ban deadline in the -1003 message, e.g. "Way too many
24# requests; IP(1.2.3.4) banned until 1786361397856." That epoch-ms deadline is
25# the only authoritative recovery time on the response, since a 418 frequently
26# carries no Retry-After header.
27_BINANCE_BANNED_UNTIL_RE = re.compile(r"banned until (\d{13,})")
30def is_binance_rate_limit_error(exc: BinanceError) -> bool:
31 """Whether this Binance error means we are being rate limited or banned."""
32 if exc.code in BINANCE_RATE_LIMIT_CODES:
33 return True
34 # cybotrade only began carrying the HTTP status on BinanceError in 2.0.19;
35 # on older versions the attribute is absent and the body code is all there
36 # is to go on.
37 return getattr(exc, "http_status", None) in BINANCE_RATE_LIMIT_HTTP_STATUSES
40def binance_banned_until_ms(message: str) -> int | None:
41 """Epoch-ms ban deadline from a -1003 message, or None if it carries none."""
42 match = _BINANCE_BANNED_UNTIL_RE.search(message)
43 if match is None:
44 return None
45 return int(match.group(1))
48class ErrorAction(Enum):
49 TERMINAL_SUCCESS = auto() # order already gone; treat as done, no retry
50 RETRY = auto() # transient/unknown; backlog + backoff (default)
51 RATE_LIMITED = auto() # rate code; cooldown is armed in the limiter's guard()
52 FATAL = auto() # unrecoverable for these params; drop + log, no retry
55# Unlisted codes fall through to RETRY, matching legacy behaviour. FATAL entries
56# stop retries, so only add a code here once its terminal nature is confirmed.
57BYBIT_ERROR_ACTIONS: dict[int, ErrorAction] = {
58 110001: ErrorAction.TERMINAL_SUCCESS, # order not exists or too late to cancel
59 10006: ErrorAction.RATE_LIMITED, # too many visits
60}
62# Keyed on Binance's body codes. The 418/429 that used to sit here were HTTP
63# statuses and so never matched; a rate-limit error fell through to RETRY, and
64# the backlog kept re-sending orders every 2s for the length of the ban.
65BINANCE_ERROR_ACTIONS: dict[int, ErrorAction] = {
66 -2011: ErrorAction.TERMINAL_SUCCESS, # unknown order (already gone)
67 **{code: ErrorAction.RATE_LIMITED for code in BINANCE_RATE_LIMIT_CODES},
68}
71class ExchangeErrorPolicy(ABC):
72 """Maps an exchange exception to the behaviour the OMS should take."""
74 default_action: ErrorAction = ErrorAction.RETRY
76 @abstractmethod
77 def classify(self, exc: Exception) -> ErrorAction: ...
80class BybitErrorPolicy(ExchangeErrorPolicy):
81 def classify(self, exc: Exception) -> ErrorAction:
82 if isinstance(exc, BybitError):
83 if exc.http_status == 403:
84 return ErrorAction.RATE_LIMITED
85 if exc.retCode is not None:
86 return BYBIT_ERROR_ACTIONS.get(exc.retCode, self.default_action)
87 return self.default_action
90class BinanceErrorPolicy(ExchangeErrorPolicy):
91 def classify(self, exc: Exception) -> ErrorAction:
92 if not isinstance(exc, BinanceError):
93 return self.default_action
94 # Status first: a 429/418 whose body has no code at all is still a rate
95 # limit, and must not be retried as though it were transient.
96 if is_binance_rate_limit_error(exc):
97 return ErrorAction.RATE_LIMITED
98 if exc.code is not None:
99 return BINANCE_ERROR_ACTIONS.get(exc.code, self.default_action)
100 return self.default_action
103class DefaultErrorPolicy(ExchangeErrorPolicy):
104 """Everything retries — legacy behaviour for exchanges without a policy."""
106 def classify(self, exc: Exception) -> ErrorAction:
107 return self.default_action
110# Hyperliquid reports failure in the body of an HTTP 200 and offers no stable
111# numeric code, so classification is by substring. Ordered: first match wins.
112# A whitelist with an inert default -- anything unmatched keeps the legacy
113# RETRY behaviour rather than being guessed at.
114#
115# Only the first entry is confirmed against the live exchange. The rest are the
116# categories the OMS must not retry blindly; drop any that cannot be confirmed
117# rather than keep a match that silently never fires.
118HYPERLIQUID_ERROR_ACTIONS: tuple[tuple[str, ErrorAction], ...] = (
119 # Verified live: returned when cancelling an order id that is already gone.
120 # The caller wanted it gone, so this is success, not failure -- the same
121 # reading as Bybit's 110001.
122 ("was never placed, already canceled, or filled", ErrorAction.TERMINAL_SUCCESS),
123 ("rate limit", ErrorAction.RATE_LIMITED),
124 ("too many requests", ErrorAction.RATE_LIMITED),
125 ("insufficient margin", ErrorAction.FATAL),
126 ("too far", ErrorAction.FATAL),
127 ("reduce only", ErrorAction.FATAL),
128)
130# Checked as well as the message, and before it. Hyperliquid reports *action*
131# failures in the body of an HTTP 200, which is why the needles above exist at
132# all -- but a 429 comes from the edge, not the matching engine, and carries no
133# Hyperliquid body to match against. Its text is whatever the proxy wrote, so
134# the status is the only signal. Missing it is how a process polls straight
135# through a throttle and renews it.
136HYPERLIQUID_RATE_LIMIT_HTTP_STATUSES = frozenset({429})
139def _hyperliquid_action(exc: Exception) -> ErrorAction | None:
140 """The mapped action for a HyperliquidError, or None if nothing matched."""
141 if not isinstance(exc, HyperliquidError):
142 return None
143 # Status first: a 429 is a rate limit whatever its body says, and must not
144 # be read as a FATAL or fall through to a retry loop on message text alone.
145 if getattr(exc, "status", None) in HYPERLIQUID_RATE_LIMIT_HTTP_STATUSES:
146 return ErrorAction.RATE_LIMITED
147 message = (exc.message or "").lower()
148 for needle, action in HYPERLIQUID_ERROR_ACTIONS:
149 if needle in message:
150 return action
151 return None
154def is_hyperliquid_rate_limit_error(exc: Exception) -> bool:
155 """
156 Whether this error is Hyperliquid throttling us.
158 True for an HTTP 429 regardless of message text, and for the throttle
159 message needles. Used by HyperliquidRateLimiter to arm its cooldown, so it
160 deliberately requires a HyperliquidError: an unrelated exception whose text
161 happens to contain "rate limit" must not stall every call for a minute.
162 """
163 return _hyperliquid_action(exc) is ErrorAction.RATE_LIMITED
166class HyperliquidErrorPolicy(ExchangeErrorPolicy):
167 def classify(self, exc: Exception) -> ErrorAction:
168 action = _hyperliquid_action(exc)
169 return self.default_action if action is None else action