Coverage for /home/admin/Documents/AI/applications/lexigram-dev/lexigram/experimental/ai/lexigram-ai-llm/src/lexigram/ai/llm/routing/config.py: 43%

1"""Configuration schema for LLM multi-provider routing. 

2 

3Defines all typed configuration objects accepted by ``LLMRoutingProvider`` 

4and ``LLMRouter``. Use ``LLMConfig.from_env()`` to build from environment 

5variables following the ``LEX_AI_LLM__`` prefix convention. 

6""" 

7 

8from __future__ import annotations 

9 

10from dataclasses import dataclass 

11import os 

12from typing import ClassVar, Literal 

13 

14from lexigram.config.base import BaseConfig 

15from lexigram.validation import ConfigDict, Field, SecretStr 

16 

17__all__ = [ 

18 "GenerationDefaults", 

19 "LLMConfig", 

20 "LogConfig", 

21 "ProviderConfig", 

22 "QuotaConfig", 

23] 

24 

25_ENV = "LEX_AI_LLM__" 

26 

27 

28@dataclass(init=False) 

29class GenerationDefaults(BaseConfig): 

30 """Default generation parameters applied to every routing attempt. 

31 

32 Example: 

33 >>> defaults = GenerationDefaults(temperature=0.3, max_tokens=2048) 

34 """ 

35 

36 model_config: ClassVar[ConfigDict] = ConfigDict(extra="ignore") 

37 

38 temperature: float = Field( 

39 default=0.2, 

40 ge=0.0, 

41 le=2.0, 

42 description="Default sampling temperature.", 

43 ) 

44 max_tokens: int | None = Field( 

45 default=None, 

46 ge=1, 

47 description="Default maximum output tokens (None = provider default).", 

48 ) 

49 

50 

51@dataclass(init=False) 

52class QuotaConfig(BaseConfig): 

53 """Configuration for the quota tracking backend. 

54 

55 Example: 

56 >>> cfg = QuotaConfig(backend="database") 

57 """ 

58 

59 model_config: ClassVar[ConfigDict] = ConfigDict(extra="ignore") 

60 

61 backend: Literal["memory", "database"] = Field( 

62 default="memory", 

63 description="Backend type: ``memory`` or ``database``.", 

64 ) 

65 cooldown_seconds: int = Field( 

66 default=300, 

67 ge=1, 

68 description=( 

69 "Cooldown applied to a cascade entry after an HTTP 429. " 

70 "Env var: LEX_AI_LLM__QUOTA__COOLDOWN_SECONDS." 

71 ), 

72 ) 

73 

74 

75@dataclass(init=False) 

76class LogConfig(BaseConfig): 

77 """Configuration for inference attempt logging. 

78 

79 Example: 

80 >>> cfg = LogConfig(backend="database", max_entries=5000) 

81 """ 

82 

83 model_config: ClassVar[ConfigDict] = ConfigDict(extra="ignore") 

84 

85 backend: Literal["memory", "database"] = Field( 

86 default="memory", 

87 description="Backend type: ``memory`` or ``database``.", 

88 ) 

89 max_entries: int = Field( 

90 default=1000, 

91 ge=1, 

92 description="Maximum in-memory log entries before FIFO eviction.", 

93 ) 

94 

95 

96@dataclass(init=False) 

97class ProviderConfig(BaseConfig): 

98 """Configuration for a single provider in the routing cascade. 

99 

100 Every provider in the cascade has the same shape regardless of type. 

101 Provider-specific fields (Azure deployment, Cloudflare account ID, 

102 Bedrock region, Vertex project) go in ``extras``. 

103 

104 Example: 

105 >>> cfg = ProviderConfig( 

106 ... name="groq", 

107 ... model="llama-3.3-70b-versatile", 

108 ... api_key="gsk_...", 

109 ... ) 

110 """ 

111 

112 model_config: ClassVar[ConfigDict] = ConfigDict( 

113 arbitrary_types_allowed=True, extra="ignore" 

114 ) 

115 

116 name: str = Field(description="Provider name (must match ProviderRegistry key).") 

117 model: str = Field(description="Primary model identifier.") 

118 api_key: SecretStr | None = Field( 

119 default=None, 

120 description="API key for key-authenticated providers.", 

121 ) 

122 base_url: str | None = Field( 

123 default=None, 

124 description="Base URL override (local providers, Azure, custom endpoints).", 

125 ) 

126 timeout: int = Field( 

127 default=30, 

128 ge=5, 

129 description="Per-request timeout in seconds.", 

130 ) 

131 enabled: bool = Field( 

132 default=True, 

133 description="Whether this provider participates in routing.", 

134 ) 

135 suppress_thinking: bool = Field( 

136 default=False, 

137 description=( 

138 "When True, the LLM client actively disables thinking/chain-of-thought " 

139 "token generation by injecting ``enable_thinking: false`` into the request " 

140 "payload. Use for models that think by default (Qwen3, Gemma-4 via LM Studio " 

141 "/ vLLM / SGLang) to eliminate the 20-30 s latency overhead. " 

142 "Env var: ``LEX_AI_LLM__PROVIDERS__{NAME}__SUPPRESS_THINKING=true``." 

143 ), 

144 ) 

145 extras: dict[str, str] = Field( 

146 default_factory=dict, 

147 description=( 

148 "Provider-specific credential fields: " 

149 "azure_resource, azure_deployment, azure_api_version; " 

150 "cf_account_id, cf_api_token; " 

151 "aws_region, aws_access_key_id, aws_secret_access_key; " 

152 "vertex_project, vertex_location, vertex_credentials_file." 

153 ), 

154 ) 

155 

156 @property 

157 def key(self) -> str: 

158 """Unique cascade-entry identity: provider name + model. 

159 

160 Multiple entries may share ``name`` (e.g. several OpenRouter models); 

161 quota, clients, and routing state must key on this, never on ``name``. 

162 """ 

163 return f"{self.name}:{self.model}" 

164 

165 

166@dataclass(init=False) 

167class LLMConfig(BaseConfig): 

168 """Root configuration object for the LLM routing system. 

169 

170 All providers are opt-in: a provider joins the cascade only when its 

171 credential environment variable is set. Use :meth:`from_env` to build 

172 from ``LEX_AI_LLM__`` environment variables. 

173 

174 Example:: 

175 

176 config = LLMConfig( 

177 providers=[ 

178 ProviderConfig(name="groq", model="llama-3.3-70b-versatile", api_key="gsk_..."), 

179 ProviderConfig(name="gemini", model="gemini-2.5-flash", api_key="AIza..."), 

180 ], 

181 defaults=GenerationDefaults(temperature=0.3), 

182 ) 

183 

184 Environment variables (prefix ``LEX_AI_LLM__``):: 

185 

186 Global: 

187 

188 LEX_AI_LLM__STRATEGY sequential | parallel_race | 

189 cost_optimized | latency_optimized 

190 LEX_AI_LLM__DEFAULTS__TEMPERATURE float (default 0.2) 

191 LEX_AI_LLM__DEFAULTS__MAX_TOKENS int (default: provider default) 

192 LEX_AI_LLM__QUOTA__BACKEND memory | database (default memory) 

193 LEX_AI_LLM__LOG__BACKEND memory | database (default memory) 

194 LEX_AI_LLM__LOG__MAX_ENTRIES int (default 1000) 

195 

196 Per-provider (pattern: LEX_AI_LLM__PROVIDERS__{NAME}__{FIELD}): 

197 

198 __{NAME}__API_KEY str API key -- activates key-auth providers 

199 __{NAME}__BASE_URL str Endpoint -- activates local/custom providers 

200 __{NAME}__MODEL str Model override (has per-provider defaults) 

201 __{NAME}__TIMEOUT int Request timeout in seconds (default 30) 

202 __{NAME}__ENABLED bool Explicit enable/disable (default true) 

203 

204 Supported provider names and their activation: 

205 

206 OPENAI API_KEY required default model: gpt-4o 

207 ANTHROPIC API_KEY required default model: claude-3-5-sonnet-20241022 

208 GROQ API_KEY required default model: llama-3.3-70b-versatile 

209 GEMINI API_KEY required default model: gemini-2.5-flash 

210 MISTRAL API_KEY required default model: mistral-large-latest 

211 COHERE API_KEY required default model: command-r-plus 

212 OPENROUTER API_KEY required default model: openai/gpt-4o-mini 

213 DEEPSEEK API_KEY required default model: deepseek-chat 

214 TOGETHER API_KEY required default model: meta-llama/Llama-3-8b-chat-hf 

215 FIREWORKS API_KEY required default model: accounts/fireworks/models/llama-v3-70b-instruct 

216 OLLAMA BASE_URL required default model: llama3.2 (default base: http://localhost:11434) 

217 OPENAI_COMPATIBLE BASE_URL + MODEL required (generic OpenAI-compatible: LM Studio, VLLM, etc.) 

218 

219 Azure-specific extras (activated by AZURE__API_KEY + AZURE__BASE_URL): 

220 

221 LEX_AI_LLM__PROVIDERS__AZURE__EXTRAS__AZURE_RESOURCE 

222 LEX_AI_LLM__PROVIDERS__AZURE__EXTRAS__AZURE_DEPLOYMENT 

223 LEX_AI_LLM__PROVIDERS__AZURE__EXTRAS__AZURE_API_VERSION 

224 

225 Cloudflare-specific extras (activated by CLOUDFLARE__EXTRAS__CF_ACCOUNT_ID): 

226 

227 LEX_AI_LLM__PROVIDERS__CLOUDFLARE__EXTRAS__CF_ACCOUNT_ID <- activates 

228 LEX_AI_LLM__PROVIDERS__CLOUDFLARE__EXTRAS__CF_API_TOKEN 

229 LEX_AI_LLM__PROVIDERS__CLOUDFLARE__MODEL 

230 

231 AWS Bedrock extras (activated by BEDROCK__EXTRAS__AWS_REGION): 

232 

233 LEX_AI_LLM__PROVIDERS__BEDROCK__EXTRAS__AWS_REGION <- activates 

234 LEX_AI_LLM__PROVIDERS__BEDROCK__MODEL 

235 LEX_AI_LLM__PROVIDERS__BEDROCK__EXTRAS__AWS_ACCESS_KEY_ID 

236 LEX_AI_LLM__PROVIDERS__BEDROCK__EXTRAS__AWS_SECRET_ACCESS_KEY 

237 

238 Google Vertex AI extras (activated by VERTEX__EXTRAS__VERTEX_PROJECT): 

239 

240 LEX_AI_LLM__PROVIDERS__VERTEX__EXTRAS__VERTEX_PROJECT <- activates 

241 LEX_AI_LLM__PROVIDERS__VERTEX__EXTRAS__VERTEX_LOCATION 

242 LEX_AI_LLM__PROVIDERS__VERTEX__MODEL 

243 LEX_AI_LLM__PROVIDERS__VERTEX__EXTRAS__VERTEX_CREDENTIALS_FILE 

244 """ 

245 

246 model_config: ClassVar[ConfigDict] = ConfigDict(extra="ignore") 

247 

248 providers: list[ProviderConfig] = Field( 

249 default_factory=list, 

250 description="Ordered list of providers (cascade order). First = highest priority.", 

251 ) 

252 defaults: GenerationDefaults = Field( 

253 default_factory=GenerationDefaults, 

254 description="Default generation parameters.", 

255 ) 

256 quota: QuotaConfig = Field( 

257 default_factory=QuotaConfig, 

258 description="Quota backend configuration.", 

259 ) 

260 logging: LogConfig = Field( 

261 default_factory=LogConfig, 

262 description="Inference logger configuration.", 

263 ) 

264 strategy: Literal[ 

265 "sequential", 

266 "parallel_race", 

267 "cost_optimized", 

268 "latency_optimized", 

269 ] = Field( 

270 default="sequential", 

271 description=( 

272 "Routing strategy. " 

273 "``sequential``: try providers in cascade order. " 

274 "``parallel_race``: fire all in parallel, take the first success. " 

275 "``cost_optimized``: cheapest providers first. " 

276 "``latency_optimized``: fastest recent provider first." 

277 ), 

278 ) 

279 

280 @classmethod 

281 def from_env(cls) -> LLMConfig: 

282 """Build a routing config from ``LEX_AI_LLM__`` environment variables. 

283 

284 Returns: 

285 Populated :class:`LLMConfig`. 

286 """ 

287 

288 def _str(key: str, default: str = "") -> str: 

289 return os.environ.get(key, default).strip() 

290 

291 def _int(key: str, default: int) -> int: 

292 raw = _str(key) 

293 return int(raw) if raw else default 

294 

295 def _bool(key: str, default: bool = True) -> bool: 

296 raw = _str(key).lower() 

297 if raw in ("false", "0", "no"): 

298 return False 

299 if raw in ("true", "1", "yes"): 

300 return True 

301 return default 

302 

303 def _secret(key: str) -> SecretStr | None: 

304 val = _str(key) 

305 return SecretStr(val) if val else None 

306 

307 def _p(name: str, fld: str) -> str: 

308 """Build provider env var key.""" 

309 return f"{_ENV}PROVIDERS__{name}__{fld}" 

310 

311 def _extras(name: str, key: str) -> str: 

312 return _str(f"{_ENV}PROVIDERS__{name}__EXTRAS__{key}") 

313 

314 providers: list[ProviderConfig] = [] 

315 

316 # Key-auth cloud providers 

317 _KEY_PROVIDERS: list[tuple[str, str]] = [ 

318 ("OPENAI", "gpt-4o"), 

319 ("ANTHROPIC", "claude-3-5-sonnet-20241022"), 

320 ("GROQ", "llama-3.3-70b-versatile"), 

321 ("GEMINI", "gemini-2.5-flash"), 

322 ("MISTRAL", "mistral-large-latest"), 

323 ("COHERE", "command-r-plus"), 

324 ("OPENROUTER", "openai/gpt-4o-mini"), 

325 ("DEEPSEEK", "deepseek-chat"), 

326 ("TOGETHER", "meta-llama/Llama-3-8b-chat-hf"), 

327 ("FIREWORKS", "accounts/fireworks/models/llama-v3-70b-instruct"), 

328 ] 

329 for name, default_model in _KEY_PROVIDERS: 

330 key = _secret(_p(name, "API_KEY")) 

331 if not key: 

332 continue 

333 providers.append( 

334 ProviderConfig( 

335 name=name.lower(), 

336 model=_str(_p(name, "MODEL"), default_model), 

337 api_key=key, 

338 base_url=_str(_p(name, "BASE_URL")) or None, 

339 timeout=_int(_p(name, "TIMEOUT"), 30), 

340 enabled=_bool(_p(name, "ENABLED")), 

341 suppress_thinking=_bool( 

342 _p(name, "SUPPRESS_THINKING"), default=False 

343 ), 

344 ) 

345 ) 

346 

347 # Azure OpenAI (key-auth + extras) 

348 azure_key = _secret(_p("AZURE", "API_KEY")) 

349 azure_base = _str(_p("AZURE", "BASE_URL")) 

350 if azure_key and azure_base: 

351 providers.append( 

352 ProviderConfig( 

353 name="azure", 

354 model=_str(_p("AZURE", "MODEL"), "gpt-4o"), 

355 api_key=azure_key, 

356 base_url=azure_base, 

357 timeout=_int(_p("AZURE", "TIMEOUT"), 30), 

358 enabled=_bool(_p("AZURE", "ENABLED")), 

359 extras={ 

360 k: v 

361 for k, v in { 

362 "azure_resource": _extras("AZURE", "AZURE_RESOURCE"), 

363 "azure_deployment": _extras("AZURE", "AZURE_DEPLOYMENT"), 

364 "azure_api_version": _extras("AZURE", "AZURE_API_VERSION"), 

365 }.items() 

366 if v 

367 }, 

368 ) 

369 ) 

370 

371 # Cloudflare Workers AI 

372 cf_account = _extras("CLOUDFLARE", "CF_ACCOUNT_ID") 

373 cf_token = _extras("CLOUDFLARE", "CF_API_TOKEN") 

374 if cf_account and cf_token: 

375 providers.append( 

376 ProviderConfig( 

377 name="cloudflare", 

378 model=_str( 

379 _p("CLOUDFLARE", "MODEL"), "@cf/meta/llama-3.1-8b-instruct" 

380 ), 

381 timeout=_int(_p("CLOUDFLARE", "TIMEOUT"), 30), 

382 enabled=_bool(_p("CLOUDFLARE", "ENABLED")), 

383 extras={"cf_account_id": cf_account, "cf_api_token": cf_token}, 

384 ) 

385 ) 

386 

387 # AWS Bedrock 

388 bedrock_region = _extras("BEDROCK", "AWS_REGION") 

389 if bedrock_region: 

390 extras: dict[str, str] = {"aws_region": bedrock_region} 

391 for k in ( 

392 "AWS_ACCESS_KEY_ID", 

393 "AWS_SECRET_ACCESS_KEY", 

394 "AWS_SESSION_TOKEN", 

395 "AWS_PROFILE", 

396 ): 

397 val = _extras("BEDROCK", k) 

398 if val: 

399 extras[k.lower()] = val 

400 providers.append( 

401 ProviderConfig( 

402 name="bedrock", 

403 model=_str( 

404 _p("BEDROCK", "MODEL"), "anthropic.claude-3-haiku-20240307-v1:0" 

405 ), 

406 timeout=_int(_p("BEDROCK", "TIMEOUT"), 60), 

407 enabled=_bool(_p("BEDROCK", "ENABLED")), 

408 extras=extras, 

409 ) 

410 ) 

411 

412 # Google Vertex AI 

413 vertex_project = _extras("VERTEX", "VERTEX_PROJECT") 

414 if vertex_project: 

415 vextras: dict[str, str] = {"vertex_project": vertex_project} 

416 for k in ("VERTEX_LOCATION", "VERTEX_CREDENTIALS_FILE"): 

417 val = _extras("VERTEX", k) 

418 if val: 

419 vextras[k.lower()] = val 

420 providers.append( 

421 ProviderConfig( 

422 name="vertex", 

423 model=_str(_p("VERTEX", "MODEL"), "gemini-1.5-pro"), 

424 timeout=_int(_p("VERTEX", "TIMEOUT"), 60), 

425 enabled=_bool(_p("VERTEX", "ENABLED")), 

426 extras=vextras, 

427 ) 

428 ) 

429 

430 # Ollama 

431 ollama_base = _str(_p("OLLAMA", "BASE_URL")) 

432 if ollama_base: 

433 providers.append( 

434 ProviderConfig( 

435 name="ollama", 

436 model=_str(_p("OLLAMA", "MODEL"), "llama3.2"), 

437 base_url=ollama_base, 

438 timeout=_int(_p("OLLAMA", "TIMEOUT"), 120), 

439 enabled=_bool(_p("OLLAMA", "ENABLED")), 

440 suppress_thinking=_bool( 

441 _p("OLLAMA", "SUPPRESS_THINKING"), default=False 

442 ), 

443 ) 

444 ) 

445 

446 # Generic OpenAI-compatible server (LM Studio, vLLM, etc.) 

447 openai_compatible_base = _str(_p("OPENAI_COMPATIBLE", "BASE_URL")) 

448 openai_compatible_model = _str(_p("OPENAI_COMPATIBLE", "MODEL")) 

449 if openai_compatible_base and openai_compatible_model: 

450 providers.append( 

451 ProviderConfig( 

452 name="openai_compatible", 

453 model=openai_compatible_model, 

454 api_key=_secret(_p("OPENAI_COMPATIBLE", "API_KEY")), 

455 base_url=openai_compatible_base, 

456 timeout=_int(_p("OPENAI_COMPATIBLE", "TIMEOUT"), 120), 

457 enabled=_bool(_p("OPENAI_COMPATIBLE", "ENABLED")), 

458 suppress_thinking=_bool( 

459 _p("OPENAI_COMPATIBLE", "SUPPRESS_THINKING"), default=False 

460 ), 

461 ) 

462 ) 

463 

464 # Global settings 

465 temperature_raw = _str(f"{_ENV}DEFAULTS__TEMPERATURE", "0.2") 

466 max_tokens_raw = _str(f"{_ENV}DEFAULTS__MAX_TOKENS") 

467 

468 return cls( 

469 providers=providers, 

470 defaults=GenerationDefaults( 

471 temperature=float(temperature_raw), 

472 max_tokens=int(max_tokens_raw) if max_tokens_raw else None, 

473 ), 

474 quota=QuotaConfig( 

475 backend=_str(f"{_ENV}QUOTA__BACKEND", "memory"), 

476 ), 

477 logging=LogConfig( 

478 backend=_str(f"{_ENV}LOG__BACKEND", "memory"), 

479 max_entries=_int(f"{_ENV}LOG__MAX_ENTRIES", 1000), 

480 ), 

481 strategy=_str(f"{_ENV}STRATEGY", "sequential") or "sequential", 

482 )