Coverage for /home/admin/Documents/AI/applications/lexigram-dev/lexigram/experimental/ai/lexigram-ai-rag/src/lexigram/ai/rag/config.py: 98%

1from __future__ import annotations 

2 

3from dataclasses import dataclass 

4from enum import StrEnum 

5from typing import Any, ClassVar, cast 

6 

7from lexigram.ai.rag.constants import ENV_NESTED_DELIMITER, ENV_PREFIX 

8from lexigram.ai.rag.synthesis.types import SynthesisConfig 

9from lexigram.config.base import BaseConfig 

10from lexigram.domain.models import DomainModel 

11from lexigram.validation import ConfigDict, Field 

12 

13 

14@dataclass(init=False) 

15class RAGTenancyConfig(DomainModel): 

16 """Optional tenant-aware RAG pipeline configuration. 

17 

18 When enabled, the RAG provider wraps the ``RAGPipelineProtocol`` binding 

19 in a ``TenantScopedRAGPipeline`` that resolves the ``collection_name`` 

20 from the current tenant context at request time, with per-tenant 

21 pipeline instance caching. 

22 

23 Note: 

24 Requires ``lexigram-tenancy`` in the module graph when ``enabled`` 

25 is ``True`` — the provider resolves ``Context`` at boot. 

26 """ 

27 

28 enabled: bool = Field( 

29 default=False, 

30 description="Enable tenant-aware collection resolution in RAG pipeline", 

31 ) 

32 

33 

34class RAGConfig(BaseConfig): 

35 """Configuration for RAG (Retrieval Augmented Generation) pipeline. 

36 

37 Example: 

38 >>> config = RAGConfig( 

39 ... vector_store_type="chroma", 

40 ... collection_name="pet_knowledge", 

41 ... top_k=5, 

42 ... enable_citations=True 

43 ... ) 

44 """ 

45 

46 config_section: ClassVar[str] = "ai_rag" 

47 

48 def with_collection(self, name: str) -> RAGConfig: 

49 """Return a copy of this config with a different *collection_name*. 

50 

51 Usage:: 

52 

53 tenant_config = base_config.with_collection("canon_t_tenant42") 

54 """ 

55 return self.model_copy(update={"collection_name": name}) 

56 

57 model_config: ClassVar[ConfigDict] = cast( 

58 "ConfigDict", 

59 { 

60 "env_prefix": ENV_PREFIX, 

61 "env_nested_delimiter": ENV_NESTED_DELIMITER, 

62 "extra": "ignore", 

63 }, 

64 ) 

65 

66 # Feature toggle 

67 enabled: bool = Field( 

68 default=True, 

69 description="Enable the RAG pipeline", 

70 ) 

71 

72 # Local persistence (for Chroma and similar file-based stores) 

73 persist_directory: str | None = Field( 

74 default=None, 

75 description="Local directory path for vector store persistence (e.g. Chroma)", 

76 ) 

77 

78 # Vector Store 

79 vector_store_type: str = Field( 

80 default="pgvector", 

81 description="Vector store backend (pgvector, chroma, qdrant, mock)", 

82 ) 

83 vector_dimension: int = Field( 

84 default=1536, 

85 ge=1, 

86 description="Embedding vector dimension (1536 for OpenAI ada-002)", 

87 ) 

88 collection_name: str = Field( 

89 default="default", 

90 description="Collection/index name for vector store", 

91 ) 

92 

93 # Retrieval 

94 top_k: int = Field( 

95 default=5, 

96 ge=1, 

97 description="Number of documents to retrieve", 

98 ) 

99 similarity_threshold: float = Field( 

100 default=0.7, 

101 ge=0.0, 

102 le=1.0, 

103 description="Minimum similarity score threshold", 

104 ) 

105 use_hybrid_search: bool = Field( 

106 default=True, 

107 description="Enable hybrid search (semantic + keyword)", 

108 ) 

109 

110 # Embedding Model 

111 embedding_provider: str = Field( 

112 default="openai", 

113 description="Embedding provider (openai, cohere, etc.)", 

114 ) 

115 embedding_model: str | None = Field( 

116 default=None, 

117 description="Embedding model identifier. Must be set explicitly — no vendor-specific default.", 

118 ) 

119 

120 # Citations 

121 enable_citations: bool = Field( 

122 default=True, 

123 description="Include source citations in responses", 

124 ) 

125 citation_style: str = Field( 

126 default="inline", 

127 description="Citation style (inline, footnote, numbered)", 

128 ) 

129 min_citation_confidence: float = Field( 

130 default=0.6, 

131 ge=0.0, 

132 le=1.0, 

133 description="Minimum confidence for citation inclusion", 

134 ) 

135 

136 # Chunking 

137 chunk_size: int = Field( 

138 default=512, 

139 ge=1, 

140 description="Text chunk size in tokens", 

141 ) 

142 chunk_overlap: int = Field( 

143 default=50, 

144 ge=0, 

145 description="Overlap between consecutive chunks", 

146 ) 

147 chunking_strategy: str = Field( 

148 default="recursive", 

149 description="Chunking strategy (recursive, semantic, token)", 

150 ) 

151 

152 # Query Enhancement 

153 enable_query_expansion: bool = Field( 

154 default=True, 

155 description="Enable query expansion techniques", 

156 ) 

157 enable_hyde: bool = Field( 

158 default=False, 

159 description="Enable HyDE (Hypothetical Document Embeddings)", 

160 ) 

161 

162 # Response Synthesis 

163 synthesis_strategy: str = Field( 

164 default="hybrid", 

165 description="Synthesis strategy (direct, extractive, abstractive, hybrid)", 

166 ) 

167 enable_hallucination_detection: bool = Field( 

168 default=True, 

169 description="Enable hallucination detection for AI responses", 

170 ) 

171 

172 # Cache 

173 enable_caching: bool = Field( 

174 default=True, 

175 description="Enable caching for RAG queries", 

176 ) 

177 cache_ttl: int = Field( 

178 default=3600, 

179 ge=0, 

180 description="Cache TTL in seconds (default: 1 hour)", 

181 ) 

182 

183 # Tenancy 

184 tenancy: RAGTenancyConfig = Field( 

185 default_factory=RAGTenancyConfig, 

186 description="Optional tenant-aware RAG pipeline configuration", 

187 ) 

188 

189 

190class PipelineStageType(StrEnum): 

191 """Types of stages in a RAG pipeline.""" 

192 

193 INGESTION = "ingestion" 

194 QUERY_PROCESSING = "query_processing" 

195 RETRIEVAL = "retrieval" 

196 CONTEXT_OPTIMIZATION = "context_optimization" 

197 SYNTHESIS = "synthesis" 

198 QUALITY_ASSURANCE = "quality_assurance" 

199 POST_PROCESSING = "post_processing" 

200 

201 

202class DocumentFormat(StrEnum): 

203 """Supported document formats for ingestion.""" 

204 

205 TEXT = "text" 

206 PDF = "pdf" 

207 MARKDOWN = "markdown" 

208 HTML = "html" 

209 

210 

211class RoutingStrategyType(StrEnum): 

212 """Routing strategy types.""" 

213 

214 RULE_BASED = "rule_based" 

215 SEMANTIC = "semantic" 

216 LLM = "llm" 

217 HYBRID = "hybrid" 

218 

219 

220class IngestionConfig(BaseConfig): 

221 """Configuration for document ingestion stage.""" 

222 

223 model_config: ClassVar[ConfigDict] = ConfigDict(extra="ignore") 

224 

225 enabled: bool = Field(default=True) 

226 document_formats: list[DocumentFormat] = Field( 

227 default_factory=lambda: [ 

228 DocumentFormat.TEXT, 

229 DocumentFormat.PDF, 

230 DocumentFormat.MARKDOWN, 

231 ], 

232 ) 

233 preprocessing_enabled: bool = Field(default=False) 

234 ocr_enabled: bool = Field(default=False) 

235 table_extraction_enabled: bool = Field(default=False) 

236 metadata_enrichment_enabled: bool = Field(default=False) 

237 chunking_strategy: str = Field(default="recursive") 

238 chunk_size: int = Field(default=1000) 

239 chunk_overlap: int = Field(default=200) 

240 min_chunk_size: int = Field(default=100) 

241 error_strategy: str = Field(default="graceful") 

242 fail_fast: bool = Field(default=False) 

243 

244 

245class QueryProcessingConfig(BaseConfig): 

246 """Configuration for query processing stage.""" 

247 

248 model_config: ClassVar[ConfigDict] = ConfigDict(extra="ignore") 

249 

250 enabled: bool = Field(default=True) 

251 transformation_enabled: bool = Field(default=False) 

252 transformation_strategies: list[str] = Field(default_factory=lambda: ["expansion"]) 

253 hyde_enabled: bool = Field(default=False) 

254 hyde_num_documents: int = Field(default=1) 

255 routing_enabled: bool = Field(default=False) 

256 routing_strategy: RoutingStrategyType = Field( 

257 default=RoutingStrategyType.RULE_BASED 

258 ) 

259 error_strategy: str = Field(default="graceful") 

260 

261 

262class RetrievalConfig(BaseConfig): 

263 """Configuration for retrieval stage.""" 

264 

265 model_config: ClassVar[ConfigDict] = ConfigDict(extra="ignore") 

266 

267 enabled: bool = Field(default=True) 

268 strategy: str = Field(default="hybrid") 

269 vector_search_enabled: bool = Field(default=True) 

270 top_k: int = Field(default=10) 

271 similarity_threshold: float = Field(default=0.0) 

272 knowledge_graph_enabled: bool = Field(default=False) 

273 max_graph_depth: int = Field(default=2) 

274 multi_hop_enabled: bool = Field(default=False) 

275 max_hops: int = Field(default=3) 

276 error_strategy: str = Field(default="fail_fast") 

277 

278 

279class ContextOptimizationConfig(BaseConfig): 

280 """Configuration for context optimization stage.""" 

281 

282 model_config: ClassVar[ConfigDict] = ConfigDict(extra="ignore") 

283 

284 enabled: bool = Field(default=True) 

285 strategy: str = Field(default="cross-encoder") 

286 top_k: int = Field(default=5) 

287 ranking_enabled: bool = Field(default=True) 

288 compression_enabled: bool = Field(default=False) 

289 compression_strategy: str = Field(default="hybrid") 

290 max_context_tokens: int = Field(default=4000) 

291 deduplication_enabled: bool = Field(default=True) 

292 deduplication_threshold: float = Field(default=0.9) 

293 citations_enabled: bool = Field(default=True) 

294 error_strategy: str = Field(default="graceful") 

295 

296 

297class QualityAssuranceConfig(BaseConfig): 

298 """Configuration for quality assurance stage.""" 

299 

300 model_config: ClassVar[ConfigDict] = ConfigDict(extra="ignore") 

301 

302 enabled: bool = Field(default=True) 

303 min_faithfulness: float = Field(default=0.7) 

304 min_relevance: float = Field(default=0.6) 

305 min_confidence: float = Field(default=0.5) 

306 hallucination_detection_enabled: bool = Field(default=True) 

307 hallucination_strict_mode: bool = Field(default=False) 

308 reject_low_quality: bool = Field(default=False) 

309 warn_low_quality: bool = Field(default=True) 

310 error_strategy: str = Field(default="graceful") 

311 

312 

313class PostProcessingConfig(BaseConfig): 

314 """Configuration for post-processing stage.""" 

315 

316 model_config: ClassVar[ConfigDict] = ConfigDict(extra="ignore") 

317 

318 enabled: bool = Field(default=True) 

319 cache_enabled: bool = Field(default=True) 

320 cache_results: bool = Field(default=True) 

321 collect_metrics: bool = Field(default=True) 

322 detailed_logging: bool = Field(default=False) 

323 error_strategy: str = Field(default="skip") 

324 

325 

326class PipelineConfig(BaseConfig): 

327 """Complete pipeline configuration.""" 

328 

329 model_config: ClassVar[ConfigDict] = ConfigDict(extra="ignore") 

330 

331 name: str = Field(default="default-rag-pipeline") 

332 description: str = Field(default="") 

333 ingestion: IngestionConfig = Field(default_factory=IngestionConfig) 

334 query_processing: QueryProcessingConfig = Field( 

335 default_factory=QueryProcessingConfig 

336 ) 

337 retrieval: RetrievalConfig = Field(default_factory=RetrievalConfig) 

338 context_optimization: ContextOptimizationConfig = Field( 

339 default_factory=ContextOptimizationConfig 

340 ) 

341 synthesis: SynthesisConfig = Field(default_factory=SynthesisConfig) 

342 quality_assurance: QualityAssuranceConfig = Field( 

343 default_factory=QualityAssuranceConfig 

344 ) 

345 post_processing: PostProcessingConfig = Field(default_factory=PostProcessingConfig) 

346 

347 stages: list[PipelineStageType] = Field( 

348 default_factory=lambda: [ 

349 PipelineStageType.RETRIEVAL, 

350 PipelineStageType.SYNTHESIS, 

351 PipelineStageType.QUALITY_ASSURANCE, 

352 ], 

353 description="Ordered list of pipeline stages to execute", 

354 ) 

355 

356 max_retries: int = Field(default=3) 

357 retry_delay: float = Field(default=1.0) 

358 default_error_strategy: str = Field(default="graceful") 

359 metadata: dict[str, Any] = Field(default_factory=dict) 

360 

361 # Auto-evaluation hook (P7.1) 

362 auto_evaluate_every_n: int | None = Field( 

363 default=None, 

364 description="Run automatic evaluation every N pipeline requests. None disables auto-evaluation.", 

365 ) 

366 

367 # Citation enforcement (P7.2) 

368 require_citations: bool = Field( 

369 default=False, 

370 description="Raise MissingCitationsError when the synthesis result contains no citations.", 

371 ) 

372 

373 @classmethod 

374 def from_dict(cls, config_dict: dict[str, Any]) -> PipelineConfig: 

375 """Create configuration from dictionary.""" 

376 return cls(**config_dict) 

377 

378 def to_dict(self) -> dict[str, Any]: 

379 """Convert configuration to dictionary.""" 

380 return { 

381 "name": self.name, 

382 "description": self.description, 

383 "max_retries": self.max_retries, 

384 "retry_delay": self.retry_delay, 

385 "default_error_strategy": self.default_error_strategy, 

386 "metadata": self.metadata, 

387 } 

388 

389 

390__all__ = [ 

391 "ContextOptimizationConfig", 

392 "DocumentFormat", 

393 "IngestionConfig", 

394 "PipelineConfig", 

395 "PipelineStageType", 

396 "PostProcessingConfig", 

397 "QualityAssuranceConfig", 

398 "QueryProcessingConfig", 

399 "RAGConfig", 

400 "RAGTenancyConfig", 

401 "RetrievalConfig", 

402 "RoutingStrategyType", 

403 "SynthesisConfig", 

404]