Coverage for /home/admin/Documents/AI/applications/lexigram-dev/lexigram/experimental/ai/lexigram-ai-rag/src/lexigram/ai/rag/evaluation/types.py: 98%

1"""Types and data structures for RAG evaluation.""" 

2 

3from __future__ import annotations 

4 

5from dataclasses import dataclass, field 

6from datetime import UTC, datetime 

7from enum import Enum 

8from typing import Any 

9 

10# Type aliases for clarity 

11LLMClientProtocol = Any # Will be protocol for LLM client 

12EmbeddingClientProtocol = Any # Will be protocol for embedding client 

13 

14 

15class MetricType(str, Enum): 

16 """Types of evaluation metrics.""" 

17 

18 # Retrieval metrics 

19 RETRIEVAL_PRECISION = "retrieval_precision" 

20 RETRIEVAL_RECALL = "retrieval_recall" 

21 RETRIEVAL_F1 = "retrieval_f1" 

22 RETRIEVAL_MRR = "retrieval_mrr" # Mean Reciprocal Rank 

23 RETRIEVAL_NDCG = "retrieval_ndcg" # Normalized Discounted Cumulative Gain 

24 

25 # Answer quality metrics 

26 ANSWER_RELEVANCE = "answer_relevance" 

27 ANSWER_FAITHFULNESS = "answer_faithfulness" 

28 ANSWER_COHERENCE = "answer_coherence" 

29 ANSWER_COMPLETENESS = "answer_completeness" 

30 

31 # Context metrics 

32 CONTEXT_RELEVANCE = "context_relevance" 

33 CONTEXT_PRECISION = "context_precision" 

34 CONTEXT_RECALL = "context_recall" 

35 

36 # Overall metrics 

37 HALLUCINATION_RATE = "hallucination_rate" 

38 LATENCY = "latency" 

39 TOKEN_USAGE = "token_usage" # noqa: S105 # metric name, not a credential 

40 COST = "cost" 

41 

42 

43@dataclass 

44class EvaluationResult: 

45 """Result of a single metric evaluation. 

46 

47 Attributes: 

48 metric_type: Type of metric evaluated. 

49 score: Numerical score (0.0 to 1.0). 

50 details: Additional details about the evaluation. 

51 timestamp: When the evaluation was performed. 

52 """ 

53 

54 metric_type: MetricType 

55 score: float 

56 details: dict[str, Any] = field(default_factory=dict) 

57 timestamp: datetime = field(default_factory=lambda: datetime.now(UTC)) 

58 

59 def __repr__(self) -> str: 

60 """String representation.""" 

61 return f"EvaluationResult({self.metric_type.value}={self.score:.3f})" 

62 

63 

64@dataclass 

65class RAGEvaluationReport: 

66 """Complete evaluation report for a RAG system. 

67 

68 Attributes: 

69 query: The original query. 

70 retrieved_docs: Retrieved document IDs or content. 

71 generated_answer: Generated answer. 

72 reference_answer: Optional reference/ground truth answer. 

73 results: Individual metric results. 

74 overall_score: Aggregated overall score. 

75 metadata: Additional metadata. 

76 timestamp: When the evaluation was performed. 

77 """ 

78 

79 query: str 

80 retrieved_docs: list[Any] 

81 generated_answer: str 

82 reference_answer: str | None = None 

83 results: list[EvaluationResult] = field(default_factory=list) 

84 overall_score: float = 0.0 

85 metadata: dict[str, Any] = field(default_factory=dict) 

86 timestamp: datetime = field(default_factory=lambda: datetime.now(UTC)) 

87 

88 def get_metric(self, metric_type: MetricType) -> EvaluationResult | None: 

89 """Get specific metric result.""" 

90 for result in self.results: 

91 if result.metric_type == metric_type: 

92 return result 

93 return None 

94 

95 def get_score(self, metric_type: MetricType) -> float | None: 

96 """Get score for specific metric.""" 

97 result = self.get_metric(metric_type) 

98 return result.score if result else None 

99 

100 def __repr__(self) -> str: 

101 """String representation.""" 

102 return f"RAGEvaluationReport(overall={self.overall_score:.3f}, metrics={len(self.results)})"