Coverage for /home/admin/Documents/AI/applications/lexigram-dev/lexigram/experimental/ai/lexigram-ai-rag/src/lexigram/ai/rag/evaluation/answer.py: 85%

1"""Answer quality evaluation metrics.""" 

2 

3from __future__ import annotations 

4 

5from typing import TYPE_CHECKING, Any 

6 

7from lexigram.ai.rag.evaluation.base import EvaluatorBase 

8from lexigram.ai.rag.evaluation.types import EvaluationResult, MetricType 

9from lexigram.contracts import ChatMessage 

10from lexigram.di.decorators import inject 

11from lexigram.logging import ( 

12 get_logger, 

13) 

14 

15if TYPE_CHECKING: 

16 from lexigram.contracts.ai import LLMClientProtocol 

17 

18logger = get_logger(__name__) 

19 

20 

21@inject 

22class AnswerRelevanceEvaluator(EvaluatorBase): 

23 """Evaluates answer relevance using LLM-as-judge. 

24 

25 Measures how well the answer addresses the query. 

26 """ 

27 

28 def __init__(self, llm_client: LLMClientProtocol): 

29 """Initialize answer relevance evaluator. 

30 

31 Args: 

32 llm_client: LLM client for evaluation. 

33 """ 

34 super().__init__("answer_relevance") 

35 self.llm_client = llm_client 

36 

37 async def evaluate( 

38 self, 

39 query: str, 

40 retrieved_docs: list[Any], 

41 generated_answer: str, 

42 reference_answer: str | None = None, 

43 **kwargs, 

44 ) -> EvaluationResult: 

45 """Evaluate answer relevance. 

46 

47 Args: 

48 query: The query. 

49 retrieved_docs: Retrieved documents (not used). 

50 generated_answer: Generated answer to evaluate. 

51 reference_answer: Not used. 

52 **kwargs: Additional parameters. 

53 

54 Returns: 

55 Relevance score. 

56 """ 

57 prompt = f"""Evaluate how relevant the following answer is to the query. 

58Score from 0.0 (completely irrelevant) to 1.0 (perfectly relevant). 

59 

60Query: {query} 

61 

62Answer: {generated_answer} 

63 

64Provide only a number between 0.0 and 1.0 as your response.""" 

65 

66 try: 

67 result = await self.llm_client.complete( 

68 messages=[ChatMessage(role="user", content=prompt)] 

69 ) 

70 if result.is_err(): 

71 raise result.unwrap_err() 

72 response = result.unwrap() 

73 # Extract score from response 

74 score_text = response.content.strip() 

75 score = float(score_text) 

76 score = max(0.0, min(1.0, score)) # Clamp to [0, 1] 

77 

78 return EvaluationResult( 

79 metric_type=MetricType.ANSWER_RELEVANCE, 

80 score=score, 

81 details={"llm_response": score_text}, 

82 ) 

83 except Exception as e: 

84 logger.exception("Error computing evaluation metric") 

85 return EvaluationResult( 

86 metric_type=MetricType.ANSWER_RELEVANCE, 

87 score=0.0, 

88 details={"error": str(e)}, 

89 ) 

90 

91 

92@inject 

93class AnswerFaithfulnessEvaluator(EvaluatorBase): 

94 """Evaluates answer faithfulness using LLM-as-judge. 

95 

96 Measures whether the answer is grounded in the retrieved context 

97 (i.e., not hallucinated). 

98 """ 

99 

100 def __init__(self, llm_client: LLMClientProtocol): 

101 """Initialize answer faithfulness evaluator. 

102 

103 Args: 

104 llm_client: LLM client for evaluation. 

105 """ 

106 super().__init__("answer_faithfulness") 

107 self.llm_client = llm_client 

108 

109 async def evaluate( 

110 self, 

111 query: str, 

112 retrieved_docs: list[Any], 

113 generated_answer: str, 

114 reference_answer: str | None = None, 

115 **kwargs, 

116 ) -> EvaluationResult: 

117 """Evaluate answer faithfulness. 

118 

119 Args: 

120 query: The query. 

121 retrieved_docs: Retrieved documents to check against. 

122 generated_answer: Generated answer to evaluate. 

123 reference_answer: Not used. 

124 **kwargs: Additional parameters. 

125 

126 Returns: 

127 Faithfulness score. 

128 """ 

129 # Build context from retrieved docs 

130 context_parts = [] 

131 for i, doc in enumerate(retrieved_docs): 

132 if isinstance(doc, dict): 

133 content = doc.get("content", str(doc)) 

134 elif hasattr(doc, "content"): 

135 content = doc.content 

136 else: 

137 content = str(doc) 

138 context_parts.append(f"[{i + 1}] {content}") 

139 

140 context_str = "\n".join(context_parts) 

141 

142 prompt = f"""Evaluate whether the answer is faithful to the provided context. 

143Score from 0.0 (completely unfaithful/hallucinated) to 1.0 (perfectly faithful). 

144 

145Context: 

146{context_str} 

147 

148Answer: {generated_answer} 

149 

150The answer should only contain information that can be verified from the context. 

151Provide only a number between 0.0 and 1.0 as your response.""" 

152 

153 try: 

154 result = await self.llm_client.complete( 

155 messages=[ChatMessage(role="user", content=prompt)] 

156 ) 

157 if result.is_err(): 

158 raise result.unwrap_err() 

159 response = result.unwrap() 

160 score_text = response.content.strip() 

161 score = float(score_text) 

162 score = max(0.0, min(1.0, score)) 

163 

164 return EvaluationResult( 

165 metric_type=MetricType.ANSWER_FAITHFULNESS, 

166 score=score, 

167 details={ 

168 "llm_response": score_text, 

169 "context_chunks": len(retrieved_docs), 

170 }, 

171 ) 

172 except Exception as e: 

173 logger.exception("Error computing evaluation metric") 

174 return EvaluationResult( 

175 metric_type=MetricType.ANSWER_FAITHFULNESS, 

176 score=0.0, 

177 details={"error": str(e)}, 

178 )