Coverage for /home/admin/Documents/AI/applications/lexigram-dev/lexigram/experimental/ai/lexigram-ai-rag/src/lexigram/ai/rag/evaluation/context.py: 82%

1"""Context relevance evaluation metrics.""" 

2 

3from __future__ import annotations 

4 

5from typing import TYPE_CHECKING, Any 

6 

7from lexigram.ai.rag.evaluation.base import EvaluatorBase 

8from lexigram.ai.rag.evaluation.types import EvaluationResult, MetricType 

9from lexigram.contracts import ChatMessage 

10from lexigram.di.decorators import inject 

11from lexigram.logging import ( 

12 get_logger, 

13) 

14 

15if TYPE_CHECKING: 

16 from lexigram.contracts.ai import LLMClientProtocol 

17 

18logger = get_logger(__name__) 

19 

20 

21@inject 

22class ContextRelevanceEvaluator(EvaluatorBase): 

23 """Evaluates context relevance using LLM-as-judge. 

24 

25 Measures how relevant the retrieved context is to the query. 

26 """ 

27 

28 def __init__(self, llm_client: LLMClientProtocol): 

29 """Initialize context relevance evaluator. 

30 

31 Args: 

32 llm_client: LLM client for evaluation. 

33 """ 

34 super().__init__("context_relevance") 

35 self.llm_client = llm_client 

36 

37 async def evaluate( 

38 self, 

39 query: str, 

40 retrieved_docs: list[Any], 

41 generated_answer: str, 

42 reference_answer: str | None = None, 

43 **kwargs, 

44 ) -> EvaluationResult: 

45 """Evaluate context relevance. 

46 

47 Args: 

48 query: The query. 

49 retrieved_docs: Retrieved documents to evaluate. 

50 generated_answer: Generated answer (not used). 

51 reference_answer: Not used. 

52 **kwargs: Additional parameters. 

53 

54 Returns: 

55 Context relevance score. 

56 """ 

57 if not retrieved_docs: 

58 return EvaluationResult( 

59 metric_type=MetricType.CONTEXT_RELEVANCE, 

60 score=0.0, 

61 details={"reason": "No documents retrieved"}, 

62 ) 

63 

64 # Build context 

65 context_parts = [] 

66 for i, doc in enumerate(retrieved_docs): 

67 if isinstance(doc, dict): 

68 content = doc.get("content", str(doc)) 

69 elif hasattr(doc, "content"): 

70 content = doc.content 

71 else: 

72 content = str(doc) 

73 context_parts.append(f"[{i + 1}] {content}") 

74 

75 context_str = "\n".join(context_parts) 

76 

77 prompt = f"""Evaluate how relevant the retrieved context is to answering the query. 

78Score from 0.0 (completely irrelevant) to 1.0 (perfectly relevant). 

79 

80Query: {query} 

81 

82Retrieved Context: 

83{context_str} 

84 

85Provide only a number between 0.0 and 1.0 as your response.""" 

86 

87 try: 

88 result = await self.llm_client.complete( 

89 messages=[ChatMessage(role="user", content=prompt)] 

90 ) 

91 if result.is_err(): 

92 raise result.unwrap_err() 

93 response = result.unwrap() 

94 score_text = response.content.strip() 

95 score = float(score_text) 

96 score = max(0.0, min(1.0, score)) 

97 

98 return EvaluationResult( 

99 metric_type=MetricType.CONTEXT_RELEVANCE, 

100 score=score, 

101 details={ 

102 "llm_response": score_text, 

103 "context_chunks": len(retrieved_docs), 

104 }, 

105 ) 

106 except Exception as e: 

107 logger.exception("Error computing evaluation metric") 

108 return EvaluationResult( 

109 metric_type=MetricType.CONTEXT_RELEVANCE, 

110 score=0.0, 

111 details={"error": str(e)}, 

112 )