Coverage for /home/admin/Documents/AI/applications/lexigram-dev/lexigram/experimental/ai/lexigram-ai-rag/src/lexigram/ai/rag/evaluation/hallucination.py: 88%

1"""Hallucination detection evaluation metrics.""" 

2 

3from __future__ import annotations 

4 

5from typing import TYPE_CHECKING, Any 

6 

7from lexigram.ai.rag.evaluation.base import EvaluatorBase 

8from lexigram.ai.rag.evaluation.types import EvaluationResult, MetricType 

9from lexigram.contracts import ChatMessage 

10from lexigram.di.decorators import inject 

11 

12if TYPE_CHECKING: 

13 from lexigram.contracts.ai import LLMClientProtocol 

14 

15 

16@inject 

17class HallucinationDetector(EvaluatorBase): 

18 """Detects hallucinations in generated answers. 

19 

20 Uses LLM to identify claims in the answer and verify them against context. 

21 """ 

22 

23 def __init__(self, llm_client: LLMClientProtocol): 

24 """Initialize hallucination detector. 

25 

26 Args: 

27 llm_client: LLM client for detection. 

28 """ 

29 super().__init__("hallucination_detector") 

30 self.llm_client = llm_client 

31 

32 async def evaluate( 

33 self, 

34 query: str, 

35 retrieved_docs: list[Any], 

36 generated_answer: str, 

37 reference_answer: str | None = None, 

38 **kwargs, 

39 ) -> EvaluationResult: 

40 """Detect hallucinations. 

41 

42 Args: 

43 query: The query. 

44 retrieved_docs: Retrieved documents for verification. 

45 generated_answer: Generated answer to check. 

46 reference_answer: Not used. 

47 **kwargs: Additional parameters. 

48 

49 Returns: 

50 Hallucination rate (0.0 = no hallucinations, 1.0 = all hallucinated). 

51 """ 

52 # Build context 

53 context_parts = [] 

54 for doc in retrieved_docs: 

55 if isinstance(doc, dict): 

56 content = doc.get("content", str(doc)) 

57 elif hasattr(doc, "content"): 

58 content = doc.content 

59 else: 

60 content = str(doc) 

61 context_parts.append(content) 

62 

63 context_str = "\n".join(context_parts) 

64 

65 prompt = f"""Analyze the answer and identify any claims that are NOT supported by the context. 

66 

67Context: 

68{context_str} 

69 

70Answer: {generated_answer} 

71 

72For each claim in the answer, check if it's supported by the context. 

73Provide a hallucination rate from 0.0 (all claims supported) to 1.0 (all claims unsupported). 

74Provide only a number between 0.0 and 1.0 as your response.""" 

75 

76 try: 

77 result = await self.llm_client.complete( 

78 messages=[ChatMessage(role="user", content=prompt)] 

79 ) 

80 if result.is_err(): 

81 raise result.unwrap_err() 

82 response = result.unwrap() 

83 score_text = response.content.strip() 

84 hallucination_rate = float(score_text) 

85 hallucination_rate = max(0.0, min(1.0, hallucination_rate)) 

86 

87 return EvaluationResult( 

88 metric_type=MetricType.HALLUCINATION_RATE, 

89 score=hallucination_rate, 

90 details={"llm_response": score_text}, 

91 ) 

92 except (ConnectionError, TimeoutError, RuntimeError, ValueError, OSError) as e: 

93 return EvaluationResult( 

94 metric_type=MetricType.HALLUCINATION_RATE, 

95 score=1.0, # Assume worst case on error 

96 details={"error": str(e)}, 

97 )