Coverage for /home/admin/Documents/AI/applications/lexigram-dev/lexigram/experimental/ai/lexigram-ai-rag/src/lexigram/ai/rag/synthesis/quality/hallucination.py: 26%

1"""Hallucination detector for response quality. 

2 

3This module implements hallucination detection to identify unsupported 

4claims in synthesized responses. 

5""" 

6 

7from __future__ import annotations 

8 

9import re 

10 

11from lexigram.ai.rag.synthesis.types import ContextChunk 

12 

13 

14class HallucinationChecker: 

15 """Detect potential hallucinations in responses. 

16 

17 This component identifies claims in responses that are not supported 

18 by the context chunks. 

19 

20 Attributes: 

21 strict_mode: Whether to use strict detection (lower threshold) 

22 min_support_ratio: Minimum keyword support ratio 

23 """ 

24 

25 def __init__( 

26 self, 

27 strict_mode: bool = False, 

28 min_support_ratio: float = 0.4, 

29 ): 

30 """Initialize the hallucination detector. 

31 

32 Args: 

33 strict_mode: Use strict detection criteria 

34 min_support_ratio: Minimum support ratio for claims 

35 """ 

36 self.strict_mode = strict_mode 

37 self.min_support_ratio = min_support_ratio if not strict_mode else 0.6 

38 

39 def _extract_claims(self, response: str) -> list[str]: 

40 """Extract factual claims from response. 

41 

42 Args: 

43 response: The response text 

44 

45 Returns: 

46 List of claims 

47 """ 

48 # Split into sentences 

49 sentences = re.split(r"[.!?]+\s+", response) 

50 return list(map(str.strip, filter(lambda s: len(s.strip()) > 10, sentences))) 

51 

52 def _check_claim_support( 

53 self, 

54 claim: str, 

55 context_text: str, 

56 ) -> tuple[bool, float]: 

57 """Check if claim is supported by context. 

58 

59 Args: 

60 claim: The claim to check 

61 context_text: The context text 

62 

63 Returns: 

64 Tuple of (is_supported, support_ratio) 

65 """ 

66 # Extract keywords from claim 

67 claim_words = set(re.findall(r"\b\w{3,}\b", claim.lower())) 

68 

69 # Remove common words 

70 stop_words = { 

71 "the", 

72 "this", 

73 "that", 

74 "these", 

75 "those", 

76 "what", 

77 "which", 

78 "who", 

79 "when", 

80 "where", 

81 "why", 

82 "how", 

83 "can", 

84 "will", 

85 "would", 

86 } 

87 claim_words -= stop_words 

88 

89 if not claim_words: 

90 return True, 1.0 

91 

92 # Check presence in context 

93 context_lower = context_text.lower() 

94 supported_words = sum(1 for word in claim_words if word in context_lower) 

95 

96 support_ratio = supported_words / len(claim_words) 

97 is_supported = support_ratio >= self.min_support_ratio 

98 

99 return is_supported, support_ratio 

100 

101 async def detect_hallucinations( 

102 self, 

103 response: str, 

104 context_chunks: list[ContextChunk], 

105 ) -> tuple[list[str], int]: 

106 """Detect potential hallucinations in response. 

107 

108 Args: 

109 response: The synthesized response 

110 context_chunks: The context chunks used 

111 

112 Returns: 

113 Tuple of (list of potential hallucinations, count) 

114 """ 

115 if not response or not context_chunks: 

116 return [], 0 

117 

118 # Combine context 

119 context_text = " ".join(chunk.text for chunk in context_chunks) 

120 

121 # Extract claims 

122 claims = self._extract_claims(response) 

123 

124 # Check each claim 

125 hallucinations = [] 

126 

127 for claim in claims: 

128 is_supported, _support_ratio = self._check_claim_support( 

129 claim, 

130 context_text, 

131 ) 

132 

133 if not is_supported: 

134 hallucinations.append(claim) 

135 

136 return hallucinations, len(hallucinations) 

137 

138 async def has_hallucinations( 

139 self, 

140 response: str, 

141 context_chunks: list[ContextChunk], 

142 ) -> bool: 

143 """Check if response has any hallucinations. 

144 

145 Args: 

146 response: The synthesized response 

147 context_chunks: The context chunks used 

148 

149 Returns: 

150 True if hallucinations detected 

151 """ 

152 _, count = await self.detect_hallucinations(response, context_chunks) 

153 return count > 0