Coverage for /home/admin/Documents/AI/applications/lexigram-dev/lexigram/experimental/ai/lexigram-ai-rag/src/lexigram/ai/rag/synthesis/quality/confidence.py: 22%

1"""Confidence scorer for response quality. 

2 

3This module implements confidence scoring that combines multiple quality 

4metrics into an overall confidence score. 

5""" 

6 

7from __future__ import annotations 

8 

9from lexigram.ai.rag.synthesis.quality.faithfulness import FaithfulnessChecker 

10from lexigram.ai.rag.synthesis.quality.hallucination import ( 

11 HallucinationChecker, 

12) 

13from lexigram.ai.rag.synthesis.quality.relevance import RelevanceFilter 

14from lexigram.ai.rag.synthesis.types import ContextChunk, QualityMetrics 

15 

16 

17class ConfidenceScorer: 

18 """Calculate overall confidence score for responses. 

19 

20 This component combines faithfulness, relevance, and hallucination 

21 detection into a single confidence score. 

22 

23 Attributes: 

24 faithfulness_checker: Component for faithfulness checking 

25 relevance_filter: Component for relevance filtering 

26 hallucination_detector: Component for hallucination detection 

27 weights: Weights for combining scores 

28 """ 

29 

30 def __init__( 

31 self, 

32 faithfulness_checker: FaithfulnessChecker | None = None, 

33 relevance_filter: RelevanceFilter | None = None, 

34 hallucination_detector: HallucinationChecker | None = None, 

35 faithfulness_weight: float = 0.4, 

36 relevance_weight: float = 0.4, 

37 coherence_weight: float = 0.2, 

38 ): 

39 """Initialize the confidence scorer. 

40 

41 Args: 

42 faithfulness_checker: Faithfulness checker instance 

43 relevance_filter: Relevance filter instance 

44 hallucination_detector: Hallucination detector instance 

45 faithfulness_weight: Weight for faithfulness score 

46 relevance_weight: Weight for relevance score 

47 coherence_weight: Weight for coherence score 

48 """ 

49 self.faithfulness_checker = faithfulness_checker or FaithfulnessChecker() 

50 self.relevance_filter = relevance_filter or RelevanceFilter() 

51 self.hallucination_detector = hallucination_detector or HallucinationChecker() 

52 

53 # Normalize weights 

54 total = faithfulness_weight + relevance_weight + coherence_weight 

55 self.faithfulness_weight = faithfulness_weight / total 

56 self.relevance_weight = relevance_weight / total 

57 self.coherence_weight = coherence_weight / total 

58 

59 def _calculate_coherence(self, response: str) -> float: 

60 """Calculate coherence score for response. 

61 

62 Args: 

63 response: The response text 

64 

65 Returns: 

66 Coherence score (0-1) 

67 """ 

68 if not response: 

69 return 0.0 

70 

71 # Simple heuristics for coherence 

72 score = 0.5 # Base score 

73 

74 # Check length (not too short, not too long) 

75 length = len(response) 

76 if 50 <= length <= 500: 

77 score += 0.2 

78 elif 20 <= length < 50 or 500 < length <= 1000: 

79 score += 0.1 

80 

81 # Check sentence structure 

82 import re 

83 

84 sentences = re.split(r"[.!?]+\s+", response) 

85 if 2 <= len(sentences) <= 10: 

86 score += 0.2 

87 elif 1 <= len(sentences) < 2 or 10 < len(sentences) <= 20: 

88 score += 0.1 

89 

90 # Check capitalization (proper sentences) 

91 if response[0].isupper(): 

92 score += 0.1 

93 

94 return min(1.0, score) 

95 

96 async def calculate_quality_metrics( 

97 self, 

98 query: str, 

99 response: str, 

100 context_chunks: list[ContextChunk], 

101 ) -> QualityMetrics: 

102 """Calculate comprehensive quality metrics. 

103 

104 Args: 

105 query: The original query 

106 response: The synthesized response 

107 context_chunks: The context chunks used 

108 

109 Returns: 

110 QualityMetrics with all scores 

111 """ 

112 # Calculate faithfulness 

113 faithfulness = await self.faithfulness_checker.check_faithfulness( 

114 response, 

115 context_chunks, 

116 ) 

117 

118 # Calculate relevance 

119 relevance = await self.relevance_filter.check_relevance(query, response) 

120 

121 # Calculate coherence 

122 coherence = self._calculate_coherence(response) 

123 

124 # Detect hallucinations 

125 ( 

126 _hallucinations, 

127 hall_count, 

128 ) = await self.hallucination_detector.detect_hallucinations( 

129 response, 

130 context_chunks, 

131 ) 

132 

133 # Calculate overall confidence 

134 confidence = ( 

135 self.faithfulness_weight * faithfulness 

136 + self.relevance_weight * relevance 

137 + self.coherence_weight * coherence 

138 ) 

139 

140 # Penalize for hallucinations 

141 if hall_count > 0: 

142 penalty = min(0.3 * hall_count, 0.5) 

143 confidence = max(0.0, confidence - penalty) 

144 

145 return QualityMetrics( 

146 faithfulness=faithfulness, 

147 relevance=relevance, 

148 coherence=coherence, 

149 confidence=confidence, 

150 has_hallucinations=hall_count > 0, 

151 hallucination_count=hall_count, 

152 ) 

153 

154 async def calculate_confidence( 

155 self, 

156 query: str, 

157 response: str, 

158 context_chunks: list[ContextChunk], 

159 ) -> float: 

160 """Calculate overall confidence score. 

161 

162 Args: 

163 query: The original query 

164 response: The synthesized response 

165 context_chunks: The context chunks used 

166 

167 Returns: 

168 Confidence score (0-1) 

169 """ 

170 metrics = await self.calculate_quality_metrics( 

171 query, 

172 response, 

173 context_chunks, 

174 ) 

175 return metrics.confidence