« prev ^ index » next coverage.py v7.15.4, created at 2026-08-25 07:19 +0800
1"""Faithfulness checker for response quality.
3This module implements faithfulness checking to verify that synthesized
4responses are grounded in the provided context.
5"""
7from __future__ import annotations
9import re
10from typing import TYPE_CHECKING
12if TYPE_CHECKING:
13 from lexigram.contracts.ai import LLMClientProtocol
15from lexigram.ai.rag.synthesis.types import ContextChunk
18class FaithfulnessChecker:
19 """Check if response is faithful to context.
21 This component verifies that claims in the response are supported
22 by the context chunks.
24 Attributes:
25 use_llm: Whether to use LLM for verification (more accurate)
26 llm_client: Optional LLM client for verification
27 threshold: Faithfulness threshold (0-1)
28 """
30 def __init__(
31 self,
32 use_llm: bool = False,
33 llm_client: LLMClientProtocol | None = None,
34 threshold: float = 0.7,
35 ):
36 """Initialize the faithfulness checker.
38 Args:
39 use_llm: Whether to use LLM verification
40 llm_client: LLM client (required if use_llm=True)
41 threshold: Faithfulness threshold
42 """
43 self.use_llm = use_llm
44 self.llm_client = llm_client
45 self.threshold = threshold
47 if use_llm and not llm_client:
48 msg = "LLM client required when use_llm is True"
49 raise ValueError(msg)
51 def _extract_claims(self, response: str) -> list[str]:
52 """Extract factual claims from response.
54 Args:
55 response: The response text
57 Returns:
58 List of claim strings
59 """
60 # Simple sentence splitting as proxy for claims
61 sentences = re.split(r"[.!?]+\s+", response)
62 return list(map(str.strip, filter(lambda s: len(s.strip()) > 10, sentences)))
64 def _check_claim_support(
65 self,
66 claim: str,
67 context_text: str,
68 ) -> bool:
69 """Check if a claim is supported by context.
71 Args:
72 claim: The claim to verify
73 context_text: The context text
75 Returns:
76 True if claim appears supported
77 """
78 # Simple keyword overlap check
79 claim_words = set(re.findall(r"\b\w+\b", claim.lower()))
80 context_words = set(re.findall(r"\b\w+\b", context_text.lower()))
82 # Filter stop words
83 stop_words = {
84 "the",
85 "a",
86 "an",
87 "and",
88 "or",
89 "but",
90 "in",
91 "on",
92 "at",
93 "to",
94 "for",
95 "of",
96 "with",
97 "by",
98 "from",
99 "is",
100 "was",
101 }
102 claim_words -= stop_words
103 context_words -= stop_words
105 if not claim_words:
106 return True
108 # Check overlap ratio
109 overlap = len(claim_words & context_words)
110 overlap_ratio = overlap / len(claim_words)
112 return overlap_ratio >= 0.5
114 async def check_faithfulness(
115 self,
116 response: str,
117 context_chunks: list[ContextChunk],
118 ) -> float:
119 """Check response faithfulness to context.
121 Args:
122 response: The synthesized response
123 context_chunks: The context chunks used
125 Returns:
126 Faithfulness score (0-1)
127 """
128 if not response or not context_chunks:
129 return 0.0
131 # Combine all context
132 context_text = " ".join(chunk.text for chunk in context_chunks)
134 # Extract claims from response
135 claims = self._extract_claims(response)
137 if not claims:
138 return 1.0 # No claims to verify
140 # Check each claim
141 supported_claims = 0
143 for claim in claims:
144 if self._check_claim_support(claim, context_text):
145 supported_claims += 1
147 # Calculate faithfulness score
148 return supported_claims / len(claims)
150 async def is_faithful(
151 self,
152 response: str,
153 context_chunks: list[ContextChunk],
154 ) -> bool:
155 """Check if response meets faithfulness threshold.
157 Args:
158 response: The synthesized response
159 context_chunks: The context chunks used
161 Returns:
162 True if response is faithful
163 """
164 score = await self.check_faithfulness(response, context_chunks)
165 return score >= self.threshold