Coverage for /home/admin/Documents/AI/applications/lexigram-dev/lexigram/experimental/ai/lexigram-ai-evaluation/src/lexigram/ai/evaluation/evaluators/qa.py: 40%

1"""Question-Answer evaluation.""" 

2 

3from __future__ import annotations 

4 

5from lexigram.ai.evaluation.evaluators.base import BaseEvaluator 

6from lexigram.contracts.ai.evaluation import ( 

7 EvaluationResult, 

8 EvaluationScoreType, 

9 EvaluatorProtocol, 

10) 

11from lexigram.logging import get_logger 

12from lexigram.result import Ok, Result 

13 

14logger = get_logger(__name__) 

15 

16 

17class QAEvaluator(BaseEvaluator, EvaluatorProtocol): 

18 """Question-Answer matching evaluator. 

19 

20 Evaluates whether the output correctly answers the question 

21 based on the reference answer. Uses keyword overlap and 

22 semantic similarity. When the reference yields no extractable 

23 keywords (numeric, short-word, or stopword-only answers), falls 

24 back to a case-insensitive containment match of the stripped 

25 reference text within the stripped output text. 

26 """ 

27 

28 def __init__(self) -> None: 

29 super().__init__(EvaluationScoreType.PARTIAL_MATCH) 

30 

31 @property 

32 def name(self) -> str: 

33 return "qa" 

34 

35 async def evaluate( 

36 self, 

37 input: str, 

38 output: str, 

39 reference: str, 

40 ) -> Result[EvaluationResult, Exception]: 

41 reference_keywords = self._extract_keywords(reference) 

42 output_keywords = self._extract_keywords(output) 

43 

44 if not reference_keywords: 

45 # No comparable keywords in the reference (numeric, short-word, 

46 # stopword-only, or empty) — fall back to matching the raw text. 

47 ref_text = reference.strip().lower() 

48 output_text = output.strip().lower() 

49 score = 1.0 if ref_text and ref_text in output_text else 0.0 

50 else: 

51 overlap = len(set(output_keywords) & set(reference_keywords)) 

52 score = overlap / len(reference_keywords) 

53 

54 common = set(output_keywords) & set(reference_keywords) 

55 missing = set(reference_keywords) - set(output_keywords) 

56 

57 details: dict[str, float | str] = { 

58 "reference_keywords": str(reference_keywords), 

59 "output_keywords": str(output_keywords), 

60 "common_keywords": str(list(common)), 

61 "missing_keywords": str(list(missing)), 

62 } 

63 

64 if not reference_keywords: 

65 feedback = ( 

66 "Reference has no extractable keywords; " 

67 "used case-insensitive containment match" 

68 ) 

69 else: 

70 feedback = f"Matched {len(common)}/{len(reference_keywords)} key concepts" 

71 

72 return Ok(self._create_result(score, feedback, details)) 

73 

74 def _extract_keywords(self, text: str) -> list[str]: 

75 import re 

76 

77 text = text.lower() 

78 words = re.findall(r"\b[a-z]{3,}\b", text) 

79 stopwords = { 

80 "the", 

81 "and", 

82 "are", 

83 "for", 

84 "that", 

85 "this", 

86 "with", 

87 "from", 

88 "have", 

89 "has", 

90 "had", 

91 "was", 

92 "were", 

93 "been", 

94 "being", 

95 "will", 

96 "would", 

97 "could", 

98 "should", 

99 "can", 

100 "may", 

101 "might", 

102 "must", 

103 } 

104 return [w for w in words if w not in stopwords] 

105 

106 

107__all__ = ["QAEvaluator"]