Coverage for /home/admin/Documents/AI/applications/lexigram-dev/lexigram/experimental/ai/lexigram-ai-evaluation/src/lexigram/ai/evaluation/evaluators/criteria.py: 34%

1"""Criteria-based evaluator.""" 

2 

3from __future__ import annotations 

4 

5from typing import Any 

6 

7from lexigram.ai.evaluation.evaluators.base import BaseEvaluator 

8from lexigram.contracts.ai.evaluation import ( 

9 EvaluationResult, 

10 EvaluationScoreType, 

11 EvaluatorProtocol, 

12) 

13from lexigram.logging import get_logger 

14from lexigram.result import Ok, Result 

15 

16logger = get_logger(__name__) 

17 

18 

19class CriteriaEvaluator(BaseEvaluator, EvaluatorProtocol): 

20 """Rule-based criteria evaluation. 

21 

22 Evaluates outputs against predefined rules/criteria. 

23 Supports exact match, contains, regex, and custom predicate checks. 

24 """ 

25 

26 def __init__( 

27 self, 

28 criteria: list[dict[str, Any]] | None = None, 

29 ) -> None: 

30 super().__init__(EvaluationScoreType.EXACT_MATCH) 

31 self._criteria = criteria or [] 

32 

33 @property 

34 def name(self) -> str: 

35 return "criteria" 

36 

37 async def evaluate( 

38 self, 

39 input: str, 

40 output: str, 

41 reference: str, 

42 ) -> Result[EvaluationResult, Exception]: 

43 passed = 0 

44 total = len(self._criteria) or 1 

45 details: dict[str, Any] = {"criteria_results": []} 

46 

47 if not self._criteria: 

48 exact_match = output.strip().lower() == reference.strip().lower() 

49 passed = 1 if exact_match else 0 

50 details["exact_match"] = exact_match 

51 else: 

52 for criterion in self._criteria: 

53 criterion_type = criterion.get("type", "exact_match") 

54 result = self._evaluate_criterion( 

55 criterion_type, output, reference, criterion 

56 ) 

57 details["criteria_results"].append(result) 

58 if result.get("passed"): 

59 passed += 1 

60 

61 score = passed / total if total > 0 else 0.0 

62 feedback = f"Passed {passed}/{total} criteria" 

63 

64 return Ok(self._create_result(score, feedback, details)) 

65 

66 def _evaluate_criterion( 

67 self, 

68 criterion_type: str, 

69 output: str, 

70 reference: str, 

71 criterion: dict[str, Any], 

72 ) -> dict[str, Any]: 

73 result = {"type": criterion_type, "passed": False} 

74 

75 if criterion_type == "exact_match": 

76 result["passed"] = output.strip().lower() == reference.strip().lower() 

77 elif criterion_type == "contains": 

78 expected = criterion.get("expected", "") 

79 result["passed"] = expected.lower() in output.lower() 

80 elif criterion_type == "contains_all": 

81 expected = criterion.get("expected", []) 

82 result["passed"] = all(e.lower() in output.lower() for e in expected) 

83 elif criterion_type == "regex": 

84 import re 

85 

86 pattern = criterion.get("pattern", "") 

87 result["passed"] = bool(re.search(pattern, output)) 

88 

89 return result 

90 

91 

92__all__ = ["CriteriaEvaluator"]