Coverage for src / lexigram / contracts / ai / evaluation.py: 0%
48 statements
« prev ^ index » next coverage.py v7.13.5, created at 2026-08-19 05:41 +0800
1"""Evaluation protocols and types for AI evaluation."""
3from __future__ import annotations
5from dataclasses import dataclass
6from enum import Enum
7from typing import Any, Protocol
9from lexigram.contracts.ai.exceptions import EvaluationError
10from lexigram.contracts.core.result import Result
13class EvaluationScoreType(str, Enum):
14 """Type of evaluation score."""
16 EXACT_MATCH = "exact_match"
17 PARTIAL_MATCH = "partial_match"
18 SEMANTIC_SIMILARITY = "semantic_similarity"
19 STRING_DISTANCE = "string_distance"
20 TRAJECTORY_FIDELITY = "trajectory_fidelity"
21 CUSTOM = "custom"
24@dataclass(frozen=True)
25class EvaluationResult:
26 """Result of an evaluation run on a single sample.
28 Attributes:
29 score: The evaluation score (0.0 to 1.0).
30 score_type: The type of scoring method used.
31 feedback: Human-readable feedback about the evaluation.
32 metrics: Additional metrics computed during evaluation.
33 """
35 score: float
36 score_type: EvaluationScoreType
37 feedback: str
38 metrics: dict[str, Any]
41@dataclass(frozen=True)
42class EvaluationSample:
43 """A single sample in an evaluation dataset.
45 Attributes:
46 id: Unique identifier for this sample.
47 input: The input prompt or query.
48 reference: The expected reference output.
49 metadata: Additional metadata for this sample.
50 """
52 id: str
53 input: str
54 reference: str
55 metadata: dict[str, Any]
58@dataclass(frozen=True)
59class EvaluationDataset:
60 """A collection of evaluation samples.
62 Attributes:
63 name: Name of the dataset.
64 samples: List of evaluation samples.
65 metadata: Additional dataset metadata.
66 """
68 name: str
69 samples: list[EvaluationSample]
70 metadata: dict[str, Any]
73@dataclass(frozen=True)
74class RunReport:
75 """Report from running an evaluator on a dataset.
77 Attributes:
78 dataset_name: Name of the evaluated dataset.
79 evaluator_name: Name of the evaluator used.
80 total_samples: Total number of samples evaluated.
81 passed_samples: Number of samples that passed the evaluation.
82 average_score: Average score across all samples.
83 results: Individual sample results.
84 metadata: Additional report metadata.
85 """
87 dataset_name: str
88 evaluator_name: str
89 total_samples: int
90 passed_samples: int
91 average_score: float
92 results: list[EvaluationResult]
93 metadata: dict[str, Any]
96class EvaluatorProtocol(Protocol):
97 """Protocol for evaluation implementations.
99 An evaluator takes an input, output, and reference and produces
100 an evaluation result with a score and feedback.
101 """
103 @property
104 def name(self) -> str:
105 """Return the evaluator name."""
107 async def evaluate(
108 self,
109 input: str,
110 output: str,
111 reference: str,
112 ) -> Result[EvaluationResult, Exception]:
113 """Evaluate a single sample.
115 Args:
116 input: The input prompt or query.
117 output: The generated output to evaluate.
118 reference: The expected reference output.
120 Returns:
121 Ok(EvaluationResult) on success, Err(error) on failure.
122 """
125class EvaluationHarnessProtocol(Protocol):
126 """Protocol for evaluation harness implementations.
128 A harness runs an evaluator against a dataset and produces
129 a run report with aggregated results.
130 """
132 @property
133 def name(self) -> str:
134 """Return the harness name."""
136 async def run(
137 self,
138 dataset: EvaluationDataset,
139 evaluator: EvaluatorProtocol,
140 ) -> Result[RunReport, Exception]:
141 """Run an evaluator against a dataset.
143 Args:
144 dataset: The evaluation dataset.
145 evaluator: The evaluator to use.
147 Returns:
148 Ok(RunReport) on success, Err(error) on failure.
149 """
152__all__ = [
153 "EvaluationDataset",
154 "EvaluationError",
155 "EvaluationHarnessProtocol",
156 "EvaluationResult",
157 "EvaluationSample",
158 "EvaluationScoreType",
159 "EvaluatorProtocol",
160 "RunReport",
161]