Coverage for little_loops / fsm / evaluators.py: 8%
544 statements
« prev ^ index » next coverage.py v7.12.0, created at 2026-06-29 00:54 -0500
1"""FSM Evaluators for loop execution.
3This module provides evaluators that interpret action output and produce
4verdicts for state transitions.
6Supported evaluator types:
8Tier 1 (Deterministic - no API calls):
9 exit_code: Map Unix exit codes to verdicts (0=success, 1=failure, 2+=error)
10 output_numeric: Compare numeric output to target value
11 output_json: Extract and compare JSON path values
12 output_contains: Pattern matching on stdout
13 convergence: Track progress toward a target value
14 diff_stall: Detect stalled iterations via git diff comparison
15 action_stall: Detect when the same action string or output repeats for N consecutive iterations
16 harbor_scorer: Interpret Harbor-format benchmark scorer exit code and float stdout
18Tier 2 (LLM-based):
19 llm_structured: Use LLM with structured output for natural language evaluation
20 contract: Read producer/consumer file pairs and assert contract alignment via LLM judge
22Tier 3 (External process):
23 mcp_result: Parse MCP tool call response envelope
24"""
26from __future__ import annotations
28import hashlib
29import json
30import random
31import re
32import subprocess
33import time
34from collections.abc import Callable
35from dataclasses import dataclass
36from pathlib import Path
37from typing import Any
39from little_loops.fsm.interpolation import (
40 InterpolationContext,
41 InterpolationError,
42 interpolate,
43)
44from little_loops.fsm.schema import DEFAULT_LLM_MODEL, EvaluateConfig
45from little_loops.host_runner import resolve_host
48@dataclass
49class EvaluationResult:
50 """Result from an evaluator.
52 Attributes:
53 verdict: The routing key for state transitions
54 details: Evaluator-specific metadata for debugging/logging
55 """
57 verdict: str
58 details: dict[str, Any]
61# Evidence contract injected into every LLM evaluator prompt (ENH-2342).
62# Requires the model to cite verbatim output text for each verdict; absent evidence
63# is coerced to "no" at the parsing layer so self-grade optimism can't slip through.
64CHECK_SEMANTIC_EVIDENCE_CONTRACT = """
65IMPORTANT: For each condition you evaluate:
661. State your verdict: Yes / No / Partial
672. Provide a VERBATIM quote from the output that supports your verdict (exact text, in quotes)
683. If you cannot quote specific text, your verdict is automatically No (or Partial if context suggests partial progress)
70Do not assert a verdict without evidence. "The task appears complete" is not evidence.
71"""
73# Default schema for LLM structured evaluation
74DEFAULT_LLM_SCHEMA: dict[str, Any] = {
75 "type": "object",
76 "properties": {
77 "verdict": {
78 "type": "string",
79 "enum": ["yes", "no", "blocked", "partial"],
80 "description": (
81 "- yes: The condition/check evaluated to true\n"
82 "- no: The condition/check evaluated to false\n"
83 "- blocked: Cannot proceed without external help\n"
84 "- partial: Made progress but not complete"
85 ),
86 },
87 "confidence": {
88 "type": "number",
89 "minimum": 0,
90 "maximum": 1,
91 "description": "Confidence in this verdict (0-1)",
92 },
93 "reason": {
94 "type": "string",
95 "description": "Brief explanation",
96 },
97 "evidence": {
98 "type": "string",
99 "description": (
100 "Verbatim quote from the action output that supports the verdict. "
101 "Empty string means no evidence was found; verdict will be coerced to 'no'."
102 ),
103 },
104 },
105 "required": ["verdict", "confidence", "reason", "evidence"],
106}
108DEFAULT_LLM_PROMPT = "Evaluate whether this action succeeded based on its output."
110# Schema for blind A/B comparator: evaluates two anonymized outputs
111BLIND_COMPARATOR_SCHEMA: dict[str, Any] = {
112 "type": "object",
113 "properties": {
114 "verdict_a": {
115 "type": "string",
116 "enum": ["yes", "no"],
117 "description": "Whether Output A meets the evaluation criteria",
118 },
119 "verdict_b": {
120 "type": "string",
121 "enum": ["yes", "no"],
122 "description": "Whether Output B meets the evaluation criteria",
123 },
124 "confidence": {
125 "type": "number",
126 "minimum": 0,
127 "maximum": 1,
128 "description": "Confidence in these verdicts (0-1)",
129 },
130 "reason": {
131 "type": "string",
132 "description": "Brief explanation comparing the two outputs",
133 },
134 },
135 "required": ["verdict_a", "verdict_b", "confidence", "reason"],
136}
138DEFAULT_BLIND_COMPARATOR_PROMPT = (
139 "You are evaluating two outputs (labeled 'Output A' and 'Output B') that were "
140 "produced by independent runs of the same task. Judge whether each output meets "
141 "the evaluation criteria below. Be objective and impartial — the labels 'A' and "
142 "'B' are arbitrary and do not indicate which is better."
143)
145_NUMERIC_OPERATORS: dict[str, Callable[[float, float], bool]] = {
146 "eq": lambda v, t: v == t,
147 "ne": lambda v, t: v != t,
148 "lt": lambda v, t: v < t,
149 "le": lambda v, t: v <= t,
150 "gt": lambda v, t: v > t,
151 "ge": lambda v, t: v >= t,
152}
155def evaluate_exit_code(exit_code: int) -> EvaluationResult:
156 """Map Unix exit code to verdict.
158 Args:
159 exit_code: The process exit code
161 Returns:
162 EvaluationResult with verdict:
163 - 0 -> yes
164 - 1 -> no
165 - 2+ -> error
166 """
167 if exit_code == 0:
168 verdict = "yes"
169 elif exit_code == 1:
170 verdict = "no"
171 else:
172 verdict = "error"
174 return EvaluationResult(verdict=verdict, details={"exit_code": exit_code})
177def evaluate_output_numeric(
178 output: str,
179 operator: str,
180 target: float,
181) -> EvaluationResult:
182 """Parse stdout as number and compare to target.
184 Args:
185 output: The action stdout to parse as a number
186 operator: Comparison operator (eq, ne, lt, le, gt, ge)
187 target: Target value to compare against
189 Returns:
190 EvaluationResult with verdict:
191 - Condition met -> yes
192 - Condition not met -> no
193 - Parse error -> error
194 """
195 try:
196 value = float(output.strip())
197 except ValueError:
198 return EvaluationResult(
199 verdict="error",
200 details={"error": f"Cannot parse as number: {output[:100]}"},
201 )
203 if operator not in _NUMERIC_OPERATORS:
204 return EvaluationResult(
205 verdict="error",
206 details={"error": f"Unknown operator: {operator}"},
207 )
209 condition_met = _NUMERIC_OPERATORS[operator](value, target)
210 return EvaluationResult(
211 verdict="yes" if condition_met else "no",
212 details={"value": value, "target": target, "operator": operator},
213 )
216def _extract_json_path(data: Any, path: str) -> Any:
217 """Extract value from dict using jq-style path like '.summary.failed'.
219 Args:
220 data: The parsed JSON data (dict or list)
221 path: Dot-separated path, optionally starting with '.'
223 Returns:
224 The value at the specified path
226 Raises:
227 KeyError: If path not found in data
228 """
229 if path.startswith("."):
230 path = path[1:]
231 parts = path.split(".")
232 current = data
233 for part in parts:
234 if isinstance(current, dict) and part in current:
235 current = current[part]
236 elif isinstance(current, list) and part.isdigit():
237 idx = int(part)
238 if 0 <= idx < len(current):
239 current = current[idx]
240 else:
241 raise KeyError(path)
242 else:
243 raise KeyError(path)
244 return current
247def _compare_values(
248 value: int | float, operator: str, target: int | float, path: str
249) -> EvaluationResult:
250 """Compare numeric values using operator.
252 Args:
253 value: The extracted value to compare
254 operator: Comparison operator
255 target: Target value
256 path: JSON path for details
258 Returns:
259 EvaluationResult with comparison result
260 """
261 if operator not in _NUMERIC_OPERATORS:
262 return EvaluationResult(
263 verdict="error",
264 details={"error": f"Unknown operator: {operator}"},
265 )
267 condition_met = _NUMERIC_OPERATORS[operator](value, target)
268 return EvaluationResult(
269 verdict="yes" if condition_met else "no",
270 details={"value": value, "path": path, "target": target, "operator": operator},
271 )
274def evaluate_output_json(
275 output: str,
276 path: str,
277 operator: str,
278 target: Any,
279) -> EvaluationResult:
280 """Parse JSON and extract value at path, then compare.
282 Args:
283 output: The action stdout containing JSON
284 path: jq-style dot notation path (e.g., '.summary.failed')
285 operator: Comparison operator (eq, ne, lt, le, gt, ge)
286 target: Target value for comparison
288 Returns:
289 EvaluationResult with verdict:
290 - Condition met -> yes
291 - Condition not met -> no
292 - Parse/path error -> error
293 """
294 try:
295 data = json.loads(output)
296 except json.JSONDecodeError as e:
297 return EvaluationResult(
298 verdict="error",
299 details={"error": f"Invalid JSON: {e}"},
300 )
302 try:
303 value = _extract_json_path(data, path)
304 except KeyError:
305 return EvaluationResult(
306 verdict="error",
307 details={"error": f"Path not found: {path}"},
308 )
310 # Use numeric comparison if both values are numeric
311 if isinstance(value, (int, float)) and isinstance(target, (int, float)):
312 return _compare_values(value, operator, target, path)
314 # For non-numeric values, only eq and ne are supported
315 if operator == "eq":
316 verdict = "yes" if value == target else "no"
317 elif operator == "ne":
318 verdict = "yes" if value != target else "no"
319 else:
320 return EvaluationResult(
321 verdict="error",
322 details={"error": f"Operator {operator} not supported for non-numeric values"},
323 )
325 return EvaluationResult(
326 verdict=verdict,
327 details={"value": value, "path": path, "target": target, "operator": operator},
328 )
331def evaluate_output_contains(
332 output: str,
333 pattern: str,
334 negate: bool = False,
335 error_patterns: list[str] | None = None,
336) -> EvaluationResult:
337 """Check if pattern exists in output.
339 Pattern can be regex or substring. If regex fails to compile,
340 falls back to substring matching.
342 Args:
343 output: The action stdout to search
344 pattern: Regex pattern or substring
345 negate: If True, invert the match result
346 error_patterns: Optional list of substrings that, when matched in output
347 and the main pattern is not found, yield verdict="error". This allows
348 loops to route auth/error output to on_error without raising an exception.
350 Returns:
351 EvaluationResult with verdict:
352 - Found (negate=False) -> yes
353 - Found (negate=True) -> no
354 - Not found (negate=False) -> no (or "error" if error_patterns matched)
355 - Not found (negate=True) -> yes
356 """
357 # Try regex first, fall back to substring
358 try:
359 matched = bool(re.search(pattern, output))
360 except re.error:
361 matched = pattern in output
363 if negate:
364 verdict = "no" if matched else "yes"
365 else:
366 verdict = "yes" if matched else "no"
368 # Check error_patterns before returning "no" — only when main pattern didn't match
369 if verdict == "no" and not negate and error_patterns:
370 for ep in error_patterns:
371 if ep in output:
372 return EvaluationResult(
373 verdict="error",
374 details={
375 "matched": False,
376 "pattern": pattern,
377 "negate": negate,
378 "error_pattern": ep,
379 },
380 )
382 return EvaluationResult(
383 verdict=verdict,
384 details={"matched": matched, "pattern": pattern, "negate": negate},
385 )
388def evaluate_convergence(
389 current: float,
390 previous: float | None,
391 target: float,
392 tolerance: float = 0,
393 direction: str = "minimize",
394) -> EvaluationResult:
395 """Compare current value to target and previous.
397 Args:
398 current: Current metric value
399 previous: Previous metric value (None if first iteration)
400 target: Target value to reach
401 tolerance: Acceptable distance from target
402 direction: 'minimize' or 'maximize'
404 Returns:
405 EvaluationResult with verdict:
406 - Value within tolerance of target -> target
407 - Value improved toward target -> progress
408 - Value unchanged or worsened -> stall
409 """
410 # Check if target reached (within tolerance)
411 if abs(current - target) <= tolerance:
412 return EvaluationResult(
413 verdict="target",
414 details={"current": current, "target": target, "delta": 0},
415 )
417 # First iteration has no previous value
418 if previous is None:
419 return EvaluationResult(
420 verdict="progress",
421 details={
422 "current": current,
423 "previous": None,
424 "target": target,
425 "delta": None,
426 },
427 )
429 # Calculate progress
430 delta = current - previous
432 if direction == "minimize":
433 # For minimizing, negative delta is progress
434 made_progress = delta < 0
435 else:
436 # For maximizing, positive delta is progress
437 made_progress = delta > 0
439 verdict = "progress" if made_progress else "stall"
441 return EvaluationResult(
442 verdict=verdict,
443 details={
444 "current": current,
445 "previous": previous,
446 "target": target,
447 "delta": delta,
448 "direction": direction,
449 },
450 )
453def evaluate_classify(
454 output: str,
455 line: str | int | None = None,
456) -> EvaluationResult:
457 """Read a token from stdout and return it as the verdict.
459 Intended for single-state multi-way routing: the action prints exactly one
460 token to stdout and the route: table maps that token to the next state.
462 Args:
463 output: The action stdout to read the token from
464 line: Which line to select. 'last' (default) picks the last non-empty
465 line; 'first' picks the first non-empty line; an integer index
466 selects that line (0-based, negative indices supported).
468 Returns:
469 EvaluationResult with verdict = trimmed token, or "" when output is
470 empty (which _route() maps to the route.default fallback).
471 """
472 lines = [ln for ln in output.splitlines() if ln.strip()]
473 if not lines:
474 return EvaluationResult(
475 verdict="",
476 details={"token": "", "line": line, "source_lines": 0},
477 )
479 selector = line if line is not None else "last"
480 if selector == "last":
481 selected = lines[-1]
482 elif selector == "first":
483 selected = lines[0]
484 elif isinstance(selector, int):
485 try:
486 selected = lines[selector]
487 except IndexError:
488 return EvaluationResult(
489 verdict="",
490 details={
491 "token": "",
492 "line": line,
493 "source_lines": len(lines),
494 "error": "index out of range",
495 },
496 )
497 else:
498 selected = lines[-1]
500 token = selected.strip()
501 return EvaluationResult(
502 verdict=token,
503 details={"token": token, "line": line},
504 )
507def evaluate_diff_stall(
508 scope: list[str] | None = None,
509 max_stall: int = 1,
510) -> EvaluationResult:
511 """Detect stalled iterations by comparing git diff --stat between runs.
513 On first call, snapshots the current diff and returns 'yes'.
514 On subsequent calls, compares current diff to the previous snapshot.
515 If the diff is identical for max_stall consecutive iterations, returns
516 'no' (stalled). If different, resets the stall counter and returns
517 'yes' (progress).
519 State is persisted in /tmp using a key derived from the scope argument,
520 so different loops with different scopes maintain independent stall counters.
522 Args:
523 scope: Optional list of paths to limit the git diff to. Defaults to
524 the entire working tree.
525 max_stall: Number of consecutive no-change iterations before stall
526 verdict. Defaults to 1.
528 Returns:
529 EvaluationResult with verdict:
530 - yes: diff changed since last iteration (progress made)
531 - no: diff unchanged for max_stall iterations (stalled)
532 - error: git command failed or timed out
533 """
534 cmd = ["git", "diff", "--stat"]
535 if scope:
536 cmd += ["--"] + scope
538 try:
539 proc = subprocess.run(cmd, capture_output=True, text=True, timeout=30)
540 except subprocess.TimeoutExpired:
541 return EvaluationResult(verdict="error", details={"error": "git diff timed out"})
542 except FileNotFoundError:
543 return EvaluationResult(verdict="error", details={"error": "git not found in PATH"})
545 if proc.returncode != 0:
546 return EvaluationResult(
547 verdict="error",
548 details={"error": f"git diff failed: {proc.stderr[:200]}"},
549 )
551 current_diff = proc.stdout
553 # Derive a stable cache key from the scope so independent loops don't collide
554 scope_str = "|".join(sorted(scope)) if scope else "_root_"
555 cache_key = hashlib.md5(scope_str.encode()).hexdigest()[:12]
556 loops_tmp = Path.cwd() / ".loops" / "tmp"
557 loops_tmp.mkdir(parents=True, exist_ok=True)
558 state_file = loops_tmp / f"ll-diff-stall-{cache_key}.txt"
559 count_file = loops_tmp / f"ll-diff-stall-{cache_key}.count"
561 # Read previous snapshot and stall count
562 previous_diff: str | None = None
563 stall_count = 0
564 try:
565 previous_diff = state_file.read_text()
566 stall_count = int(count_file.read_text().strip())
567 except (FileNotFoundError, ValueError):
568 pass
570 # First iteration: save snapshot and report progress
571 if previous_diff is None:
572 state_file.write_text(current_diff)
573 count_file.write_text("0")
574 return EvaluationResult(
575 verdict="yes",
576 details={"stall_count": 0, "max_stall": max_stall, "diff_changed": True},
577 )
579 if current_diff == previous_diff:
580 stall_count += 1
581 count_file.write_text(str(stall_count))
582 if stall_count >= max_stall:
583 return EvaluationResult(
584 verdict="no",
585 details={"stall_count": stall_count, "max_stall": max_stall, "diff_changed": False},
586 )
587 # Not yet at max_stall threshold — still report yes so loop continues
588 return EvaluationResult(
589 verdict="yes",
590 details={"stall_count": stall_count, "max_stall": max_stall, "diff_changed": False},
591 )
592 else:
593 # Progress: update snapshot and reset counter
594 state_file.write_text(current_diff)
595 count_file.write_text("0")
596 return EvaluationResult(
597 verdict="yes",
598 details={"stall_count": 0, "max_stall": max_stall, "diff_changed": True},
599 )
602def evaluate_action_stall(
603 track: list[str] | None = None,
604 max_repeat: int = 2,
605 context: InterpolationContext | None = None,
606) -> EvaluationResult:
607 """Detect when the same action string or output repeats for N consecutive iterations.
609 On first call, snapshots the hashed values of the tracked context keys and returns
610 'yes'. On subsequent calls, compares the current hash to the previous snapshot.
611 If the hash is identical for max_repeat consecutive iterations, returns 'no'
612 (stalled). If different, resets the stall counter and returns 'yes' (progress).
614 State is persisted in .loops/tmp using a key derived from the tracked keys,
615 so different states/loops maintain independent stall counters.
617 Args:
618 track: Context keys to track. Defaults to ["action"] when None.
619 max_repeat: Number of consecutive identical-hash iterations before stall verdict.
620 Defaults to 2.
621 context: Runtime interpolation context for resolving tracked keys.
623 Returns:
624 EvaluationResult with verdict:
625 - yes: tracked values changed since last iteration (progress made)
626 - no: tracked values identical for max_repeat iterations (stalled)
627 """
628 effective_track: list[str] = track if track is not None else ["action"]
630 # Resolve each tracked key from context and hash the combined values.
631 # Keys may be bare names (e.g. "action") or namespaced (e.g. "context.action").
632 # Try namespaced forms first: context.<key>, captured.<key>, then bare ${key}.
633 parts: list[str] = []
634 for key in effective_track:
635 value: str = ""
636 if context is not None:
637 # If key already contains a dot it's already namespaced; use as-is.
638 if "." in key:
639 try:
640 value = str(interpolate(f"${{{key}}}", context))
641 except InterpolationError:
642 value = ""
643 else:
644 # Try context.<key> first, then captured.<key>, then give up.
645 resolved = False
646 for namespace in ("context", "captured", "prev", "result"):
647 try:
648 value = str(interpolate(f"${{{namespace}.{key}}}", context))
649 resolved = True
650 break
651 except InterpolationError:
652 continue
653 if not resolved:
654 value = ""
655 parts.append(f"{key}={value}")
657 combined = "|".join(parts)
658 current_hash = hashlib.md5(combined.encode()).hexdigest()
660 # Derive a stable cache key from the tracked keys
661 track_str = "|".join(sorted(effective_track))
662 cache_key = hashlib.md5(track_str.encode()).hexdigest()[:12]
663 loops_tmp = Path.cwd() / ".loops" / "tmp"
664 loops_tmp.mkdir(parents=True, exist_ok=True)
665 state_file = loops_tmp / f"ll-action-stall-{cache_key}.txt"
666 count_file = loops_tmp / f"ll-action-stall-{cache_key}.count"
668 # Read previous hash and stall count
669 previous_hash: str | None = None
670 stall_count = 0
671 try:
672 previous_hash = state_file.read_text().strip()
673 stall_count = int(count_file.read_text().strip())
674 except (FileNotFoundError, ValueError):
675 pass
677 # First iteration: save hash and report progress
678 if previous_hash is None:
679 state_file.write_text(current_hash)
680 count_file.write_text("0")
681 return EvaluationResult(
682 verdict="yes",
683 details={
684 "stall_count": 0,
685 "max_repeat": max_repeat,
686 "hash_changed": True,
687 "tracked_keys": effective_track,
688 },
689 )
691 hash_changed = current_hash != previous_hash
693 if hash_changed:
694 # Progress: update snapshot and reset counter
695 state_file.write_text(current_hash)
696 count_file.write_text("0")
697 return EvaluationResult(
698 verdict="yes",
699 details={
700 "stall_count": 0,
701 "max_repeat": max_repeat,
702 "hash_changed": True,
703 "tracked_keys": effective_track,
704 },
705 )
706 else:
707 # Same hash as last time
708 stall_count += 1
709 count_file.write_text(str(stall_count))
710 if stall_count >= max_repeat:
711 return EvaluationResult(
712 verdict="no",
713 details={
714 "stall_count": stall_count,
715 "max_repeat": max_repeat,
716 "hash_changed": False,
717 "tracked_keys": effective_track,
718 "repeated_hash": current_hash,
719 },
720 )
721 # Not yet at max_repeat threshold — still report yes so loop continues
722 return EvaluationResult(
723 verdict="yes",
724 details={
725 "stall_count": stall_count,
726 "max_repeat": max_repeat,
727 "hash_changed": False,
728 "tracked_keys": effective_track,
729 },
730 )
733def evaluate_mcp_result(output: str, exit_code: int) -> EvaluationResult:
734 """Evaluate an MCP tool call result from the mcp-call subprocess.
736 Maps exit codes and MCP response envelope fields to routing verdicts.
738 Exit code conventions (set by mcp-call):
739 0 → parse isError from JSON envelope
740 1 → tool_error (tool ran but isError: true)
741 124 → timeout (transport-level timeout)
742 127 → not_found (server or tool missing from .mcp.json)
744 Args:
745 output: stdout from mcp-call (MCP response envelope JSON)
746 exit_code: Exit code from mcp-call subprocess
748 Returns:
749 EvaluationResult with verdict:
750 - success → isError: false
751 - tool_error → isError: true
752 - not_found → server/tool not in .mcp.json (exit 127)
753 - timeout → transport-level timeout (exit 124)
754 """
755 if exit_code == 127:
756 return EvaluationResult(
757 verdict="not_found",
758 details={"exit_code": exit_code, "error": "Server or tool not found in .mcp.json"},
759 )
761 if exit_code == 124:
762 return EvaluationResult(
763 verdict="timeout",
764 details={"exit_code": exit_code, "error": "MCP tool call timed out"},
765 )
767 # Parse MCP envelope JSON from stdout
768 try:
769 envelope = json.loads(output.strip()) if output.strip() else {}
770 except json.JSONDecodeError:
771 return EvaluationResult(
772 verdict="tool_error",
773 details={
774 "exit_code": exit_code,
775 "error": f"Invalid JSON from mcp-call: {output[:200]}",
776 },
777 )
779 is_error = envelope.get("isError", exit_code != 0)
781 if is_error:
782 return EvaluationResult(
783 verdict="tool_error",
784 details={"exit_code": exit_code, "envelope": envelope},
785 )
787 return EvaluationResult(
788 verdict="success",
789 details={"exit_code": exit_code, "envelope": envelope},
790 )
793def evaluate_harbor_scorer(output: str, exit_code: int) -> EvaluationResult:
794 """Evaluate a Harbor-format benchmark scorer result.
796 The scorer is a shell command that prints a float score (0.0–1.0) to stdout
797 and exits 0 on success or non-zero on failure.
799 Args:
800 output: stdout from the scorer subprocess (expected: a bare float)
801 exit_code: Exit code from the scorer subprocess
803 Returns:
804 EvaluationResult with verdict:
805 - yes → exit 0 and stdout parses as a float
806 - no → exit non-zero (scorer determined failure)
807 - error → exit 0 but stdout is not parseable as a float
808 """
809 if exit_code != 0:
810 return EvaluationResult(
811 verdict="no",
812 details={"exit_code": exit_code},
813 )
815 try:
816 score = float(output.strip())
817 except (ValueError, AttributeError):
818 return EvaluationResult(
819 verdict="error",
820 details={
821 "exit_code": exit_code,
822 "error": f"Scorer stdout is not a float: {output[:200]}",
823 },
824 )
826 return EvaluationResult(
827 verdict="yes",
828 details={"score": score, "exit_code": 0},
829 )
832def evaluate_llm_structured(
833 output: str,
834 prompt: str | None = None,
835 schema: dict[str, Any] | None = None,
836 min_confidence: float = 0.5,
837 uncertain_suffix: bool = False,
838 model: str = DEFAULT_LLM_MODEL,
839 max_tokens: int = 256,
840 timeout: int = 1800,
841) -> EvaluationResult:
842 """Evaluate action output using LLM with structured output via Claude CLI.
844 This is the ONLY place in the FSM system that uses LLM structured output.
845 Requires the ``claude`` CLI to be installed and authenticated.
847 Args:
848 output: Action stdout to evaluate
849 prompt: Custom evaluation prompt (defaults to basic success check)
850 schema: Custom JSON schema for structured response
851 min_confidence: Minimum confidence threshold (0-1)
852 uncertain_suffix: If True, append _uncertain to low-confidence verdicts
853 model: Model identifier (CLI aliases like "sonnet" or full names)
854 max_tokens: Maximum tokens for response (passed to --max-turns is not
855 applicable; kept for signature compat)
856 timeout: Timeout in seconds
858 Returns:
859 EvaluationResult with verdict from LLM and confidence/reason in details
860 """
861 effective_schema = schema or DEFAULT_LLM_SCHEMA
862 effective_prompt = (prompt or DEFAULT_LLM_PROMPT) + "\n\n" + CHECK_SEMANTIC_EVIDENCE_CONTRACT
864 # Truncate output to avoid context limits (keep last 4000 chars)
865 truncated = output[-4000:] if len(output) > 4000 else output
867 user_prompt = f"{effective_prompt}\n\n<action_output>\n{truncated}\n</action_output>"
869 invocation = resolve_host().build_blocking_json(prompt=user_prompt, model=model)
870 # Builder drops json_schema (Protocol surface only) and omits the
871 # claude-CLI-specific --no-session-persistence flag; augment at call site.
872 args = list(invocation.args) + [
873 "--json-schema",
874 json.dumps(effective_schema),
875 "--no-session-persistence",
876 ]
878 t0 = time.monotonic()
879 try:
880 proc = subprocess.run(
881 [invocation.binary, *args], capture_output=True, text=True, timeout=timeout
882 )
883 except subprocess.TimeoutExpired:
884 return EvaluationResult(
885 verdict="error",
886 details={"error": "LLM evaluation timeout", "timeout": True},
887 )
888 except FileNotFoundError:
889 return EvaluationResult(
890 verdict="error",
891 details={
892 "error": f"{invocation.binary} CLI not found. Install the active host CLI (see LL_HOST_CLI).",
893 "missing_dependency": True,
894 },
895 )
896 llm_latency_ms = int((time.monotonic() - t0) * 1000)
898 if proc.returncode != 0:
899 return EvaluationResult(
900 verdict="error",
901 details={
902 "error": f"{invocation.binary} CLI error: {proc.stderr.strip()}",
903 "api_error": True,
904 },
905 )
907 # Guard: empty stdout with exit 0 (API error not reflected in exit code)
908 if not proc.stdout.strip():
909 stderr_info = proc.stderr.strip()[:200] if proc.stderr else ""
910 error_msg = f"{invocation.binary} CLI returned empty output"
911 if stderr_info:
912 error_msg += f" (stderr: {stderr_info})"
913 return EvaluationResult(
914 verdict="error",
915 details={"error": error_msg, "empty_output": True},
916 )
918 # Parse the CLI JSON envelope and extract structured result.
919 # With --json-schema the envelope is:
920 # success: {"type":"result","subtype":"success","structured_output":{...},...}
921 # failure: {"type":"result","subtype":"error_max_structured_output_retries",...}
922 # If stdout is JSONL (multiple JSON objects), use the last non-empty line.
923 try:
924 stdout = proc.stdout.strip()
925 try:
926 envelope = json.loads(stdout)
927 except json.JSONDecodeError:
928 # Try JSONL: take the last non-empty line
929 lines = [line for line in stdout.split("\n") if line.strip()]
930 if not lines:
931 raise
932 envelope = json.loads(lines[-1])
934 # Check structured-output retry exhaustion (--json-schema failure mode)
935 if envelope.get("subtype") == "error_max_structured_output_retries":
936 return EvaluationResult(
937 verdict="error",
938 details={
939 "error": "Claude CLI could not produce valid structured output after retries",
940 "api_error": True,
941 },
942 )
944 # Check legacy is_error flag (some CLI versions exit 0 but report error in envelope)
945 if envelope.get("is_error", False):
946 err_text = str(envelope.get("result", "") or "")[:200]
947 return EvaluationResult(
948 verdict="error",
949 details={"error": f"Claude CLI reported error: {err_text}", "api_error": True},
950 )
952 # --json-schema mode returns validated dict in "structured_output"
953 if isinstance(envelope.get("structured_output"), dict):
954 llm_result: dict[str, Any] = envelope["structured_output"]
955 else:
956 raw_result = envelope.get("result", "")
957 if isinstance(raw_result, dict):
958 llm_result = raw_result
959 elif raw_result:
960 llm_result = json.loads(raw_result)
961 elif "verdict" in envelope:
962 llm_result = envelope
963 else:
964 raw_preview = proc.stdout[:300]
965 return EvaluationResult(
966 verdict="error",
967 details={
968 "error": "Empty result field in Claude CLI response",
969 "raw_preview": raw_preview,
970 },
971 )
972 except (json.JSONDecodeError, TypeError, ValueError) as e:
973 raw_preview = proc.stdout[:300] if proc.stdout else "(empty)"
974 return EvaluationResult(
975 verdict="error",
976 details={"error": f"Failed to parse LLM response: {e}", "raw_preview": raw_preview},
977 )
979 # Build result with confidence handling
980 verdict = str(llm_result.get("verdict", "error"))
981 confidence = float(llm_result.get("confidence", 1.0))
982 confident = confidence >= min_confidence
984 # Evidence coercion (ENH-2342): when using the default schema, a verdict without
985 # verbatim evidence is treated as "no" — prevents LLM optimism from passing silently.
986 # Custom schemas bypass this; callers that supply their own schema control the contract.
987 evidence = llm_result.get("evidence", "")
988 evidence_coerced = schema is None and not evidence.strip() and verdict not in ("error",)
989 if evidence_coerced:
990 verdict = "no"
992 # Optionally modify verdict for low confidence
993 if uncertain_suffix and not confident:
994 verdict = f"{verdict}_uncertain"
996 return EvaluationResult(
997 verdict=verdict,
998 details={
999 "confidence": confidence,
1000 "confident": confident,
1001 "reason": llm_result.get("reason", ""),
1002 "evidence": evidence,
1003 "evidence_coerced": evidence_coerced,
1004 "raw": llm_result,
1005 "llm_model": model,
1006 "llm_latency_ms": llm_latency_ms,
1007 "llm_prompt": user_prompt[:500],
1008 "llm_raw_output": proc.stdout[:500] if proc.stdout else "",
1009 },
1010 )
1013def evaluate_blind_comparator(
1014 output_harness: str,
1015 output_baseline: str,
1016 prompt: str | None = None,
1017 model: str = DEFAULT_LLM_MODEL,
1018 timeout: int = 1800,
1019) -> dict[str, Any]:
1020 """Blindly evaluate two outputs, returning pass/fail for each arm.
1022 Outputs are randomly labeled "Output A" / "Output B" so the LLM judge
1023 cannot distinguish the harness arm from the baseline arm. The mapping is
1024 de-anonymized after judgment so callers receive harness/baseline verdicts.
1026 Args:
1027 output_harness: stdout from the harness (gated) arm
1028 output_baseline: stdout from the baseline (ungated) arm
1029 prompt: Custom evaluation prompt (appended to default framing)
1030 model: Model identifier for the judge
1031 timeout: Timeout in seconds
1033 Returns:
1034 Dict with keys: harness_pass (bool), baseline_pass (bool),
1035 confidence (float), reason (str), raw (dict with A/B verdicts)
1036 """
1037 effective_prompt = prompt or DEFAULT_BLIND_COMPARATOR_PROMPT
1039 # Truncate outputs to avoid context limits
1040 truncated_harness = output_harness[-4000:] if len(output_harness) > 4000 else output_harness
1041 truncated_baseline = output_baseline[-4000:] if len(output_baseline) > 4000 else output_baseline
1043 # Randomize order: coin flip determines whether harness→A / baseline→B
1044 harness_is_a = random.choice([True, False])
1045 if harness_is_a:
1046 output_a, output_b = truncated_harness, truncated_baseline
1047 else:
1048 output_a, output_b = truncated_baseline, truncated_harness
1050 user_prompt = (
1051 f"{effective_prompt}\n\n"
1052 f"<output_a>\n{output_a}\n</output_a>\n\n"
1053 f"<output_b>\n{output_b}\n</output_b>"
1054 )
1056 invocation = resolve_host().build_blocking_json(prompt=user_prompt, model=model)
1057 args = list(invocation.args) + [
1058 "--json-schema",
1059 json.dumps(BLIND_COMPARATOR_SCHEMA),
1060 "--no-session-persistence",
1061 ]
1063 try:
1064 proc = subprocess.run(
1065 [invocation.binary, *args], capture_output=True, text=True, timeout=timeout
1066 )
1067 except subprocess.TimeoutExpired:
1068 # On timeout, both fail — conservative default
1069 return {
1070 "harness_pass": False,
1071 "baseline_pass": False,
1072 "confidence": 0.0,
1073 "reason": "LLM evaluation timed out",
1074 "raw": {"verdict_a": "timeout", "verdict_b": "timeout"},
1075 "error": "timeout",
1076 }
1077 except FileNotFoundError:
1078 return {
1079 "harness_pass": False,
1080 "baseline_pass": False,
1081 "confidence": 0.0,
1082 "reason": f"{invocation.binary} CLI not found",
1083 "raw": {"verdict_a": "error", "verdict_b": "error"},
1084 "error": "missing_cli",
1085 }
1087 if proc.returncode != 0:
1088 return {
1089 "harness_pass": False,
1090 "baseline_pass": False,
1091 "confidence": 0.0,
1092 "reason": f"Judge CLI error: {proc.stderr.strip()[:200]}",
1093 "raw": {"verdict_a": "error", "verdict_b": "error"},
1094 "error": "api_error",
1095 }
1097 if not proc.stdout.strip():
1098 return {
1099 "harness_pass": False,
1100 "baseline_pass": False,
1101 "confidence": 0.0,
1102 "reason": "Judge returned empty output",
1103 "raw": {"verdict_a": "error", "verdict_b": "error"},
1104 "error": "empty_output",
1105 }
1107 try:
1108 stdout = proc.stdout.strip()
1109 try:
1110 envelope = json.loads(stdout)
1111 except json.JSONDecodeError:
1112 lines = [line for line in stdout.split("\n") if line.strip()]
1113 if not lines:
1114 raise
1115 envelope = json.loads(lines[-1])
1117 if envelope.get("subtype") == "error_max_structured_output_retries":
1118 return {
1119 "harness_pass": False,
1120 "baseline_pass": False,
1121 "confidence": 0.0,
1122 "reason": "Judge could not produce valid structured output after retries",
1123 "raw": {"verdict_a": "error", "verdict_b": "error"},
1124 "error": "retry_exhausted",
1125 }
1127 if envelope.get("is_error", False):
1128 err_text = str(envelope.get("result", "") or "")[:200]
1129 return {
1130 "harness_pass": False,
1131 "baseline_pass": False,
1132 "confidence": 0.0,
1133 "reason": f"Judge reported error: {err_text}",
1134 "raw": {"verdict_a": "error", "verdict_b": "error"},
1135 "error": "api_error",
1136 }
1138 if isinstance(envelope.get("structured_output"), dict):
1139 result: dict[str, Any] = envelope["structured_output"]
1140 else:
1141 raw_result = envelope.get("result", "")
1142 if isinstance(raw_result, dict):
1143 result = raw_result
1144 elif raw_result:
1145 result = json.loads(raw_result)
1146 else:
1147 return {
1148 "harness_pass": False,
1149 "baseline_pass": False,
1150 "confidence": 0.0,
1151 "reason": "Empty result field in judge response",
1152 "raw": {"verdict_a": "error", "verdict_b": "error"},
1153 "error": "empty_result",
1154 }
1155 except (json.JSONDecodeError, TypeError, ValueError):
1156 return {
1157 "harness_pass": False,
1158 "baseline_pass": False,
1159 "confidence": 0.0,
1160 "reason": "Failed to parse judge response",
1161 "raw": {"verdict_a": "error", "verdict_b": "error"},
1162 "error": "parse_error",
1163 }
1165 # De-anonymize
1166 verdict_a = str(result.get("verdict_a", "no"))
1167 verdict_b = str(result.get("verdict_b", "no"))
1168 confidence = float(result.get("confidence", 0.0))
1169 reason = str(result.get("reason", ""))
1171 if harness_is_a:
1172 harness_pass = verdict_a == "yes"
1173 baseline_pass = verdict_b == "yes"
1174 else:
1175 harness_pass = verdict_b == "yes"
1176 baseline_pass = verdict_a == "yes"
1178 return {
1179 "harness_pass": harness_pass,
1180 "baseline_pass": baseline_pass,
1181 "confidence": confidence,
1182 "reason": reason,
1183 "raw": {"verdict_a": verdict_a, "verdict_b": verdict_b, "harness_is_a": harness_is_a},
1184 }
1187def evaluate_contract(
1188 config: EvaluateConfig,
1189 context: InterpolationContext,
1190 model: str = DEFAULT_LLM_MODEL,
1191 timeout: int = 1800,
1192) -> EvaluationResult:
1193 """Evaluate producer/consumer contract alignment using an LLM judge.
1195 Reads each producer/consumer file pair, applies optional regex extraction,
1196 then asks an LLM judge whether the producer satisfies the consumer contract.
1197 Returns yes only when all pairs align; any failure routes no/error.
1199 Args:
1200 config: EvaluateConfig with type="contract" and pairs list
1201 context: Interpolation context (unused by this evaluator directly)
1202 model: LLM model identifier
1203 timeout: Subprocess timeout in seconds
1205 Returns:
1206 EvaluationResult with verdict yes/no/error and pair_results in details
1207 """
1208 pairs = config.pairs
1209 if not pairs:
1210 return EvaluationResult(
1211 verdict="error",
1212 details={"error": "contract evaluator requires at least one pair in evaluate.pairs"},
1213 )
1215 contract_schema = {
1216 "type": "object",
1217 "properties": {
1218 "verdict": {"type": "string", "enum": ["yes", "no"]},
1219 "confidence": {"type": "number"},
1220 "reason": {"type": "string"},
1221 },
1222 "required": ["verdict", "confidence", "reason"],
1223 }
1225 pair_results: list[dict[str, Any]] = []
1227 for pair in pairs:
1228 producer_path = pair.get("producer", "")
1229 consumer_path = pair.get("consumer", "")
1230 producer_pattern = pair.get("producer_pattern")
1231 consumer_pattern = pair.get("consumer_pattern")
1232 contract_rule = pair.get("contract", "the producer and consumer must be compatible")
1234 # Read producer file
1235 try:
1236 producer_content = Path(producer_path).read_text()
1237 except OSError as e:
1238 pair_results.append(
1239 {
1240 "producer": producer_path,
1241 "consumer": consumer_path,
1242 "verdict": "error",
1243 "error": f"cannot read producer file: {e}",
1244 }
1245 )
1246 continue
1248 # Read consumer file
1249 try:
1250 consumer_content = Path(consumer_path).read_text()
1251 except OSError as e:
1252 pair_results.append(
1253 {
1254 "producer": producer_path,
1255 "consumer": consumer_path,
1256 "verdict": "error",
1257 "error": f"cannot read consumer file: {e}",
1258 }
1259 )
1260 continue
1262 # Apply optional regex extraction
1263 if producer_pattern:
1264 matches = re.findall(producer_pattern, producer_content, re.DOTALL)
1265 if not matches:
1266 pair_results.append(
1267 {
1268 "producer": producer_path,
1269 "consumer": consumer_path,
1270 "verdict": "error",
1271 "error": f"producer_pattern matched nothing in {producer_path}",
1272 }
1273 )
1274 continue
1275 producer_slice = "\n".join(matches)
1276 else:
1277 producer_slice = (
1278 producer_content[-4000:] if len(producer_content) > 4000 else producer_content
1279 )
1281 if consumer_pattern:
1282 matches = re.findall(consumer_pattern, consumer_content, re.DOTALL)
1283 if not matches:
1284 pair_results.append(
1285 {
1286 "producer": producer_path,
1287 "consumer": consumer_path,
1288 "verdict": "error",
1289 "error": f"consumer_pattern matched nothing in {consumer_path}",
1290 }
1291 )
1292 continue
1293 consumer_slice = "\n".join(matches)
1294 else:
1295 consumer_slice = (
1296 consumer_content[-4000:] if len(consumer_content) > 4000 else consumer_content
1297 )
1299 judge_prompt = (
1300 f"You are evaluating whether a producer output satisfies a consumer contract.\n\n"
1301 f"Contract rule: {contract_rule}\n\n"
1302 f'<producer path="{producer_path}">\n{producer_slice}\n</producer>\n\n'
1303 f'<consumer path="{consumer_path}">\n{consumer_slice}\n</consumer>\n\n'
1304 "Does the producer satisfy the consumer contract? "
1305 "Consider field names, types, casing, and structure. "
1306 "Answer yes if aligned, no if mismatched."
1307 )
1309 invocation = resolve_host().build_blocking_json(prompt=judge_prompt, model=model)
1310 args = list(invocation.args) + [
1311 "--json-schema",
1312 json.dumps(contract_schema),
1313 "--no-session-persistence",
1314 ]
1316 t0 = time.monotonic()
1317 try:
1318 proc = subprocess.run(
1319 [invocation.binary, *args], capture_output=True, text=True, timeout=timeout
1320 )
1321 except subprocess.TimeoutExpired:
1322 pair_results.append(
1323 {
1324 "producer": producer_path,
1325 "consumer": consumer_path,
1326 "verdict": "error",
1327 "error": "LLM judge timed out",
1328 "llm_latency_ms": int((time.monotonic() - t0) * 1000),
1329 }
1330 )
1331 continue
1332 except FileNotFoundError:
1333 return EvaluationResult(
1334 verdict="error",
1335 details={
1336 "error": f"{invocation.binary} CLI not found. Install the active host CLI (see LL_HOST_CLI).",
1337 "missing_dependency": True,
1338 },
1339 )
1340 llm_latency_ms = int((time.monotonic() - t0) * 1000)
1342 if proc.returncode != 0:
1343 pair_results.append(
1344 {
1345 "producer": producer_path,
1346 "consumer": consumer_path,
1347 "verdict": "error",
1348 "error": f"CLI error: {proc.stderr.strip()}",
1349 "llm_latency_ms": llm_latency_ms,
1350 }
1351 )
1352 continue
1354 if not proc.stdout.strip():
1355 pair_results.append(
1356 {
1357 "producer": producer_path,
1358 "consumer": consumer_path,
1359 "verdict": "error",
1360 "error": "CLI returned empty output",
1361 "llm_latency_ms": llm_latency_ms,
1362 }
1363 )
1364 continue
1366 try:
1367 stdout = proc.stdout.strip()
1368 try:
1369 envelope = json.loads(stdout)
1370 except json.JSONDecodeError:
1371 lines = [line for line in stdout.split("\n") if line.strip()]
1372 if not lines:
1373 raise
1374 envelope = json.loads(lines[-1])
1376 if envelope.get("subtype") == "error_max_structured_output_retries":
1377 pair_results.append(
1378 {
1379 "producer": producer_path,
1380 "consumer": consumer_path,
1381 "verdict": "error",
1382 "error": "Claude CLI could not produce valid structured output after retries",
1383 "llm_latency_ms": llm_latency_ms,
1384 }
1385 )
1386 continue
1388 if envelope.get("is_error", False):
1389 err_text = str(envelope.get("result", "") or "")[:200]
1390 pair_results.append(
1391 {
1392 "producer": producer_path,
1393 "consumer": consumer_path,
1394 "verdict": "error",
1395 "error": f"Claude CLI reported error: {err_text}",
1396 "llm_latency_ms": llm_latency_ms,
1397 }
1398 )
1399 continue
1401 if isinstance(envelope.get("structured_output"), dict):
1402 llm_result: dict[str, Any] = envelope["structured_output"]
1403 else:
1404 raw_result = envelope.get("result", "")
1405 if isinstance(raw_result, dict):
1406 llm_result = raw_result
1407 elif raw_result:
1408 llm_result = json.loads(raw_result)
1409 elif "verdict" in envelope:
1410 llm_result = envelope
1411 else:
1412 pair_results.append(
1413 {
1414 "producer": producer_path,
1415 "consumer": consumer_path,
1416 "verdict": "error",
1417 "error": "empty result field in CLI response",
1418 "llm_latency_ms": llm_latency_ms,
1419 }
1420 )
1421 continue
1423 except (json.JSONDecodeError, TypeError, ValueError) as e:
1424 pair_results.append(
1425 {
1426 "producer": producer_path,
1427 "consumer": consumer_path,
1428 "verdict": "error",
1429 "error": f"failed to parse LLM response: {e}",
1430 "llm_latency_ms": llm_latency_ms,
1431 }
1432 )
1433 continue
1435 pair_results.append(
1436 {
1437 "producer": producer_path,
1438 "consumer": consumer_path,
1439 "verdict": str(llm_result.get("verdict", "error")),
1440 "confidence": float(llm_result.get("confidence", 1.0)),
1441 "reason": llm_result.get("reason", ""),
1442 "llm_latency_ms": llm_latency_ms,
1443 }
1444 )
1446 # Aggregate: yes only if all pairs aligned; error takes precedence over no
1447 if any(p["verdict"] == "error" for p in pair_results):
1448 overall = "error"
1449 elif all(p["verdict"] == "yes" for p in pair_results):
1450 overall = "yes"
1451 else:
1452 overall = "no"
1454 return EvaluationResult(
1455 verdict=overall,
1456 details={"pair_results": pair_results},
1457 )
1460def evaluate_comparator(
1461 config: EvaluateConfig,
1462 output: str,
1463 context: InterpolationContext,
1464) -> EvaluationResult:
1465 """Evaluate using blind A/B comparison against a stored baseline."""
1466 from pathlib import Path
1468 if config.baseline_path is None:
1469 return EvaluationResult(
1470 verdict="no_baseline",
1471 details={"reason": "No baseline_path configured"},
1472 )
1474 baseline_file = Path(config.baseline_path) / "output.txt"
1475 if not baseline_file.exists():
1476 if config.auto_promote:
1477 baseline_file.parent.mkdir(parents=True, exist_ok=True)
1478 baseline_file.write_text(output)
1479 return EvaluationResult(
1480 verdict="yes",
1481 details={
1482 "reason": "No baseline found; current output promoted as new baseline.",
1483 "bootstrapped": True,
1484 },
1485 )
1486 return EvaluationResult(
1487 verdict="no_baseline",
1488 details={"reason": f"Baseline file not found: {baseline_file}"},
1489 )
1491 baseline_text = baseline_file.read_text()
1492 min_pairs = max(1, config.min_pairs if config.min_pairs is not None else 1)
1493 harness_wins = 0
1494 baseline_wins = 0
1495 last_reason = ""
1496 last_raw: dict[str, Any] = {}
1498 for _ in range(min_pairs):
1499 result = evaluate_blind_comparator(output, baseline_text, prompt=config.prompt)
1500 if result.get("harness_pass"):
1501 harness_wins += 1
1502 if result.get("baseline_pass"):
1503 baseline_wins += 1
1504 last_reason = result.get("reason", "")
1505 last_raw = result.get("raw", {})
1507 if harness_wins > baseline_wins:
1508 verdict = "yes"
1509 elif baseline_wins > harness_wins:
1510 verdict = "no"
1511 else:
1512 verdict = "tie"
1514 if config.auto_promote and verdict == "yes":
1515 baseline_file.write_text(output)
1517 return EvaluationResult(
1518 verdict=verdict,
1519 details={
1520 "harness_wins": harness_wins,
1521 "baseline_wins": baseline_wins,
1522 "min_pairs": min_pairs,
1523 "reason": last_reason,
1524 "raw": last_raw,
1525 },
1526 )
1529def evaluate(
1530 config: EvaluateConfig,
1531 output: str,
1532 exit_code: int,
1533 context: InterpolationContext,
1534) -> EvaluationResult:
1535 """Dispatch to appropriate evaluator based on config type.
1537 Args:
1538 config: Evaluator configuration with type and parameters
1539 output: Action stdout
1540 exit_code: Action exit code
1541 context: Runtime context for variable interpolation
1543 Returns:
1544 EvaluationResult from the appropriate evaluator
1546 Raises:
1547 ValueError: If evaluator type is unknown
1548 """
1549 eval_type = config.type
1551 # BUG-1640: Action-level timeouts (exit_code=124) short-circuit to "error"
1552 # so loop authors' on_error: branches fire instead of being routed via
1553 # on_no: based on truncated output. mcp_result is exempted because it has
1554 # its own established "timeout" verdict (see evaluate_mcp_result).
1555 if exit_code == 124 and eval_type != "mcp_result":
1556 return EvaluationResult(
1557 verdict="error",
1558 details={"exit_code": exit_code, "error": "action timed out"},
1559 )
1561 # BUG-1815: Non-timeout non-zero exit codes short-circuit to "error" for
1562 # evaluator types that don't intrinsically check exit codes. Exit-code-aware
1563 # evaluators (exit_code, mcp_result, harbor_scorer, diff_stall, llm_structured)
1564 # are exempt because they handle exit codes via their own logic.
1565 _EXIT_CODE_AWARE_EVALUATORS: frozenset[str] = frozenset(
1566 {
1567 "exit_code",
1568 "mcp_result",
1569 "harbor_scorer",
1570 "diff_stall",
1571 "action_stall",
1572 "llm_structured",
1573 "contract",
1574 }
1575 )
1576 if exit_code != 0 and eval_type not in _EXIT_CODE_AWARE_EVALUATORS:
1577 return EvaluationResult(
1578 verdict="error",
1579 details={
1580 "exit_code": exit_code,
1581 "error": f"action exited with code {exit_code}",
1582 },
1583 )
1585 if eval_type == "exit_code":
1586 return evaluate_exit_code(exit_code)
1588 elif eval_type == "output_numeric":
1589 if config.target is None:
1590 raise ValueError("output_numeric evaluator requires 'target' to be set")
1591 elif isinstance(config.target, str):
1592 try:
1593 resolved = interpolate(config.target, context) if context else config.target
1594 numeric_target = float(resolved)
1595 except (InterpolationError, ValueError) as e:
1596 raise ValueError(
1597 f"output_numeric target must be numeric, got: {config.target!r}"
1598 ) from e
1599 else:
1600 numeric_target = float(config.target)
1601 return evaluate_output_numeric(
1602 output=output,
1603 operator=config.operator or "eq",
1604 target=numeric_target,
1605 )
1607 elif eval_type == "output_json":
1608 return evaluate_output_json(
1609 output=output,
1610 path=config.path or "",
1611 operator=config.operator or "eq",
1612 target=config.target,
1613 )
1615 elif eval_type == "output_contains":
1616 return evaluate_output_contains(
1617 output=output,
1618 pattern=config.pattern or "",
1619 negate=config.negate,
1620 error_patterns=config.error_patterns,
1621 )
1623 elif eval_type == "convergence":
1624 # Resolve previous value from interpolation if configured
1625 previous: float | None = None
1626 if config.previous:
1627 try:
1628 previous = float(interpolate(config.previous, context))
1629 except (InterpolationError, ValueError):
1630 # Previous unavailable on first iteration, continue with None
1631 pass
1633 # Parse current value from output
1634 try:
1635 current = float(output.strip())
1636 except ValueError:
1637 return EvaluationResult(
1638 verdict="error",
1639 details={"error": f"Cannot parse output as number: {output[:100]}"},
1640 )
1642 # Resolve target (may be interpolated string like "${context.target}")
1643 convergence_target: float
1644 if isinstance(config.target, str):
1645 try:
1646 convergence_target = float(interpolate(config.target, context))
1647 except (InterpolationError, ValueError) as e:
1648 return EvaluationResult(
1649 verdict="error",
1650 details={"error": f"Cannot resolve target: {e}"},
1651 )
1652 else:
1653 if config.target is None:
1654 raise ValueError("convergence evaluator requires 'target' to be set")
1655 convergence_target = float(config.target)
1657 # Resolve tolerance (may be interpolated string)
1658 tolerance: float = 0.0
1659 if config.tolerance is not None:
1660 if isinstance(config.tolerance, str):
1661 try:
1662 tolerance = float(interpolate(config.tolerance, context))
1663 except (InterpolationError, ValueError):
1664 tolerance = 0.0
1665 else:
1666 tolerance = float(config.tolerance)
1668 return evaluate_convergence(
1669 current=current,
1670 previous=previous,
1671 target=convergence_target,
1672 tolerance=tolerance,
1673 direction=config.direction,
1674 )
1676 elif eval_type == "diff_stall":
1677 return evaluate_diff_stall(
1678 scope=config.scope,
1679 max_stall=config.max_stall,
1680 )
1682 elif eval_type == "action_stall":
1683 return evaluate_action_stall(
1684 track=config.track,
1685 max_repeat=config.max_repeat,
1686 context=context,
1687 )
1689 elif eval_type == "llm_structured":
1690 prompt = config.prompt
1691 if prompt and context:
1692 try:
1693 prompt = interpolate(prompt, context)
1694 except InterpolationError:
1695 pass # Use raw prompt on resolution failure
1696 return evaluate_llm_structured(
1697 output=output,
1698 prompt=prompt,
1699 schema=config.schema,
1700 min_confidence=config.min_confidence,
1701 uncertain_suffix=config.uncertain_suffix,
1702 )
1704 elif eval_type == "mcp_result":
1705 return evaluate_mcp_result(output=output, exit_code=exit_code)
1707 elif eval_type == "harbor_scorer":
1708 return evaluate_harbor_scorer(output=output, exit_code=exit_code)
1710 elif eval_type == "comparator":
1711 return evaluate_comparator(config=config, output=output, context=context)
1713 elif eval_type == "contract":
1714 return evaluate_contract(config=config, context=context)
1716 elif eval_type == "classify":
1717 return evaluate_classify(output=output, line=config.line)
1719 else:
1720 raise ValueError(f"Unknown evaluator type: {eval_type}")