Coverage for little_loops / fsm / evaluators.py: 7%
539 statements
« prev ^ index » next coverage.py v7.12.0, created at 2026-06-26 17:38 -0500
1"""FSM Evaluators for loop execution.
3This module provides evaluators that interpret action output and produce
4verdicts for state transitions.
6Supported evaluator types:
8Tier 1 (Deterministic - no API calls):
9 exit_code: Map Unix exit codes to verdicts (0=success, 1=failure, 2+=error)
10 output_numeric: Compare numeric output to target value
11 output_json: Extract and compare JSON path values
12 output_contains: Pattern matching on stdout
13 convergence: Track progress toward a target value
14 diff_stall: Detect stalled iterations via git diff comparison
15 action_stall: Detect when the same action string or output repeats for N consecutive iterations
16 harbor_scorer: Interpret Harbor-format benchmark scorer exit code and float stdout
18Tier 2 (LLM-based):
19 llm_structured: Use LLM with structured output for natural language evaluation
20 contract: Read producer/consumer file pairs and assert contract alignment via LLM judge
22Tier 3 (External process):
23 mcp_result: Parse MCP tool call response envelope
24"""
26from __future__ import annotations
28import hashlib
29import json
30import random
31import re
32import subprocess
33import time
34from collections.abc import Callable
35from dataclasses import dataclass
36from pathlib import Path
37from typing import Any
39from little_loops.fsm.interpolation import (
40 InterpolationContext,
41 InterpolationError,
42 interpolate,
43)
44from little_loops.fsm.schema import DEFAULT_LLM_MODEL, EvaluateConfig
45from little_loops.host_runner import resolve_host
48@dataclass
49class EvaluationResult:
50 """Result from an evaluator.
52 Attributes:
53 verdict: The routing key for state transitions
54 details: Evaluator-specific metadata for debugging/logging
55 """
57 verdict: str
58 details: dict[str, Any]
61# Default schema for LLM structured evaluation
62DEFAULT_LLM_SCHEMA: dict[str, Any] = {
63 "type": "object",
64 "properties": {
65 "verdict": {
66 "type": "string",
67 "enum": ["yes", "no", "blocked", "partial"],
68 "description": (
69 "- yes: The condition/check evaluated to true\n"
70 "- no: The condition/check evaluated to false\n"
71 "- blocked: Cannot proceed without external help\n"
72 "- partial: Made progress but not complete"
73 ),
74 },
75 "confidence": {
76 "type": "number",
77 "minimum": 0,
78 "maximum": 1,
79 "description": "Confidence in this verdict (0-1)",
80 },
81 "reason": {
82 "type": "string",
83 "description": "Brief explanation",
84 },
85 },
86 "required": ["verdict", "confidence", "reason"],
87}
89DEFAULT_LLM_PROMPT = "Evaluate whether this action succeeded based on its output."
91# Schema for blind A/B comparator: evaluates two anonymized outputs
92BLIND_COMPARATOR_SCHEMA: dict[str, Any] = {
93 "type": "object",
94 "properties": {
95 "verdict_a": {
96 "type": "string",
97 "enum": ["yes", "no"],
98 "description": "Whether Output A meets the evaluation criteria",
99 },
100 "verdict_b": {
101 "type": "string",
102 "enum": ["yes", "no"],
103 "description": "Whether Output B meets the evaluation criteria",
104 },
105 "confidence": {
106 "type": "number",
107 "minimum": 0,
108 "maximum": 1,
109 "description": "Confidence in these verdicts (0-1)",
110 },
111 "reason": {
112 "type": "string",
113 "description": "Brief explanation comparing the two outputs",
114 },
115 },
116 "required": ["verdict_a", "verdict_b", "confidence", "reason"],
117}
119DEFAULT_BLIND_COMPARATOR_PROMPT = (
120 "You are evaluating two outputs (labeled 'Output A' and 'Output B') that were "
121 "produced by independent runs of the same task. Judge whether each output meets "
122 "the evaluation criteria below. Be objective and impartial — the labels 'A' and "
123 "'B' are arbitrary and do not indicate which is better."
124)
126_NUMERIC_OPERATORS: dict[str, Callable[[float, float], bool]] = {
127 "eq": lambda v, t: v == t,
128 "ne": lambda v, t: v != t,
129 "lt": lambda v, t: v < t,
130 "le": lambda v, t: v <= t,
131 "gt": lambda v, t: v > t,
132 "ge": lambda v, t: v >= t,
133}
136def evaluate_exit_code(exit_code: int) -> EvaluationResult:
137 """Map Unix exit code to verdict.
139 Args:
140 exit_code: The process exit code
142 Returns:
143 EvaluationResult with verdict:
144 - 0 -> yes
145 - 1 -> no
146 - 2+ -> error
147 """
148 if exit_code == 0:
149 verdict = "yes"
150 elif exit_code == 1:
151 verdict = "no"
152 else:
153 verdict = "error"
155 return EvaluationResult(verdict=verdict, details={"exit_code": exit_code})
158def evaluate_output_numeric(
159 output: str,
160 operator: str,
161 target: float,
162) -> EvaluationResult:
163 """Parse stdout as number and compare to target.
165 Args:
166 output: The action stdout to parse as a number
167 operator: Comparison operator (eq, ne, lt, le, gt, ge)
168 target: Target value to compare against
170 Returns:
171 EvaluationResult with verdict:
172 - Condition met -> yes
173 - Condition not met -> no
174 - Parse error -> error
175 """
176 try:
177 value = float(output.strip())
178 except ValueError:
179 return EvaluationResult(
180 verdict="error",
181 details={"error": f"Cannot parse as number: {output[:100]}"},
182 )
184 if operator not in _NUMERIC_OPERATORS:
185 return EvaluationResult(
186 verdict="error",
187 details={"error": f"Unknown operator: {operator}"},
188 )
190 condition_met = _NUMERIC_OPERATORS[operator](value, target)
191 return EvaluationResult(
192 verdict="yes" if condition_met else "no",
193 details={"value": value, "target": target, "operator": operator},
194 )
197def _extract_json_path(data: Any, path: str) -> Any:
198 """Extract value from dict using jq-style path like '.summary.failed'.
200 Args:
201 data: The parsed JSON data (dict or list)
202 path: Dot-separated path, optionally starting with '.'
204 Returns:
205 The value at the specified path
207 Raises:
208 KeyError: If path not found in data
209 """
210 if path.startswith("."):
211 path = path[1:]
212 parts = path.split(".")
213 current = data
214 for part in parts:
215 if isinstance(current, dict) and part in current:
216 current = current[part]
217 elif isinstance(current, list) and part.isdigit():
218 idx = int(part)
219 if 0 <= idx < len(current):
220 current = current[idx]
221 else:
222 raise KeyError(path)
223 else:
224 raise KeyError(path)
225 return current
228def _compare_values(
229 value: int | float, operator: str, target: int | float, path: str
230) -> EvaluationResult:
231 """Compare numeric values using operator.
233 Args:
234 value: The extracted value to compare
235 operator: Comparison operator
236 target: Target value
237 path: JSON path for details
239 Returns:
240 EvaluationResult with comparison result
241 """
242 if operator not in _NUMERIC_OPERATORS:
243 return EvaluationResult(
244 verdict="error",
245 details={"error": f"Unknown operator: {operator}"},
246 )
248 condition_met = _NUMERIC_OPERATORS[operator](value, target)
249 return EvaluationResult(
250 verdict="yes" if condition_met else "no",
251 details={"value": value, "path": path, "target": target, "operator": operator},
252 )
255def evaluate_output_json(
256 output: str,
257 path: str,
258 operator: str,
259 target: Any,
260) -> EvaluationResult:
261 """Parse JSON and extract value at path, then compare.
263 Args:
264 output: The action stdout containing JSON
265 path: jq-style dot notation path (e.g., '.summary.failed')
266 operator: Comparison operator (eq, ne, lt, le, gt, ge)
267 target: Target value for comparison
269 Returns:
270 EvaluationResult with verdict:
271 - Condition met -> yes
272 - Condition not met -> no
273 - Parse/path error -> error
274 """
275 try:
276 data = json.loads(output)
277 except json.JSONDecodeError as e:
278 return EvaluationResult(
279 verdict="error",
280 details={"error": f"Invalid JSON: {e}"},
281 )
283 try:
284 value = _extract_json_path(data, path)
285 except KeyError:
286 return EvaluationResult(
287 verdict="error",
288 details={"error": f"Path not found: {path}"},
289 )
291 # Use numeric comparison if both values are numeric
292 if isinstance(value, (int, float)) and isinstance(target, (int, float)):
293 return _compare_values(value, operator, target, path)
295 # For non-numeric values, only eq and ne are supported
296 if operator == "eq":
297 verdict = "yes" if value == target else "no"
298 elif operator == "ne":
299 verdict = "yes" if value != target else "no"
300 else:
301 return EvaluationResult(
302 verdict="error",
303 details={"error": f"Operator {operator} not supported for non-numeric values"},
304 )
306 return EvaluationResult(
307 verdict=verdict,
308 details={"value": value, "path": path, "target": target, "operator": operator},
309 )
312def evaluate_output_contains(
313 output: str,
314 pattern: str,
315 negate: bool = False,
316 error_patterns: list[str] | None = None,
317) -> EvaluationResult:
318 """Check if pattern exists in output.
320 Pattern can be regex or substring. If regex fails to compile,
321 falls back to substring matching.
323 Args:
324 output: The action stdout to search
325 pattern: Regex pattern or substring
326 negate: If True, invert the match result
327 error_patterns: Optional list of substrings that, when matched in output
328 and the main pattern is not found, yield verdict="error". This allows
329 loops to route auth/error output to on_error without raising an exception.
331 Returns:
332 EvaluationResult with verdict:
333 - Found (negate=False) -> yes
334 - Found (negate=True) -> no
335 - Not found (negate=False) -> no (or "error" if error_patterns matched)
336 - Not found (negate=True) -> yes
337 """
338 # Try regex first, fall back to substring
339 try:
340 matched = bool(re.search(pattern, output))
341 except re.error:
342 matched = pattern in output
344 if negate:
345 verdict = "no" if matched else "yes"
346 else:
347 verdict = "yes" if matched else "no"
349 # Check error_patterns before returning "no" — only when main pattern didn't match
350 if verdict == "no" and not negate and error_patterns:
351 for ep in error_patterns:
352 if ep in output:
353 return EvaluationResult(
354 verdict="error",
355 details={
356 "matched": False,
357 "pattern": pattern,
358 "negate": negate,
359 "error_pattern": ep,
360 },
361 )
363 return EvaluationResult(
364 verdict=verdict,
365 details={"matched": matched, "pattern": pattern, "negate": negate},
366 )
369def evaluate_convergence(
370 current: float,
371 previous: float | None,
372 target: float,
373 tolerance: float = 0,
374 direction: str = "minimize",
375) -> EvaluationResult:
376 """Compare current value to target and previous.
378 Args:
379 current: Current metric value
380 previous: Previous metric value (None if first iteration)
381 target: Target value to reach
382 tolerance: Acceptable distance from target
383 direction: 'minimize' or 'maximize'
385 Returns:
386 EvaluationResult with verdict:
387 - Value within tolerance of target -> target
388 - Value improved toward target -> progress
389 - Value unchanged or worsened -> stall
390 """
391 # Check if target reached (within tolerance)
392 if abs(current - target) <= tolerance:
393 return EvaluationResult(
394 verdict="target",
395 details={"current": current, "target": target, "delta": 0},
396 )
398 # First iteration has no previous value
399 if previous is None:
400 return EvaluationResult(
401 verdict="progress",
402 details={
403 "current": current,
404 "previous": None,
405 "target": target,
406 "delta": None,
407 },
408 )
410 # Calculate progress
411 delta = current - previous
413 if direction == "minimize":
414 # For minimizing, negative delta is progress
415 made_progress = delta < 0
416 else:
417 # For maximizing, positive delta is progress
418 made_progress = delta > 0
420 verdict = "progress" if made_progress else "stall"
422 return EvaluationResult(
423 verdict=verdict,
424 details={
425 "current": current,
426 "previous": previous,
427 "target": target,
428 "delta": delta,
429 "direction": direction,
430 },
431 )
434def evaluate_classify(
435 output: str,
436 line: str | int | None = None,
437) -> EvaluationResult:
438 """Read a token from stdout and return it as the verdict.
440 Intended for single-state multi-way routing: the action prints exactly one
441 token to stdout and the route: table maps that token to the next state.
443 Args:
444 output: The action stdout to read the token from
445 line: Which line to select. 'last' (default) picks the last non-empty
446 line; 'first' picks the first non-empty line; an integer index
447 selects that line (0-based, negative indices supported).
449 Returns:
450 EvaluationResult with verdict = trimmed token, or "" when output is
451 empty (which _route() maps to the route.default fallback).
452 """
453 lines = [ln for ln in output.splitlines() if ln.strip()]
454 if not lines:
455 return EvaluationResult(
456 verdict="",
457 details={"token": "", "line": line, "source_lines": 0},
458 )
460 selector = line if line is not None else "last"
461 if selector == "last":
462 selected = lines[-1]
463 elif selector == "first":
464 selected = lines[0]
465 elif isinstance(selector, int):
466 try:
467 selected = lines[selector]
468 except IndexError:
469 return EvaluationResult(
470 verdict="",
471 details={
472 "token": "",
473 "line": line,
474 "source_lines": len(lines),
475 "error": "index out of range",
476 },
477 )
478 else:
479 selected = lines[-1]
481 token = selected.strip()
482 return EvaluationResult(
483 verdict=token,
484 details={"token": token, "line": line},
485 )
488def evaluate_diff_stall(
489 scope: list[str] | None = None,
490 max_stall: int = 1,
491) -> EvaluationResult:
492 """Detect stalled iterations by comparing git diff --stat between runs.
494 On first call, snapshots the current diff and returns 'yes'.
495 On subsequent calls, compares current diff to the previous snapshot.
496 If the diff is identical for max_stall consecutive iterations, returns
497 'no' (stalled). If different, resets the stall counter and returns
498 'yes' (progress).
500 State is persisted in /tmp using a key derived from the scope argument,
501 so different loops with different scopes maintain independent stall counters.
503 Args:
504 scope: Optional list of paths to limit the git diff to. Defaults to
505 the entire working tree.
506 max_stall: Number of consecutive no-change iterations before stall
507 verdict. Defaults to 1.
509 Returns:
510 EvaluationResult with verdict:
511 - yes: diff changed since last iteration (progress made)
512 - no: diff unchanged for max_stall iterations (stalled)
513 - error: git command failed or timed out
514 """
515 cmd = ["git", "diff", "--stat"]
516 if scope:
517 cmd += ["--"] + scope
519 try:
520 proc = subprocess.run(cmd, capture_output=True, text=True, timeout=30)
521 except subprocess.TimeoutExpired:
522 return EvaluationResult(verdict="error", details={"error": "git diff timed out"})
523 except FileNotFoundError:
524 return EvaluationResult(verdict="error", details={"error": "git not found in PATH"})
526 if proc.returncode != 0:
527 return EvaluationResult(
528 verdict="error",
529 details={"error": f"git diff failed: {proc.stderr[:200]}"},
530 )
532 current_diff = proc.stdout
534 # Derive a stable cache key from the scope so independent loops don't collide
535 scope_str = "|".join(sorted(scope)) if scope else "_root_"
536 cache_key = hashlib.md5(scope_str.encode()).hexdigest()[:12]
537 loops_tmp = Path.cwd() / ".loops" / "tmp"
538 loops_tmp.mkdir(parents=True, exist_ok=True)
539 state_file = loops_tmp / f"ll-diff-stall-{cache_key}.txt"
540 count_file = loops_tmp / f"ll-diff-stall-{cache_key}.count"
542 # Read previous snapshot and stall count
543 previous_diff: str | None = None
544 stall_count = 0
545 try:
546 previous_diff = state_file.read_text()
547 stall_count = int(count_file.read_text().strip())
548 except (FileNotFoundError, ValueError):
549 pass
551 # First iteration: save snapshot and report progress
552 if previous_diff is None:
553 state_file.write_text(current_diff)
554 count_file.write_text("0")
555 return EvaluationResult(
556 verdict="yes",
557 details={"stall_count": 0, "max_stall": max_stall, "diff_changed": True},
558 )
560 if current_diff == previous_diff:
561 stall_count += 1
562 count_file.write_text(str(stall_count))
563 if stall_count >= max_stall:
564 return EvaluationResult(
565 verdict="no",
566 details={"stall_count": stall_count, "max_stall": max_stall, "diff_changed": False},
567 )
568 # Not yet at max_stall threshold — still report yes so loop continues
569 return EvaluationResult(
570 verdict="yes",
571 details={"stall_count": stall_count, "max_stall": max_stall, "diff_changed": False},
572 )
573 else:
574 # Progress: update snapshot and reset counter
575 state_file.write_text(current_diff)
576 count_file.write_text("0")
577 return EvaluationResult(
578 verdict="yes",
579 details={"stall_count": 0, "max_stall": max_stall, "diff_changed": True},
580 )
583def evaluate_action_stall(
584 track: list[str] | None = None,
585 max_repeat: int = 2,
586 context: InterpolationContext | None = None,
587) -> EvaluationResult:
588 """Detect when the same action string or output repeats for N consecutive iterations.
590 On first call, snapshots the hashed values of the tracked context keys and returns
591 'yes'. On subsequent calls, compares the current hash to the previous snapshot.
592 If the hash is identical for max_repeat consecutive iterations, returns 'no'
593 (stalled). If different, resets the stall counter and returns 'yes' (progress).
595 State is persisted in .loops/tmp using a key derived from the tracked keys,
596 so different states/loops maintain independent stall counters.
598 Args:
599 track: Context keys to track. Defaults to ["action"] when None.
600 max_repeat: Number of consecutive identical-hash iterations before stall verdict.
601 Defaults to 2.
602 context: Runtime interpolation context for resolving tracked keys.
604 Returns:
605 EvaluationResult with verdict:
606 - yes: tracked values changed since last iteration (progress made)
607 - no: tracked values identical for max_repeat iterations (stalled)
608 """
609 effective_track: list[str] = track if track is not None else ["action"]
611 # Resolve each tracked key from context and hash the combined values.
612 # Keys may be bare names (e.g. "action") or namespaced (e.g. "context.action").
613 # Try namespaced forms first: context.<key>, captured.<key>, then bare ${key}.
614 parts: list[str] = []
615 for key in effective_track:
616 value: str = ""
617 if context is not None:
618 # If key already contains a dot it's already namespaced; use as-is.
619 if "." in key:
620 try:
621 value = str(interpolate(f"${{{key}}}", context))
622 except InterpolationError:
623 value = ""
624 else:
625 # Try context.<key> first, then captured.<key>, then give up.
626 resolved = False
627 for namespace in ("context", "captured", "prev", "result"):
628 try:
629 value = str(interpolate(f"${{{namespace}.{key}}}", context))
630 resolved = True
631 break
632 except InterpolationError:
633 continue
634 if not resolved:
635 value = ""
636 parts.append(f"{key}={value}")
638 combined = "|".join(parts)
639 current_hash = hashlib.md5(combined.encode()).hexdigest()
641 # Derive a stable cache key from the tracked keys
642 track_str = "|".join(sorted(effective_track))
643 cache_key = hashlib.md5(track_str.encode()).hexdigest()[:12]
644 loops_tmp = Path.cwd() / ".loops" / "tmp"
645 loops_tmp.mkdir(parents=True, exist_ok=True)
646 state_file = loops_tmp / f"ll-action-stall-{cache_key}.txt"
647 count_file = loops_tmp / f"ll-action-stall-{cache_key}.count"
649 # Read previous hash and stall count
650 previous_hash: str | None = None
651 stall_count = 0
652 try:
653 previous_hash = state_file.read_text().strip()
654 stall_count = int(count_file.read_text().strip())
655 except (FileNotFoundError, ValueError):
656 pass
658 # First iteration: save hash and report progress
659 if previous_hash is None:
660 state_file.write_text(current_hash)
661 count_file.write_text("0")
662 return EvaluationResult(
663 verdict="yes",
664 details={
665 "stall_count": 0,
666 "max_repeat": max_repeat,
667 "hash_changed": True,
668 "tracked_keys": effective_track,
669 },
670 )
672 hash_changed = current_hash != previous_hash
674 if hash_changed:
675 # Progress: update snapshot and reset counter
676 state_file.write_text(current_hash)
677 count_file.write_text("0")
678 return EvaluationResult(
679 verdict="yes",
680 details={
681 "stall_count": 0,
682 "max_repeat": max_repeat,
683 "hash_changed": True,
684 "tracked_keys": effective_track,
685 },
686 )
687 else:
688 # Same hash as last time
689 stall_count += 1
690 count_file.write_text(str(stall_count))
691 if stall_count >= max_repeat:
692 return EvaluationResult(
693 verdict="no",
694 details={
695 "stall_count": stall_count,
696 "max_repeat": max_repeat,
697 "hash_changed": False,
698 "tracked_keys": effective_track,
699 "repeated_hash": current_hash,
700 },
701 )
702 # Not yet at max_repeat threshold — still report yes so loop continues
703 return EvaluationResult(
704 verdict="yes",
705 details={
706 "stall_count": stall_count,
707 "max_repeat": max_repeat,
708 "hash_changed": False,
709 "tracked_keys": effective_track,
710 },
711 )
714def evaluate_mcp_result(output: str, exit_code: int) -> EvaluationResult:
715 """Evaluate an MCP tool call result from the mcp-call subprocess.
717 Maps exit codes and MCP response envelope fields to routing verdicts.
719 Exit code conventions (set by mcp-call):
720 0 → parse isError from JSON envelope
721 1 → tool_error (tool ran but isError: true)
722 124 → timeout (transport-level timeout)
723 127 → not_found (server or tool missing from .mcp.json)
725 Args:
726 output: stdout from mcp-call (MCP response envelope JSON)
727 exit_code: Exit code from mcp-call subprocess
729 Returns:
730 EvaluationResult with verdict:
731 - success → isError: false
732 - tool_error → isError: true
733 - not_found → server/tool not in .mcp.json (exit 127)
734 - timeout → transport-level timeout (exit 124)
735 """
736 if exit_code == 127:
737 return EvaluationResult(
738 verdict="not_found",
739 details={"exit_code": exit_code, "error": "Server or tool not found in .mcp.json"},
740 )
742 if exit_code == 124:
743 return EvaluationResult(
744 verdict="timeout",
745 details={"exit_code": exit_code, "error": "MCP tool call timed out"},
746 )
748 # Parse MCP envelope JSON from stdout
749 try:
750 envelope = json.loads(output.strip()) if output.strip() else {}
751 except json.JSONDecodeError:
752 return EvaluationResult(
753 verdict="tool_error",
754 details={
755 "exit_code": exit_code,
756 "error": f"Invalid JSON from mcp-call: {output[:200]}",
757 },
758 )
760 is_error = envelope.get("isError", exit_code != 0)
762 if is_error:
763 return EvaluationResult(
764 verdict="tool_error",
765 details={"exit_code": exit_code, "envelope": envelope},
766 )
768 return EvaluationResult(
769 verdict="success",
770 details={"exit_code": exit_code, "envelope": envelope},
771 )
774def evaluate_harbor_scorer(output: str, exit_code: int) -> EvaluationResult:
775 """Evaluate a Harbor-format benchmark scorer result.
777 The scorer is a shell command that prints a float score (0.0–1.0) to stdout
778 and exits 0 on success or non-zero on failure.
780 Args:
781 output: stdout from the scorer subprocess (expected: a bare float)
782 exit_code: Exit code from the scorer subprocess
784 Returns:
785 EvaluationResult with verdict:
786 - yes → exit 0 and stdout parses as a float
787 - no → exit non-zero (scorer determined failure)
788 - error → exit 0 but stdout is not parseable as a float
789 """
790 if exit_code != 0:
791 return EvaluationResult(
792 verdict="no",
793 details={"exit_code": exit_code},
794 )
796 try:
797 score = float(output.strip())
798 except (ValueError, AttributeError):
799 return EvaluationResult(
800 verdict="error",
801 details={
802 "exit_code": exit_code,
803 "error": f"Scorer stdout is not a float: {output[:200]}",
804 },
805 )
807 return EvaluationResult(
808 verdict="yes",
809 details={"score": score, "exit_code": 0},
810 )
813def evaluate_llm_structured(
814 output: str,
815 prompt: str | None = None,
816 schema: dict[str, Any] | None = None,
817 min_confidence: float = 0.5,
818 uncertain_suffix: bool = False,
819 model: str = DEFAULT_LLM_MODEL,
820 max_tokens: int = 256,
821 timeout: int = 1800,
822) -> EvaluationResult:
823 """Evaluate action output using LLM with structured output via Claude CLI.
825 This is the ONLY place in the FSM system that uses LLM structured output.
826 Requires the ``claude`` CLI to be installed and authenticated.
828 Args:
829 output: Action stdout to evaluate
830 prompt: Custom evaluation prompt (defaults to basic success check)
831 schema: Custom JSON schema for structured response
832 min_confidence: Minimum confidence threshold (0-1)
833 uncertain_suffix: If True, append _uncertain to low-confidence verdicts
834 model: Model identifier (CLI aliases like "sonnet" or full names)
835 max_tokens: Maximum tokens for response (passed to --max-turns is not
836 applicable; kept for signature compat)
837 timeout: Timeout in seconds
839 Returns:
840 EvaluationResult with verdict from LLM and confidence/reason in details
841 """
842 effective_schema = schema or DEFAULT_LLM_SCHEMA
843 effective_prompt = prompt or DEFAULT_LLM_PROMPT
845 # Truncate output to avoid context limits (keep last 4000 chars)
846 truncated = output[-4000:] if len(output) > 4000 else output
848 user_prompt = f"{effective_prompt}\n\n<action_output>\n{truncated}\n</action_output>"
850 invocation = resolve_host().build_blocking_json(prompt=user_prompt, model=model)
851 # Builder drops json_schema (Protocol surface only) and omits the
852 # claude-CLI-specific --no-session-persistence flag; augment at call site.
853 args = list(invocation.args) + [
854 "--json-schema",
855 json.dumps(effective_schema),
856 "--no-session-persistence",
857 ]
859 t0 = time.monotonic()
860 try:
861 proc = subprocess.run(
862 [invocation.binary, *args], capture_output=True, text=True, timeout=timeout
863 )
864 except subprocess.TimeoutExpired:
865 return EvaluationResult(
866 verdict="error",
867 details={"error": "LLM evaluation timeout", "timeout": True},
868 )
869 except FileNotFoundError:
870 return EvaluationResult(
871 verdict="error",
872 details={
873 "error": f"{invocation.binary} CLI not found. Install the active host CLI (see LL_HOST_CLI).",
874 "missing_dependency": True,
875 },
876 )
877 llm_latency_ms = int((time.monotonic() - t0) * 1000)
879 if proc.returncode != 0:
880 return EvaluationResult(
881 verdict="error",
882 details={
883 "error": f"{invocation.binary} CLI error: {proc.stderr.strip()}",
884 "api_error": True,
885 },
886 )
888 # Guard: empty stdout with exit 0 (API error not reflected in exit code)
889 if not proc.stdout.strip():
890 stderr_info = proc.stderr.strip()[:200] if proc.stderr else ""
891 error_msg = f"{invocation.binary} CLI returned empty output"
892 if stderr_info:
893 error_msg += f" (stderr: {stderr_info})"
894 return EvaluationResult(
895 verdict="error",
896 details={"error": error_msg, "empty_output": True},
897 )
899 # Parse the CLI JSON envelope and extract structured result.
900 # With --json-schema the envelope is:
901 # success: {"type":"result","subtype":"success","structured_output":{...},...}
902 # failure: {"type":"result","subtype":"error_max_structured_output_retries",...}
903 # If stdout is JSONL (multiple JSON objects), use the last non-empty line.
904 try:
905 stdout = proc.stdout.strip()
906 try:
907 envelope = json.loads(stdout)
908 except json.JSONDecodeError:
909 # Try JSONL: take the last non-empty line
910 lines = [line for line in stdout.split("\n") if line.strip()]
911 if not lines:
912 raise
913 envelope = json.loads(lines[-1])
915 # Check structured-output retry exhaustion (--json-schema failure mode)
916 if envelope.get("subtype") == "error_max_structured_output_retries":
917 return EvaluationResult(
918 verdict="error",
919 details={
920 "error": "Claude CLI could not produce valid structured output after retries",
921 "api_error": True,
922 },
923 )
925 # Check legacy is_error flag (some CLI versions exit 0 but report error in envelope)
926 if envelope.get("is_error", False):
927 err_text = str(envelope.get("result", "") or "")[:200]
928 return EvaluationResult(
929 verdict="error",
930 details={"error": f"Claude CLI reported error: {err_text}", "api_error": True},
931 )
933 # --json-schema mode returns validated dict in "structured_output"
934 if isinstance(envelope.get("structured_output"), dict):
935 llm_result: dict[str, Any] = envelope["structured_output"]
936 else:
937 raw_result = envelope.get("result", "")
938 if isinstance(raw_result, dict):
939 llm_result = raw_result
940 elif raw_result:
941 llm_result = json.loads(raw_result)
942 elif "verdict" in envelope:
943 llm_result = envelope
944 else:
945 raw_preview = proc.stdout[:300]
946 return EvaluationResult(
947 verdict="error",
948 details={
949 "error": "Empty result field in Claude CLI response",
950 "raw_preview": raw_preview,
951 },
952 )
953 except (json.JSONDecodeError, TypeError, ValueError) as e:
954 raw_preview = proc.stdout[:300] if proc.stdout else "(empty)"
955 return EvaluationResult(
956 verdict="error",
957 details={"error": f"Failed to parse LLM response: {e}", "raw_preview": raw_preview},
958 )
960 # Build result with confidence handling
961 verdict = str(llm_result.get("verdict", "error"))
962 confidence = float(llm_result.get("confidence", 1.0))
963 confident = confidence >= min_confidence
965 # Optionally modify verdict for low confidence
966 if uncertain_suffix and not confident:
967 verdict = f"{verdict}_uncertain"
969 return EvaluationResult(
970 verdict=verdict,
971 details={
972 "confidence": confidence,
973 "confident": confident,
974 "reason": llm_result.get("reason", ""),
975 "raw": llm_result,
976 "llm_model": model,
977 "llm_latency_ms": llm_latency_ms,
978 "llm_prompt": user_prompt[:500],
979 "llm_raw_output": proc.stdout[:500] if proc.stdout else "",
980 },
981 )
984def evaluate_blind_comparator(
985 output_harness: str,
986 output_baseline: str,
987 prompt: str | None = None,
988 model: str = DEFAULT_LLM_MODEL,
989 timeout: int = 1800,
990) -> dict[str, Any]:
991 """Blindly evaluate two outputs, returning pass/fail for each arm.
993 Outputs are randomly labeled "Output A" / "Output B" so the LLM judge
994 cannot distinguish the harness arm from the baseline arm. The mapping is
995 de-anonymized after judgment so callers receive harness/baseline verdicts.
997 Args:
998 output_harness: stdout from the harness (gated) arm
999 output_baseline: stdout from the baseline (ungated) arm
1000 prompt: Custom evaluation prompt (appended to default framing)
1001 model: Model identifier for the judge
1002 timeout: Timeout in seconds
1004 Returns:
1005 Dict with keys: harness_pass (bool), baseline_pass (bool),
1006 confidence (float), reason (str), raw (dict with A/B verdicts)
1007 """
1008 effective_prompt = prompt or DEFAULT_BLIND_COMPARATOR_PROMPT
1010 # Truncate outputs to avoid context limits
1011 truncated_harness = output_harness[-4000:] if len(output_harness) > 4000 else output_harness
1012 truncated_baseline = output_baseline[-4000:] if len(output_baseline) > 4000 else output_baseline
1014 # Randomize order: coin flip determines whether harness→A / baseline→B
1015 harness_is_a = random.choice([True, False])
1016 if harness_is_a:
1017 output_a, output_b = truncated_harness, truncated_baseline
1018 else:
1019 output_a, output_b = truncated_baseline, truncated_harness
1021 user_prompt = (
1022 f"{effective_prompt}\n\n"
1023 f"<output_a>\n{output_a}\n</output_a>\n\n"
1024 f"<output_b>\n{output_b}\n</output_b>"
1025 )
1027 invocation = resolve_host().build_blocking_json(prompt=user_prompt, model=model)
1028 args = list(invocation.args) + [
1029 "--json-schema",
1030 json.dumps(BLIND_COMPARATOR_SCHEMA),
1031 "--no-session-persistence",
1032 ]
1034 try:
1035 proc = subprocess.run(
1036 [invocation.binary, *args], capture_output=True, text=True, timeout=timeout
1037 )
1038 except subprocess.TimeoutExpired:
1039 # On timeout, both fail — conservative default
1040 return {
1041 "harness_pass": False,
1042 "baseline_pass": False,
1043 "confidence": 0.0,
1044 "reason": "LLM evaluation timed out",
1045 "raw": {"verdict_a": "timeout", "verdict_b": "timeout"},
1046 "error": "timeout",
1047 }
1048 except FileNotFoundError:
1049 return {
1050 "harness_pass": False,
1051 "baseline_pass": False,
1052 "confidence": 0.0,
1053 "reason": f"{invocation.binary} CLI not found",
1054 "raw": {"verdict_a": "error", "verdict_b": "error"},
1055 "error": "missing_cli",
1056 }
1058 if proc.returncode != 0:
1059 return {
1060 "harness_pass": False,
1061 "baseline_pass": False,
1062 "confidence": 0.0,
1063 "reason": f"Judge CLI error: {proc.stderr.strip()[:200]}",
1064 "raw": {"verdict_a": "error", "verdict_b": "error"},
1065 "error": "api_error",
1066 }
1068 if not proc.stdout.strip():
1069 return {
1070 "harness_pass": False,
1071 "baseline_pass": False,
1072 "confidence": 0.0,
1073 "reason": "Judge returned empty output",
1074 "raw": {"verdict_a": "error", "verdict_b": "error"},
1075 "error": "empty_output",
1076 }
1078 try:
1079 stdout = proc.stdout.strip()
1080 try:
1081 envelope = json.loads(stdout)
1082 except json.JSONDecodeError:
1083 lines = [line for line in stdout.split("\n") if line.strip()]
1084 if not lines:
1085 raise
1086 envelope = json.loads(lines[-1])
1088 if envelope.get("subtype") == "error_max_structured_output_retries":
1089 return {
1090 "harness_pass": False,
1091 "baseline_pass": False,
1092 "confidence": 0.0,
1093 "reason": "Judge could not produce valid structured output after retries",
1094 "raw": {"verdict_a": "error", "verdict_b": "error"},
1095 "error": "retry_exhausted",
1096 }
1098 if envelope.get("is_error", False):
1099 err_text = str(envelope.get("result", "") or "")[:200]
1100 return {
1101 "harness_pass": False,
1102 "baseline_pass": False,
1103 "confidence": 0.0,
1104 "reason": f"Judge reported error: {err_text}",
1105 "raw": {"verdict_a": "error", "verdict_b": "error"},
1106 "error": "api_error",
1107 }
1109 if isinstance(envelope.get("structured_output"), dict):
1110 result: dict[str, Any] = envelope["structured_output"]
1111 else:
1112 raw_result = envelope.get("result", "")
1113 if isinstance(raw_result, dict):
1114 result = raw_result
1115 elif raw_result:
1116 result = json.loads(raw_result)
1117 else:
1118 return {
1119 "harness_pass": False,
1120 "baseline_pass": False,
1121 "confidence": 0.0,
1122 "reason": "Empty result field in judge response",
1123 "raw": {"verdict_a": "error", "verdict_b": "error"},
1124 "error": "empty_result",
1125 }
1126 except (json.JSONDecodeError, TypeError, ValueError):
1127 return {
1128 "harness_pass": False,
1129 "baseline_pass": False,
1130 "confidence": 0.0,
1131 "reason": "Failed to parse judge response",
1132 "raw": {"verdict_a": "error", "verdict_b": "error"},
1133 "error": "parse_error",
1134 }
1136 # De-anonymize
1137 verdict_a = str(result.get("verdict_a", "no"))
1138 verdict_b = str(result.get("verdict_b", "no"))
1139 confidence = float(result.get("confidence", 0.0))
1140 reason = str(result.get("reason", ""))
1142 if harness_is_a:
1143 harness_pass = verdict_a == "yes"
1144 baseline_pass = verdict_b == "yes"
1145 else:
1146 harness_pass = verdict_b == "yes"
1147 baseline_pass = verdict_a == "yes"
1149 return {
1150 "harness_pass": harness_pass,
1151 "baseline_pass": baseline_pass,
1152 "confidence": confidence,
1153 "reason": reason,
1154 "raw": {"verdict_a": verdict_a, "verdict_b": verdict_b, "harness_is_a": harness_is_a},
1155 }
1158def evaluate_contract(
1159 config: EvaluateConfig,
1160 context: InterpolationContext,
1161 model: str = DEFAULT_LLM_MODEL,
1162 timeout: int = 1800,
1163) -> EvaluationResult:
1164 """Evaluate producer/consumer contract alignment using an LLM judge.
1166 Reads each producer/consumer file pair, applies optional regex extraction,
1167 then asks an LLM judge whether the producer satisfies the consumer contract.
1168 Returns yes only when all pairs align; any failure routes no/error.
1170 Args:
1171 config: EvaluateConfig with type="contract" and pairs list
1172 context: Interpolation context (unused by this evaluator directly)
1173 model: LLM model identifier
1174 timeout: Subprocess timeout in seconds
1176 Returns:
1177 EvaluationResult with verdict yes/no/error and pair_results in details
1178 """
1179 pairs = config.pairs
1180 if not pairs:
1181 return EvaluationResult(
1182 verdict="error",
1183 details={"error": "contract evaluator requires at least one pair in evaluate.pairs"},
1184 )
1186 contract_schema = {
1187 "type": "object",
1188 "properties": {
1189 "verdict": {"type": "string", "enum": ["yes", "no"]},
1190 "confidence": {"type": "number"},
1191 "reason": {"type": "string"},
1192 },
1193 "required": ["verdict", "confidence", "reason"],
1194 }
1196 pair_results: list[dict[str, Any]] = []
1198 for pair in pairs:
1199 producer_path = pair.get("producer", "")
1200 consumer_path = pair.get("consumer", "")
1201 producer_pattern = pair.get("producer_pattern")
1202 consumer_pattern = pair.get("consumer_pattern")
1203 contract_rule = pair.get("contract", "the producer and consumer must be compatible")
1205 # Read producer file
1206 try:
1207 producer_content = Path(producer_path).read_text()
1208 except OSError as e:
1209 pair_results.append(
1210 {
1211 "producer": producer_path,
1212 "consumer": consumer_path,
1213 "verdict": "error",
1214 "error": f"cannot read producer file: {e}",
1215 }
1216 )
1217 continue
1219 # Read consumer file
1220 try:
1221 consumer_content = Path(consumer_path).read_text()
1222 except OSError as e:
1223 pair_results.append(
1224 {
1225 "producer": producer_path,
1226 "consumer": consumer_path,
1227 "verdict": "error",
1228 "error": f"cannot read consumer file: {e}",
1229 }
1230 )
1231 continue
1233 # Apply optional regex extraction
1234 if producer_pattern:
1235 matches = re.findall(producer_pattern, producer_content, re.DOTALL)
1236 if not matches:
1237 pair_results.append(
1238 {
1239 "producer": producer_path,
1240 "consumer": consumer_path,
1241 "verdict": "error",
1242 "error": f"producer_pattern matched nothing in {producer_path}",
1243 }
1244 )
1245 continue
1246 producer_slice = "\n".join(matches)
1247 else:
1248 producer_slice = (
1249 producer_content[-4000:] if len(producer_content) > 4000 else producer_content
1250 )
1252 if consumer_pattern:
1253 matches = re.findall(consumer_pattern, consumer_content, re.DOTALL)
1254 if not matches:
1255 pair_results.append(
1256 {
1257 "producer": producer_path,
1258 "consumer": consumer_path,
1259 "verdict": "error",
1260 "error": f"consumer_pattern matched nothing in {consumer_path}",
1261 }
1262 )
1263 continue
1264 consumer_slice = "\n".join(matches)
1265 else:
1266 consumer_slice = (
1267 consumer_content[-4000:] if len(consumer_content) > 4000 else consumer_content
1268 )
1270 judge_prompt = (
1271 f"You are evaluating whether a producer output satisfies a consumer contract.\n\n"
1272 f"Contract rule: {contract_rule}\n\n"
1273 f'<producer path="{producer_path}">\n{producer_slice}\n</producer>\n\n'
1274 f'<consumer path="{consumer_path}">\n{consumer_slice}\n</consumer>\n\n'
1275 "Does the producer satisfy the consumer contract? "
1276 "Consider field names, types, casing, and structure. "
1277 "Answer yes if aligned, no if mismatched."
1278 )
1280 invocation = resolve_host().build_blocking_json(prompt=judge_prompt, model=model)
1281 args = list(invocation.args) + [
1282 "--json-schema",
1283 json.dumps(contract_schema),
1284 "--no-session-persistence",
1285 ]
1287 t0 = time.monotonic()
1288 try:
1289 proc = subprocess.run(
1290 [invocation.binary, *args], capture_output=True, text=True, timeout=timeout
1291 )
1292 except subprocess.TimeoutExpired:
1293 pair_results.append(
1294 {
1295 "producer": producer_path,
1296 "consumer": consumer_path,
1297 "verdict": "error",
1298 "error": "LLM judge timed out",
1299 "llm_latency_ms": int((time.monotonic() - t0) * 1000),
1300 }
1301 )
1302 continue
1303 except FileNotFoundError:
1304 return EvaluationResult(
1305 verdict="error",
1306 details={
1307 "error": f"{invocation.binary} CLI not found. Install the active host CLI (see LL_HOST_CLI).",
1308 "missing_dependency": True,
1309 },
1310 )
1311 llm_latency_ms = int((time.monotonic() - t0) * 1000)
1313 if proc.returncode != 0:
1314 pair_results.append(
1315 {
1316 "producer": producer_path,
1317 "consumer": consumer_path,
1318 "verdict": "error",
1319 "error": f"CLI error: {proc.stderr.strip()}",
1320 "llm_latency_ms": llm_latency_ms,
1321 }
1322 )
1323 continue
1325 if not proc.stdout.strip():
1326 pair_results.append(
1327 {
1328 "producer": producer_path,
1329 "consumer": consumer_path,
1330 "verdict": "error",
1331 "error": "CLI returned empty output",
1332 "llm_latency_ms": llm_latency_ms,
1333 }
1334 )
1335 continue
1337 try:
1338 stdout = proc.stdout.strip()
1339 try:
1340 envelope = json.loads(stdout)
1341 except json.JSONDecodeError:
1342 lines = [line for line in stdout.split("\n") if line.strip()]
1343 if not lines:
1344 raise
1345 envelope = json.loads(lines[-1])
1347 if envelope.get("subtype") == "error_max_structured_output_retries":
1348 pair_results.append(
1349 {
1350 "producer": producer_path,
1351 "consumer": consumer_path,
1352 "verdict": "error",
1353 "error": "Claude CLI could not produce valid structured output after retries",
1354 "llm_latency_ms": llm_latency_ms,
1355 }
1356 )
1357 continue
1359 if envelope.get("is_error", False):
1360 err_text = str(envelope.get("result", "") or "")[:200]
1361 pair_results.append(
1362 {
1363 "producer": producer_path,
1364 "consumer": consumer_path,
1365 "verdict": "error",
1366 "error": f"Claude CLI reported error: {err_text}",
1367 "llm_latency_ms": llm_latency_ms,
1368 }
1369 )
1370 continue
1372 if isinstance(envelope.get("structured_output"), dict):
1373 llm_result: dict[str, Any] = envelope["structured_output"]
1374 else:
1375 raw_result = envelope.get("result", "")
1376 if isinstance(raw_result, dict):
1377 llm_result = raw_result
1378 elif raw_result:
1379 llm_result = json.loads(raw_result)
1380 elif "verdict" in envelope:
1381 llm_result = envelope
1382 else:
1383 pair_results.append(
1384 {
1385 "producer": producer_path,
1386 "consumer": consumer_path,
1387 "verdict": "error",
1388 "error": "empty result field in CLI response",
1389 "llm_latency_ms": llm_latency_ms,
1390 }
1391 )
1392 continue
1394 except (json.JSONDecodeError, TypeError, ValueError) as e:
1395 pair_results.append(
1396 {
1397 "producer": producer_path,
1398 "consumer": consumer_path,
1399 "verdict": "error",
1400 "error": f"failed to parse LLM response: {e}",
1401 "llm_latency_ms": llm_latency_ms,
1402 }
1403 )
1404 continue
1406 pair_results.append(
1407 {
1408 "producer": producer_path,
1409 "consumer": consumer_path,
1410 "verdict": str(llm_result.get("verdict", "error")),
1411 "confidence": float(llm_result.get("confidence", 1.0)),
1412 "reason": llm_result.get("reason", ""),
1413 "llm_latency_ms": llm_latency_ms,
1414 }
1415 )
1417 # Aggregate: yes only if all pairs aligned; error takes precedence over no
1418 if any(p["verdict"] == "error" for p in pair_results):
1419 overall = "error"
1420 elif all(p["verdict"] == "yes" for p in pair_results):
1421 overall = "yes"
1422 else:
1423 overall = "no"
1425 return EvaluationResult(
1426 verdict=overall,
1427 details={"pair_results": pair_results},
1428 )
1431def evaluate_comparator(
1432 config: EvaluateConfig,
1433 output: str,
1434 context: InterpolationContext,
1435) -> EvaluationResult:
1436 """Evaluate using blind A/B comparison against a stored baseline."""
1437 from pathlib import Path
1439 if config.baseline_path is None:
1440 return EvaluationResult(
1441 verdict="no_baseline",
1442 details={"reason": "No baseline_path configured"},
1443 )
1445 baseline_file = Path(config.baseline_path) / "output.txt"
1446 if not baseline_file.exists():
1447 if config.auto_promote:
1448 baseline_file.parent.mkdir(parents=True, exist_ok=True)
1449 baseline_file.write_text(output)
1450 return EvaluationResult(
1451 verdict="yes",
1452 details={
1453 "reason": "No baseline found; current output promoted as new baseline.",
1454 "bootstrapped": True,
1455 },
1456 )
1457 return EvaluationResult(
1458 verdict="no_baseline",
1459 details={"reason": f"Baseline file not found: {baseline_file}"},
1460 )
1462 baseline_text = baseline_file.read_text()
1463 min_pairs = max(1, config.min_pairs if config.min_pairs is not None else 1)
1464 harness_wins = 0
1465 baseline_wins = 0
1466 last_reason = ""
1467 last_raw: dict[str, Any] = {}
1469 for _ in range(min_pairs):
1470 result = evaluate_blind_comparator(output, baseline_text, prompt=config.prompt)
1471 if result.get("harness_pass"):
1472 harness_wins += 1
1473 if result.get("baseline_pass"):
1474 baseline_wins += 1
1475 last_reason = result.get("reason", "")
1476 last_raw = result.get("raw", {})
1478 if harness_wins > baseline_wins:
1479 verdict = "yes"
1480 elif baseline_wins > harness_wins:
1481 verdict = "no"
1482 else:
1483 verdict = "tie"
1485 if config.auto_promote and verdict == "yes":
1486 baseline_file.write_text(output)
1488 return EvaluationResult(
1489 verdict=verdict,
1490 details={
1491 "harness_wins": harness_wins,
1492 "baseline_wins": baseline_wins,
1493 "min_pairs": min_pairs,
1494 "reason": last_reason,
1495 "raw": last_raw,
1496 },
1497 )
1500def evaluate(
1501 config: EvaluateConfig,
1502 output: str,
1503 exit_code: int,
1504 context: InterpolationContext,
1505) -> EvaluationResult:
1506 """Dispatch to appropriate evaluator based on config type.
1508 Args:
1509 config: Evaluator configuration with type and parameters
1510 output: Action stdout
1511 exit_code: Action exit code
1512 context: Runtime context for variable interpolation
1514 Returns:
1515 EvaluationResult from the appropriate evaluator
1517 Raises:
1518 ValueError: If evaluator type is unknown
1519 """
1520 eval_type = config.type
1522 # BUG-1640: Action-level timeouts (exit_code=124) short-circuit to "error"
1523 # so loop authors' on_error: branches fire instead of being routed via
1524 # on_no: based on truncated output. mcp_result is exempted because it has
1525 # its own established "timeout" verdict (see evaluate_mcp_result).
1526 if exit_code == 124 and eval_type != "mcp_result":
1527 return EvaluationResult(
1528 verdict="error",
1529 details={"exit_code": exit_code, "error": "action timed out"},
1530 )
1532 # BUG-1815: Non-timeout non-zero exit codes short-circuit to "error" for
1533 # evaluator types that don't intrinsically check exit codes. Exit-code-aware
1534 # evaluators (exit_code, mcp_result, harbor_scorer, diff_stall, llm_structured)
1535 # are exempt because they handle exit codes via their own logic.
1536 _EXIT_CODE_AWARE_EVALUATORS: frozenset[str] = frozenset(
1537 {
1538 "exit_code",
1539 "mcp_result",
1540 "harbor_scorer",
1541 "diff_stall",
1542 "action_stall",
1543 "llm_structured",
1544 "contract",
1545 }
1546 )
1547 if exit_code != 0 and eval_type not in _EXIT_CODE_AWARE_EVALUATORS:
1548 return EvaluationResult(
1549 verdict="error",
1550 details={
1551 "exit_code": exit_code,
1552 "error": f"action exited with code {exit_code}",
1553 },
1554 )
1556 if eval_type == "exit_code":
1557 return evaluate_exit_code(exit_code)
1559 elif eval_type == "output_numeric":
1560 if config.target is None:
1561 raise ValueError("output_numeric evaluator requires 'target' to be set")
1562 elif isinstance(config.target, str):
1563 try:
1564 resolved = interpolate(config.target, context) if context else config.target
1565 numeric_target = float(resolved)
1566 except (InterpolationError, ValueError) as e:
1567 raise ValueError(
1568 f"output_numeric target must be numeric, got: {config.target!r}"
1569 ) from e
1570 else:
1571 numeric_target = float(config.target)
1572 return evaluate_output_numeric(
1573 output=output,
1574 operator=config.operator or "eq",
1575 target=numeric_target,
1576 )
1578 elif eval_type == "output_json":
1579 return evaluate_output_json(
1580 output=output,
1581 path=config.path or "",
1582 operator=config.operator or "eq",
1583 target=config.target,
1584 )
1586 elif eval_type == "output_contains":
1587 return evaluate_output_contains(
1588 output=output,
1589 pattern=config.pattern or "",
1590 negate=config.negate,
1591 error_patterns=config.error_patterns,
1592 )
1594 elif eval_type == "convergence":
1595 # Resolve previous value from interpolation if configured
1596 previous: float | None = None
1597 if config.previous:
1598 try:
1599 previous = float(interpolate(config.previous, context))
1600 except (InterpolationError, ValueError):
1601 # Previous unavailable on first iteration, continue with None
1602 pass
1604 # Parse current value from output
1605 try:
1606 current = float(output.strip())
1607 except ValueError:
1608 return EvaluationResult(
1609 verdict="error",
1610 details={"error": f"Cannot parse output as number: {output[:100]}"},
1611 )
1613 # Resolve target (may be interpolated string like "${context.target}")
1614 convergence_target: float
1615 if isinstance(config.target, str):
1616 try:
1617 convergence_target = float(interpolate(config.target, context))
1618 except (InterpolationError, ValueError) as e:
1619 return EvaluationResult(
1620 verdict="error",
1621 details={"error": f"Cannot resolve target: {e}"},
1622 )
1623 else:
1624 if config.target is None:
1625 raise ValueError("convergence evaluator requires 'target' to be set")
1626 convergence_target = float(config.target)
1628 # Resolve tolerance (may be interpolated string)
1629 tolerance: float = 0.0
1630 if config.tolerance is not None:
1631 if isinstance(config.tolerance, str):
1632 try:
1633 tolerance = float(interpolate(config.tolerance, context))
1634 except (InterpolationError, ValueError):
1635 tolerance = 0.0
1636 else:
1637 tolerance = float(config.tolerance)
1639 return evaluate_convergence(
1640 current=current,
1641 previous=previous,
1642 target=convergence_target,
1643 tolerance=tolerance,
1644 direction=config.direction,
1645 )
1647 elif eval_type == "diff_stall":
1648 return evaluate_diff_stall(
1649 scope=config.scope,
1650 max_stall=config.max_stall,
1651 )
1653 elif eval_type == "action_stall":
1654 return evaluate_action_stall(
1655 track=config.track,
1656 max_repeat=config.max_repeat,
1657 context=context,
1658 )
1660 elif eval_type == "llm_structured":
1661 prompt = config.prompt
1662 if prompt and context:
1663 try:
1664 prompt = interpolate(prompt, context)
1665 except InterpolationError:
1666 pass # Use raw prompt on resolution failure
1667 return evaluate_llm_structured(
1668 output=output,
1669 prompt=prompt,
1670 schema=config.schema,
1671 min_confidence=config.min_confidence,
1672 uncertain_suffix=config.uncertain_suffix,
1673 )
1675 elif eval_type == "mcp_result":
1676 return evaluate_mcp_result(output=output, exit_code=exit_code)
1678 elif eval_type == "harbor_scorer":
1679 return evaluate_harbor_scorer(output=output, exit_code=exit_code)
1681 elif eval_type == "comparator":
1682 return evaluate_comparator(config=config, output=output, context=context)
1684 elif eval_type == "contract":
1685 return evaluate_contract(config=config, context=context)
1687 elif eval_type == "classify":
1688 return evaluate_classify(output=output, line=config.line)
1690 else:
1691 raise ValueError(f"Unknown evaluator type: {eval_type}")