Coverage for little_loops / fsm / evaluators.py: 9%
535 statements
« prev ^ index » next coverage.py v7.12.0, created at 2026-06-16 13:12 -0500
1"""FSM Evaluators for loop execution.
3This module provides evaluators that interpret action output and produce
4verdicts for state transitions.
6Supported evaluator types:
8Tier 1 (Deterministic - no API calls):
9 exit_code: Map Unix exit codes to verdicts (0=success, 1=failure, 2+=error)
10 output_numeric: Compare numeric output to target value
11 output_json: Extract and compare JSON path values
12 output_contains: Pattern matching on stdout
13 convergence: Track progress toward a target value
14 diff_stall: Detect stalled iterations via git diff comparison
15 action_stall: Detect when the same action string or output repeats for N consecutive iterations
16 harbor_scorer: Interpret Harbor-format benchmark scorer exit code and float stdout
18Tier 2 (LLM-based):
19 llm_structured: Use LLM with structured output for natural language evaluation
20 contract: Read producer/consumer file pairs and assert contract alignment via LLM judge
22Tier 3 (External process):
23 mcp_result: Parse MCP tool call response envelope
24"""
26from __future__ import annotations
28import hashlib
29import json
30import random
31import re
32import subprocess
33import time
34from collections.abc import Callable
35from dataclasses import dataclass
36from pathlib import Path
37from typing import Any
39from little_loops.fsm.interpolation import (
40 InterpolationContext,
41 InterpolationError,
42 interpolate,
43)
44from little_loops.fsm.schema import DEFAULT_LLM_MODEL, EvaluateConfig
45from little_loops.host_runner import resolve_host
48@dataclass
49class EvaluationResult:
50 """Result from an evaluator.
52 Attributes:
53 verdict: The routing key for state transitions
54 details: Evaluator-specific metadata for debugging/logging
55 """
57 verdict: str
58 details: dict[str, Any]
61# Default schema for LLM structured evaluation
62DEFAULT_LLM_SCHEMA: dict[str, Any] = {
63 "type": "object",
64 "properties": {
65 "verdict": {
66 "type": "string",
67 "enum": ["yes", "no", "blocked", "partial"],
68 "description": (
69 "- yes: The condition/check evaluated to true\n"
70 "- no: The condition/check evaluated to false\n"
71 "- blocked: Cannot proceed without external help\n"
72 "- partial: Made progress but not complete"
73 ),
74 },
75 "confidence": {
76 "type": "number",
77 "minimum": 0,
78 "maximum": 1,
79 "description": "Confidence in this verdict (0-1)",
80 },
81 "reason": {
82 "type": "string",
83 "description": "Brief explanation",
84 },
85 },
86 "required": ["verdict", "confidence", "reason"],
87}
89DEFAULT_LLM_PROMPT = "Evaluate whether this action succeeded based on its output."
91# Schema for blind A/B comparator: evaluates two anonymized outputs
92BLIND_COMPARATOR_SCHEMA: dict[str, Any] = {
93 "type": "object",
94 "properties": {
95 "verdict_a": {
96 "type": "string",
97 "enum": ["yes", "no"],
98 "description": "Whether Output A meets the evaluation criteria",
99 },
100 "verdict_b": {
101 "type": "string",
102 "enum": ["yes", "no"],
103 "description": "Whether Output B meets the evaluation criteria",
104 },
105 "confidence": {
106 "type": "number",
107 "minimum": 0,
108 "maximum": 1,
109 "description": "Confidence in these verdicts (0-1)",
110 },
111 "reason": {
112 "type": "string",
113 "description": "Brief explanation comparing the two outputs",
114 },
115 },
116 "required": ["verdict_a", "verdict_b", "confidence", "reason"],
117}
119DEFAULT_BLIND_COMPARATOR_PROMPT = (
120 "You are evaluating two outputs (labeled 'Output A' and 'Output B') that were "
121 "produced by independent runs of the same task. Judge whether each output meets "
122 "the evaluation criteria below. Be objective and impartial — the labels 'A' and "
123 "'B' are arbitrary and do not indicate which is better."
124)
126_NUMERIC_OPERATORS: dict[str, Callable[[float, float], bool]] = {
127 "eq": lambda v, t: v == t,
128 "ne": lambda v, t: v != t,
129 "lt": lambda v, t: v < t,
130 "le": lambda v, t: v <= t,
131 "gt": lambda v, t: v > t,
132 "ge": lambda v, t: v >= t,
133}
136def evaluate_exit_code(exit_code: int) -> EvaluationResult:
137 """Map Unix exit code to verdict.
139 Args:
140 exit_code: The process exit code
142 Returns:
143 EvaluationResult with verdict:
144 - 0 -> yes
145 - 1 -> no
146 - 2+ -> error
147 """
148 if exit_code == 0:
149 verdict = "yes"
150 elif exit_code == 1:
151 verdict = "no"
152 else:
153 verdict = "error"
155 return EvaluationResult(verdict=verdict, details={"exit_code": exit_code})
158def evaluate_output_numeric(
159 output: str,
160 operator: str,
161 target: float,
162) -> EvaluationResult:
163 """Parse stdout as number and compare to target.
165 Args:
166 output: The action stdout to parse as a number
167 operator: Comparison operator (eq, ne, lt, le, gt, ge)
168 target: Target value to compare against
170 Returns:
171 EvaluationResult with verdict:
172 - Condition met -> yes
173 - Condition not met -> no
174 - Parse error -> error
175 """
176 try:
177 value = float(output.strip())
178 except ValueError:
179 return EvaluationResult(
180 verdict="error",
181 details={"error": f"Cannot parse as number: {output[:100]}"},
182 )
184 if operator not in _NUMERIC_OPERATORS:
185 return EvaluationResult(
186 verdict="error",
187 details={"error": f"Unknown operator: {operator}"},
188 )
190 condition_met = _NUMERIC_OPERATORS[operator](value, target)
191 return EvaluationResult(
192 verdict="yes" if condition_met else "no",
193 details={"value": value, "target": target, "operator": operator},
194 )
197def _extract_json_path(data: Any, path: str) -> Any:
198 """Extract value from dict using jq-style path like '.summary.failed'.
200 Args:
201 data: The parsed JSON data (dict or list)
202 path: Dot-separated path, optionally starting with '.'
204 Returns:
205 The value at the specified path
207 Raises:
208 KeyError: If path not found in data
209 """
210 if path.startswith("."):
211 path = path[1:]
212 parts = path.split(".")
213 current = data
214 for part in parts:
215 if isinstance(current, dict) and part in current:
216 current = current[part]
217 elif isinstance(current, list) and part.isdigit():
218 idx = int(part)
219 if 0 <= idx < len(current):
220 current = current[idx]
221 else:
222 raise KeyError(path)
223 else:
224 raise KeyError(path)
225 return current
228def _compare_values(
229 value: int | float, operator: str, target: int | float, path: str
230) -> EvaluationResult:
231 """Compare numeric values using operator.
233 Args:
234 value: The extracted value to compare
235 operator: Comparison operator
236 target: Target value
237 path: JSON path for details
239 Returns:
240 EvaluationResult with comparison result
241 """
242 if operator not in _NUMERIC_OPERATORS:
243 return EvaluationResult(
244 verdict="error",
245 details={"error": f"Unknown operator: {operator}"},
246 )
248 condition_met = _NUMERIC_OPERATORS[operator](value, target)
249 return EvaluationResult(
250 verdict="yes" if condition_met else "no",
251 details={"value": value, "path": path, "target": target, "operator": operator},
252 )
255def evaluate_output_json(
256 output: str,
257 path: str,
258 operator: str,
259 target: Any,
260) -> EvaluationResult:
261 """Parse JSON and extract value at path, then compare.
263 Args:
264 output: The action stdout containing JSON
265 path: jq-style dot notation path (e.g., '.summary.failed')
266 operator: Comparison operator (eq, ne, lt, le, gt, ge)
267 target: Target value for comparison
269 Returns:
270 EvaluationResult with verdict:
271 - Condition met -> yes
272 - Condition not met -> no
273 - Parse/path error -> error
274 """
275 try:
276 data = json.loads(output)
277 except json.JSONDecodeError as e:
278 return EvaluationResult(
279 verdict="error",
280 details={"error": f"Invalid JSON: {e}"},
281 )
283 try:
284 value = _extract_json_path(data, path)
285 except KeyError:
286 return EvaluationResult(
287 verdict="error",
288 details={"error": f"Path not found: {path}"},
289 )
291 # Use numeric comparison if both values are numeric
292 if isinstance(value, (int, float)) and isinstance(target, (int, float)):
293 return _compare_values(value, operator, target, path)
295 # For non-numeric values, only eq and ne are supported
296 if operator == "eq":
297 verdict = "yes" if value == target else "no"
298 elif operator == "ne":
299 verdict = "yes" if value != target else "no"
300 else:
301 return EvaluationResult(
302 verdict="error",
303 details={"error": f"Operator {operator} not supported for non-numeric values"},
304 )
306 return EvaluationResult(
307 verdict=verdict,
308 details={"value": value, "path": path, "target": target, "operator": operator},
309 )
312def evaluate_output_contains(
313 output: str,
314 pattern: str,
315 negate: bool = False,
316) -> EvaluationResult:
317 """Check if pattern exists in output.
319 Pattern can be regex or substring. If regex fails to compile,
320 falls back to substring matching.
322 Args:
323 output: The action stdout to search
324 pattern: Regex pattern or substring
325 negate: If True, invert the match result
327 Returns:
328 EvaluationResult with verdict:
329 - Found (negate=False) -> yes
330 - Found (negate=True) -> no
331 - Not found (negate=False) -> no
332 - Not found (negate=True) -> yes
333 """
334 # Try regex first, fall back to substring
335 try:
336 matched = bool(re.search(pattern, output))
337 except re.error:
338 matched = pattern in output
340 if negate:
341 verdict = "no" if matched else "yes"
342 else:
343 verdict = "yes" if matched else "no"
345 return EvaluationResult(
346 verdict=verdict,
347 details={"matched": matched, "pattern": pattern, "negate": negate},
348 )
351def evaluate_convergence(
352 current: float,
353 previous: float | None,
354 target: float,
355 tolerance: float = 0,
356 direction: str = "minimize",
357) -> EvaluationResult:
358 """Compare current value to target and previous.
360 Args:
361 current: Current metric value
362 previous: Previous metric value (None if first iteration)
363 target: Target value to reach
364 tolerance: Acceptable distance from target
365 direction: 'minimize' or 'maximize'
367 Returns:
368 EvaluationResult with verdict:
369 - Value within tolerance of target -> target
370 - Value improved toward target -> progress
371 - Value unchanged or worsened -> stall
372 """
373 # Check if target reached (within tolerance)
374 if abs(current - target) <= tolerance:
375 return EvaluationResult(
376 verdict="target",
377 details={"current": current, "target": target, "delta": 0},
378 )
380 # First iteration has no previous value
381 if previous is None:
382 return EvaluationResult(
383 verdict="progress",
384 details={
385 "current": current,
386 "previous": None,
387 "target": target,
388 "delta": None,
389 },
390 )
392 # Calculate progress
393 delta = current - previous
395 if direction == "minimize":
396 # For minimizing, negative delta is progress
397 made_progress = delta < 0
398 else:
399 # For maximizing, positive delta is progress
400 made_progress = delta > 0
402 verdict = "progress" if made_progress else "stall"
404 return EvaluationResult(
405 verdict=verdict,
406 details={
407 "current": current,
408 "previous": previous,
409 "target": target,
410 "delta": delta,
411 "direction": direction,
412 },
413 )
416def evaluate_classify(
417 output: str,
418 line: str | int | None = None,
419) -> EvaluationResult:
420 """Read a token from stdout and return it as the verdict.
422 Intended for single-state multi-way routing: the action prints exactly one
423 token to stdout and the route: table maps that token to the next state.
425 Args:
426 output: The action stdout to read the token from
427 line: Which line to select. 'last' (default) picks the last non-empty
428 line; 'first' picks the first non-empty line; an integer index
429 selects that line (0-based, negative indices supported).
431 Returns:
432 EvaluationResult with verdict = trimmed token, or "" when output is
433 empty (which _route() maps to the route.default fallback).
434 """
435 lines = [ln for ln in output.splitlines() if ln.strip()]
436 if not lines:
437 return EvaluationResult(
438 verdict="",
439 details={"token": "", "line": line, "source_lines": 0},
440 )
442 selector = line if line is not None else "last"
443 if selector == "last":
444 selected = lines[-1]
445 elif selector == "first":
446 selected = lines[0]
447 elif isinstance(selector, int):
448 try:
449 selected = lines[selector]
450 except IndexError:
451 return EvaluationResult(
452 verdict="",
453 details={
454 "token": "",
455 "line": line,
456 "source_lines": len(lines),
457 "error": "index out of range",
458 },
459 )
460 else:
461 selected = lines[-1]
463 token = selected.strip()
464 return EvaluationResult(
465 verdict=token,
466 details={"token": token, "line": line},
467 )
470def evaluate_diff_stall(
471 scope: list[str] | None = None,
472 max_stall: int = 1,
473) -> EvaluationResult:
474 """Detect stalled iterations by comparing git diff --stat between runs.
476 On first call, snapshots the current diff and returns 'yes'.
477 On subsequent calls, compares current diff to the previous snapshot.
478 If the diff is identical for max_stall consecutive iterations, returns
479 'no' (stalled). If different, resets the stall counter and returns
480 'yes' (progress).
482 State is persisted in /tmp using a key derived from the scope argument,
483 so different loops with different scopes maintain independent stall counters.
485 Args:
486 scope: Optional list of paths to limit the git diff to. Defaults to
487 the entire working tree.
488 max_stall: Number of consecutive no-change iterations before stall
489 verdict. Defaults to 1.
491 Returns:
492 EvaluationResult with verdict:
493 - yes: diff changed since last iteration (progress made)
494 - no: diff unchanged for max_stall iterations (stalled)
495 - error: git command failed or timed out
496 """
497 cmd = ["git", "diff", "--stat"]
498 if scope:
499 cmd += ["--"] + scope
501 try:
502 proc = subprocess.run(cmd, capture_output=True, text=True, timeout=30)
503 except subprocess.TimeoutExpired:
504 return EvaluationResult(verdict="error", details={"error": "git diff timed out"})
505 except FileNotFoundError:
506 return EvaluationResult(verdict="error", details={"error": "git not found in PATH"})
508 if proc.returncode != 0:
509 return EvaluationResult(
510 verdict="error",
511 details={"error": f"git diff failed: {proc.stderr[:200]}"},
512 )
514 current_diff = proc.stdout
516 # Derive a stable cache key from the scope so independent loops don't collide
517 scope_str = "|".join(sorted(scope)) if scope else "_root_"
518 cache_key = hashlib.md5(scope_str.encode()).hexdigest()[:12]
519 loops_tmp = Path.cwd() / ".loops" / "tmp"
520 loops_tmp.mkdir(parents=True, exist_ok=True)
521 state_file = loops_tmp / f"ll-diff-stall-{cache_key}.txt"
522 count_file = loops_tmp / f"ll-diff-stall-{cache_key}.count"
524 # Read previous snapshot and stall count
525 previous_diff: str | None = None
526 stall_count = 0
527 try:
528 previous_diff = state_file.read_text()
529 stall_count = int(count_file.read_text().strip())
530 except (FileNotFoundError, ValueError):
531 pass
533 # First iteration: save snapshot and report progress
534 if previous_diff is None:
535 state_file.write_text(current_diff)
536 count_file.write_text("0")
537 return EvaluationResult(
538 verdict="yes",
539 details={"stall_count": 0, "max_stall": max_stall, "diff_changed": True},
540 )
542 if current_diff == previous_diff:
543 stall_count += 1
544 count_file.write_text(str(stall_count))
545 if stall_count >= max_stall:
546 return EvaluationResult(
547 verdict="no",
548 details={"stall_count": stall_count, "max_stall": max_stall, "diff_changed": False},
549 )
550 # Not yet at max_stall threshold — still report yes so loop continues
551 return EvaluationResult(
552 verdict="yes",
553 details={"stall_count": stall_count, "max_stall": max_stall, "diff_changed": False},
554 )
555 else:
556 # Progress: update snapshot and reset counter
557 state_file.write_text(current_diff)
558 count_file.write_text("0")
559 return EvaluationResult(
560 verdict="yes",
561 details={"stall_count": 0, "max_stall": max_stall, "diff_changed": True},
562 )
565def evaluate_action_stall(
566 track: list[str] | None = None,
567 max_repeat: int = 2,
568 context: InterpolationContext | None = None,
569) -> EvaluationResult:
570 """Detect when the same action string or output repeats for N consecutive iterations.
572 On first call, snapshots the hashed values of the tracked context keys and returns
573 'yes'. On subsequent calls, compares the current hash to the previous snapshot.
574 If the hash is identical for max_repeat consecutive iterations, returns 'no'
575 (stalled). If different, resets the stall counter and returns 'yes' (progress).
577 State is persisted in .loops/tmp using a key derived from the tracked keys,
578 so different states/loops maintain independent stall counters.
580 Args:
581 track: Context keys to track. Defaults to ["action"] when None.
582 max_repeat: Number of consecutive identical-hash iterations before stall verdict.
583 Defaults to 2.
584 context: Runtime interpolation context for resolving tracked keys.
586 Returns:
587 EvaluationResult with verdict:
588 - yes: tracked values changed since last iteration (progress made)
589 - no: tracked values identical for max_repeat iterations (stalled)
590 """
591 effective_track: list[str] = track if track is not None else ["action"]
593 # Resolve each tracked key from context and hash the combined values.
594 # Keys may be bare names (e.g. "action") or namespaced (e.g. "context.action").
595 # Try namespaced forms first: context.<key>, captured.<key>, then bare ${key}.
596 parts: list[str] = []
597 for key in effective_track:
598 value: str = ""
599 if context is not None:
600 # If key already contains a dot it's already namespaced; use as-is.
601 if "." in key:
602 try:
603 value = str(interpolate(f"${{{key}}}", context))
604 except InterpolationError:
605 value = ""
606 else:
607 # Try context.<key> first, then captured.<key>, then give up.
608 resolved = False
609 for namespace in ("context", "captured", "prev", "result"):
610 try:
611 value = str(interpolate(f"${{{namespace}.{key}}}", context))
612 resolved = True
613 break
614 except InterpolationError:
615 continue
616 if not resolved:
617 value = ""
618 parts.append(f"{key}={value}")
620 combined = "|".join(parts)
621 current_hash = hashlib.md5(combined.encode()).hexdigest()
623 # Derive a stable cache key from the tracked keys
624 track_str = "|".join(sorted(effective_track))
625 cache_key = hashlib.md5(track_str.encode()).hexdigest()[:12]
626 loops_tmp = Path.cwd() / ".loops" / "tmp"
627 loops_tmp.mkdir(parents=True, exist_ok=True)
628 state_file = loops_tmp / f"ll-action-stall-{cache_key}.txt"
629 count_file = loops_tmp / f"ll-action-stall-{cache_key}.count"
631 # Read previous hash and stall count
632 previous_hash: str | None = None
633 stall_count = 0
634 try:
635 previous_hash = state_file.read_text().strip()
636 stall_count = int(count_file.read_text().strip())
637 except (FileNotFoundError, ValueError):
638 pass
640 # First iteration: save hash and report progress
641 if previous_hash is None:
642 state_file.write_text(current_hash)
643 count_file.write_text("0")
644 return EvaluationResult(
645 verdict="yes",
646 details={
647 "stall_count": 0,
648 "max_repeat": max_repeat,
649 "hash_changed": True,
650 "tracked_keys": effective_track,
651 },
652 )
654 hash_changed = current_hash != previous_hash
656 if hash_changed:
657 # Progress: update snapshot and reset counter
658 state_file.write_text(current_hash)
659 count_file.write_text("0")
660 return EvaluationResult(
661 verdict="yes",
662 details={
663 "stall_count": 0,
664 "max_repeat": max_repeat,
665 "hash_changed": True,
666 "tracked_keys": effective_track,
667 },
668 )
669 else:
670 # Same hash as last time
671 stall_count += 1
672 count_file.write_text(str(stall_count))
673 if stall_count >= max_repeat:
674 return EvaluationResult(
675 verdict="no",
676 details={
677 "stall_count": stall_count,
678 "max_repeat": max_repeat,
679 "hash_changed": False,
680 "tracked_keys": effective_track,
681 "repeated_hash": current_hash,
682 },
683 )
684 # Not yet at max_repeat threshold — still report yes so loop continues
685 return EvaluationResult(
686 verdict="yes",
687 details={
688 "stall_count": stall_count,
689 "max_repeat": max_repeat,
690 "hash_changed": False,
691 "tracked_keys": effective_track,
692 },
693 )
696def evaluate_mcp_result(output: str, exit_code: int) -> EvaluationResult:
697 """Evaluate an MCP tool call result from the mcp-call subprocess.
699 Maps exit codes and MCP response envelope fields to routing verdicts.
701 Exit code conventions (set by mcp-call):
702 0 → parse isError from JSON envelope
703 1 → tool_error (tool ran but isError: true)
704 124 → timeout (transport-level timeout)
705 127 → not_found (server or tool missing from .mcp.json)
707 Args:
708 output: stdout from mcp-call (MCP response envelope JSON)
709 exit_code: Exit code from mcp-call subprocess
711 Returns:
712 EvaluationResult with verdict:
713 - success → isError: false
714 - tool_error → isError: true
715 - not_found → server/tool not in .mcp.json (exit 127)
716 - timeout → transport-level timeout (exit 124)
717 """
718 if exit_code == 127:
719 return EvaluationResult(
720 verdict="not_found",
721 details={"exit_code": exit_code, "error": "Server or tool not found in .mcp.json"},
722 )
724 if exit_code == 124:
725 return EvaluationResult(
726 verdict="timeout",
727 details={"exit_code": exit_code, "error": "MCP tool call timed out"},
728 )
730 # Parse MCP envelope JSON from stdout
731 try:
732 envelope = json.loads(output.strip()) if output.strip() else {}
733 except json.JSONDecodeError:
734 return EvaluationResult(
735 verdict="tool_error",
736 details={
737 "exit_code": exit_code,
738 "error": f"Invalid JSON from mcp-call: {output[:200]}",
739 },
740 )
742 is_error = envelope.get("isError", exit_code != 0)
744 if is_error:
745 return EvaluationResult(
746 verdict="tool_error",
747 details={"exit_code": exit_code, "envelope": envelope},
748 )
750 return EvaluationResult(
751 verdict="success",
752 details={"exit_code": exit_code, "envelope": envelope},
753 )
756def evaluate_harbor_scorer(output: str, exit_code: int) -> EvaluationResult:
757 """Evaluate a Harbor-format benchmark scorer result.
759 The scorer is a shell command that prints a float score (0.0–1.0) to stdout
760 and exits 0 on success or non-zero on failure.
762 Args:
763 output: stdout from the scorer subprocess (expected: a bare float)
764 exit_code: Exit code from the scorer subprocess
766 Returns:
767 EvaluationResult with verdict:
768 - yes → exit 0 and stdout parses as a float
769 - no → exit non-zero (scorer determined failure)
770 - error → exit 0 but stdout is not parseable as a float
771 """
772 if exit_code != 0:
773 return EvaluationResult(
774 verdict="no",
775 details={"exit_code": exit_code},
776 )
778 try:
779 score = float(output.strip())
780 except (ValueError, AttributeError):
781 return EvaluationResult(
782 verdict="error",
783 details={
784 "exit_code": exit_code,
785 "error": f"Scorer stdout is not a float: {output[:200]}",
786 },
787 )
789 return EvaluationResult(
790 verdict="yes",
791 details={"score": score, "exit_code": 0},
792 )
795def evaluate_llm_structured(
796 output: str,
797 prompt: str | None = None,
798 schema: dict[str, Any] | None = None,
799 min_confidence: float = 0.5,
800 uncertain_suffix: bool = False,
801 model: str = DEFAULT_LLM_MODEL,
802 max_tokens: int = 256,
803 timeout: int = 1800,
804) -> EvaluationResult:
805 """Evaluate action output using LLM with structured output via Claude CLI.
807 This is the ONLY place in the FSM system that uses LLM structured output.
808 Requires the ``claude`` CLI to be installed and authenticated.
810 Args:
811 output: Action stdout to evaluate
812 prompt: Custom evaluation prompt (defaults to basic success check)
813 schema: Custom JSON schema for structured response
814 min_confidence: Minimum confidence threshold (0-1)
815 uncertain_suffix: If True, append _uncertain to low-confidence verdicts
816 model: Model identifier (CLI aliases like "sonnet" or full names)
817 max_tokens: Maximum tokens for response (passed to --max-turns is not
818 applicable; kept for signature compat)
819 timeout: Timeout in seconds
821 Returns:
822 EvaluationResult with verdict from LLM and confidence/reason in details
823 """
824 effective_schema = schema or DEFAULT_LLM_SCHEMA
825 effective_prompt = prompt or DEFAULT_LLM_PROMPT
827 # Truncate output to avoid context limits (keep last 4000 chars)
828 truncated = output[-4000:] if len(output) > 4000 else output
830 user_prompt = f"{effective_prompt}\n\n<action_output>\n{truncated}\n</action_output>"
832 invocation = resolve_host().build_blocking_json(prompt=user_prompt, model=model)
833 # Builder drops json_schema (Protocol surface only) and omits the
834 # claude-CLI-specific --no-session-persistence flag; augment at call site.
835 args = list(invocation.args) + [
836 "--json-schema",
837 json.dumps(effective_schema),
838 "--no-session-persistence",
839 ]
841 t0 = time.monotonic()
842 try:
843 proc = subprocess.run(
844 [invocation.binary, *args], capture_output=True, text=True, timeout=timeout
845 )
846 except subprocess.TimeoutExpired:
847 return EvaluationResult(
848 verdict="error",
849 details={"error": "LLM evaluation timeout", "timeout": True},
850 )
851 except FileNotFoundError:
852 return EvaluationResult(
853 verdict="error",
854 details={
855 "error": f"{invocation.binary} CLI not found. Install the active host CLI (see LL_HOST_CLI).",
856 "missing_dependency": True,
857 },
858 )
859 llm_latency_ms = int((time.monotonic() - t0) * 1000)
861 if proc.returncode != 0:
862 return EvaluationResult(
863 verdict="error",
864 details={
865 "error": f"{invocation.binary} CLI error: {proc.stderr.strip()}",
866 "api_error": True,
867 },
868 )
870 # Guard: empty stdout with exit 0 (API error not reflected in exit code)
871 if not proc.stdout.strip():
872 stderr_info = proc.stderr.strip()[:200] if proc.stderr else ""
873 error_msg = f"{invocation.binary} CLI returned empty output"
874 if stderr_info:
875 error_msg += f" (stderr: {stderr_info})"
876 return EvaluationResult(
877 verdict="error",
878 details={"error": error_msg, "empty_output": True},
879 )
881 # Parse the CLI JSON envelope and extract structured result.
882 # With --json-schema the envelope is:
883 # success: {"type":"result","subtype":"success","structured_output":{...},...}
884 # failure: {"type":"result","subtype":"error_max_structured_output_retries",...}
885 # If stdout is JSONL (multiple JSON objects), use the last non-empty line.
886 try:
887 stdout = proc.stdout.strip()
888 try:
889 envelope = json.loads(stdout)
890 except json.JSONDecodeError:
891 # Try JSONL: take the last non-empty line
892 lines = [line for line in stdout.split("\n") if line.strip()]
893 if not lines:
894 raise
895 envelope = json.loads(lines[-1])
897 # Check structured-output retry exhaustion (--json-schema failure mode)
898 if envelope.get("subtype") == "error_max_structured_output_retries":
899 return EvaluationResult(
900 verdict="error",
901 details={
902 "error": "Claude CLI could not produce valid structured output after retries",
903 "api_error": True,
904 },
905 )
907 # Check legacy is_error flag (some CLI versions exit 0 but report error in envelope)
908 if envelope.get("is_error", False):
909 err_text = str(envelope.get("result", "") or "")[:200]
910 return EvaluationResult(
911 verdict="error",
912 details={"error": f"Claude CLI reported error: {err_text}", "api_error": True},
913 )
915 # --json-schema mode returns validated dict in "structured_output"
916 if isinstance(envelope.get("structured_output"), dict):
917 llm_result: dict[str, Any] = envelope["structured_output"]
918 else:
919 raw_result = envelope.get("result", "")
920 if isinstance(raw_result, dict):
921 llm_result = raw_result
922 elif raw_result:
923 llm_result = json.loads(raw_result)
924 elif "verdict" in envelope:
925 llm_result = envelope
926 else:
927 raw_preview = proc.stdout[:300]
928 return EvaluationResult(
929 verdict="error",
930 details={
931 "error": "Empty result field in Claude CLI response",
932 "raw_preview": raw_preview,
933 },
934 )
935 except (json.JSONDecodeError, TypeError, ValueError) as e:
936 raw_preview = proc.stdout[:300] if proc.stdout else "(empty)"
937 return EvaluationResult(
938 verdict="error",
939 details={"error": f"Failed to parse LLM response: {e}", "raw_preview": raw_preview},
940 )
942 # Build result with confidence handling
943 verdict = str(llm_result.get("verdict", "error"))
944 confidence = float(llm_result.get("confidence", 1.0))
945 confident = confidence >= min_confidence
947 # Optionally modify verdict for low confidence
948 if uncertain_suffix and not confident:
949 verdict = f"{verdict}_uncertain"
951 return EvaluationResult(
952 verdict=verdict,
953 details={
954 "confidence": confidence,
955 "confident": confident,
956 "reason": llm_result.get("reason", ""),
957 "raw": llm_result,
958 "llm_model": model,
959 "llm_latency_ms": llm_latency_ms,
960 "llm_prompt": user_prompt[:500],
961 "llm_raw_output": proc.stdout[:500] if proc.stdout else "",
962 },
963 )
966def evaluate_blind_comparator(
967 output_harness: str,
968 output_baseline: str,
969 prompt: str | None = None,
970 model: str = DEFAULT_LLM_MODEL,
971 timeout: int = 1800,
972) -> dict[str, Any]:
973 """Blindly evaluate two outputs, returning pass/fail for each arm.
975 Outputs are randomly labeled "Output A" / "Output B" so the LLM judge
976 cannot distinguish the harness arm from the baseline arm. The mapping is
977 de-anonymized after judgment so callers receive harness/baseline verdicts.
979 Args:
980 output_harness: stdout from the harness (gated) arm
981 output_baseline: stdout from the baseline (ungated) arm
982 prompt: Custom evaluation prompt (appended to default framing)
983 model: Model identifier for the judge
984 timeout: Timeout in seconds
986 Returns:
987 Dict with keys: harness_pass (bool), baseline_pass (bool),
988 confidence (float), reason (str), raw (dict with A/B verdicts)
989 """
990 effective_prompt = prompt or DEFAULT_BLIND_COMPARATOR_PROMPT
992 # Truncate outputs to avoid context limits
993 truncated_harness = output_harness[-4000:] if len(output_harness) > 4000 else output_harness
994 truncated_baseline = output_baseline[-4000:] if len(output_baseline) > 4000 else output_baseline
996 # Randomize order: coin flip determines whether harness→A / baseline→B
997 harness_is_a = random.choice([True, False])
998 if harness_is_a:
999 output_a, output_b = truncated_harness, truncated_baseline
1000 else:
1001 output_a, output_b = truncated_baseline, truncated_harness
1003 user_prompt = (
1004 f"{effective_prompt}\n\n"
1005 f"<output_a>\n{output_a}\n</output_a>\n\n"
1006 f"<output_b>\n{output_b}\n</output_b>"
1007 )
1009 invocation = resolve_host().build_blocking_json(prompt=user_prompt, model=model)
1010 args = list(invocation.args) + [
1011 "--json-schema",
1012 json.dumps(BLIND_COMPARATOR_SCHEMA),
1013 "--no-session-persistence",
1014 ]
1016 try:
1017 proc = subprocess.run(
1018 [invocation.binary, *args], capture_output=True, text=True, timeout=timeout
1019 )
1020 except subprocess.TimeoutExpired:
1021 # On timeout, both fail — conservative default
1022 return {
1023 "harness_pass": False,
1024 "baseline_pass": False,
1025 "confidence": 0.0,
1026 "reason": "LLM evaluation timed out",
1027 "raw": {"verdict_a": "timeout", "verdict_b": "timeout"},
1028 "error": "timeout",
1029 }
1030 except FileNotFoundError:
1031 return {
1032 "harness_pass": False,
1033 "baseline_pass": False,
1034 "confidence": 0.0,
1035 "reason": f"{invocation.binary} CLI not found",
1036 "raw": {"verdict_a": "error", "verdict_b": "error"},
1037 "error": "missing_cli",
1038 }
1040 if proc.returncode != 0:
1041 return {
1042 "harness_pass": False,
1043 "baseline_pass": False,
1044 "confidence": 0.0,
1045 "reason": f"Judge CLI error: {proc.stderr.strip()[:200]}",
1046 "raw": {"verdict_a": "error", "verdict_b": "error"},
1047 "error": "api_error",
1048 }
1050 if not proc.stdout.strip():
1051 return {
1052 "harness_pass": False,
1053 "baseline_pass": False,
1054 "confidence": 0.0,
1055 "reason": "Judge returned empty output",
1056 "raw": {"verdict_a": "error", "verdict_b": "error"},
1057 "error": "empty_output",
1058 }
1060 try:
1061 stdout = proc.stdout.strip()
1062 try:
1063 envelope = json.loads(stdout)
1064 except json.JSONDecodeError:
1065 lines = [line for line in stdout.split("\n") if line.strip()]
1066 if not lines:
1067 raise
1068 envelope = json.loads(lines[-1])
1070 if envelope.get("subtype") == "error_max_structured_output_retries":
1071 return {
1072 "harness_pass": False,
1073 "baseline_pass": False,
1074 "confidence": 0.0,
1075 "reason": "Judge could not produce valid structured output after retries",
1076 "raw": {"verdict_a": "error", "verdict_b": "error"},
1077 "error": "retry_exhausted",
1078 }
1080 if envelope.get("is_error", False):
1081 err_text = str(envelope.get("result", "") or "")[:200]
1082 return {
1083 "harness_pass": False,
1084 "baseline_pass": False,
1085 "confidence": 0.0,
1086 "reason": f"Judge reported error: {err_text}",
1087 "raw": {"verdict_a": "error", "verdict_b": "error"},
1088 "error": "api_error",
1089 }
1091 if isinstance(envelope.get("structured_output"), dict):
1092 result: dict[str, Any] = envelope["structured_output"]
1093 else:
1094 raw_result = envelope.get("result", "")
1095 if isinstance(raw_result, dict):
1096 result = raw_result
1097 elif raw_result:
1098 result = json.loads(raw_result)
1099 else:
1100 return {
1101 "harness_pass": False,
1102 "baseline_pass": False,
1103 "confidence": 0.0,
1104 "reason": "Empty result field in judge response",
1105 "raw": {"verdict_a": "error", "verdict_b": "error"},
1106 "error": "empty_result",
1107 }
1108 except (json.JSONDecodeError, TypeError, ValueError):
1109 return {
1110 "harness_pass": False,
1111 "baseline_pass": False,
1112 "confidence": 0.0,
1113 "reason": "Failed to parse judge response",
1114 "raw": {"verdict_a": "error", "verdict_b": "error"},
1115 "error": "parse_error",
1116 }
1118 # De-anonymize
1119 verdict_a = str(result.get("verdict_a", "no"))
1120 verdict_b = str(result.get("verdict_b", "no"))
1121 confidence = float(result.get("confidence", 0.0))
1122 reason = str(result.get("reason", ""))
1124 if harness_is_a:
1125 harness_pass = verdict_a == "yes"
1126 baseline_pass = verdict_b == "yes"
1127 else:
1128 harness_pass = verdict_b == "yes"
1129 baseline_pass = verdict_a == "yes"
1131 return {
1132 "harness_pass": harness_pass,
1133 "baseline_pass": baseline_pass,
1134 "confidence": confidence,
1135 "reason": reason,
1136 "raw": {"verdict_a": verdict_a, "verdict_b": verdict_b, "harness_is_a": harness_is_a},
1137 }
1140def evaluate_contract(
1141 config: EvaluateConfig,
1142 context: InterpolationContext,
1143 model: str = DEFAULT_LLM_MODEL,
1144 timeout: int = 1800,
1145) -> EvaluationResult:
1146 """Evaluate producer/consumer contract alignment using an LLM judge.
1148 Reads each producer/consumer file pair, applies optional regex extraction,
1149 then asks an LLM judge whether the producer satisfies the consumer contract.
1150 Returns yes only when all pairs align; any failure routes no/error.
1152 Args:
1153 config: EvaluateConfig with type="contract" and pairs list
1154 context: Interpolation context (unused by this evaluator directly)
1155 model: LLM model identifier
1156 timeout: Subprocess timeout in seconds
1158 Returns:
1159 EvaluationResult with verdict yes/no/error and pair_results in details
1160 """
1161 pairs = config.pairs
1162 if not pairs:
1163 return EvaluationResult(
1164 verdict="error",
1165 details={"error": "contract evaluator requires at least one pair in evaluate.pairs"},
1166 )
1168 contract_schema = {
1169 "type": "object",
1170 "properties": {
1171 "verdict": {"type": "string", "enum": ["yes", "no"]},
1172 "confidence": {"type": "number"},
1173 "reason": {"type": "string"},
1174 },
1175 "required": ["verdict", "confidence", "reason"],
1176 }
1178 pair_results: list[dict[str, Any]] = []
1180 for pair in pairs:
1181 producer_path = pair.get("producer", "")
1182 consumer_path = pair.get("consumer", "")
1183 producer_pattern = pair.get("producer_pattern")
1184 consumer_pattern = pair.get("consumer_pattern")
1185 contract_rule = pair.get("contract", "the producer and consumer must be compatible")
1187 # Read producer file
1188 try:
1189 producer_content = Path(producer_path).read_text()
1190 except OSError as e:
1191 pair_results.append(
1192 {
1193 "producer": producer_path,
1194 "consumer": consumer_path,
1195 "verdict": "error",
1196 "error": f"cannot read producer file: {e}",
1197 }
1198 )
1199 continue
1201 # Read consumer file
1202 try:
1203 consumer_content = Path(consumer_path).read_text()
1204 except OSError as e:
1205 pair_results.append(
1206 {
1207 "producer": producer_path,
1208 "consumer": consumer_path,
1209 "verdict": "error",
1210 "error": f"cannot read consumer file: {e}",
1211 }
1212 )
1213 continue
1215 # Apply optional regex extraction
1216 if producer_pattern:
1217 matches = re.findall(producer_pattern, producer_content, re.DOTALL)
1218 if not matches:
1219 pair_results.append(
1220 {
1221 "producer": producer_path,
1222 "consumer": consumer_path,
1223 "verdict": "error",
1224 "error": f"producer_pattern matched nothing in {producer_path}",
1225 }
1226 )
1227 continue
1228 producer_slice = "\n".join(matches)
1229 else:
1230 producer_slice = (
1231 producer_content[-4000:] if len(producer_content) > 4000 else producer_content
1232 )
1234 if consumer_pattern:
1235 matches = re.findall(consumer_pattern, consumer_content, re.DOTALL)
1236 if not matches:
1237 pair_results.append(
1238 {
1239 "producer": producer_path,
1240 "consumer": consumer_path,
1241 "verdict": "error",
1242 "error": f"consumer_pattern matched nothing in {consumer_path}",
1243 }
1244 )
1245 continue
1246 consumer_slice = "\n".join(matches)
1247 else:
1248 consumer_slice = (
1249 consumer_content[-4000:] if len(consumer_content) > 4000 else consumer_content
1250 )
1252 judge_prompt = (
1253 f"You are evaluating whether a producer output satisfies a consumer contract.\n\n"
1254 f"Contract rule: {contract_rule}\n\n"
1255 f'<producer path="{producer_path}">\n{producer_slice}\n</producer>\n\n'
1256 f'<consumer path="{consumer_path}">\n{consumer_slice}\n</consumer>\n\n'
1257 "Does the producer satisfy the consumer contract? "
1258 "Consider field names, types, casing, and structure. "
1259 "Answer yes if aligned, no if mismatched."
1260 )
1262 invocation = resolve_host().build_blocking_json(prompt=judge_prompt, model=model)
1263 args = list(invocation.args) + [
1264 "--json-schema",
1265 json.dumps(contract_schema),
1266 "--no-session-persistence",
1267 ]
1269 t0 = time.monotonic()
1270 try:
1271 proc = subprocess.run(
1272 [invocation.binary, *args], capture_output=True, text=True, timeout=timeout
1273 )
1274 except subprocess.TimeoutExpired:
1275 pair_results.append(
1276 {
1277 "producer": producer_path,
1278 "consumer": consumer_path,
1279 "verdict": "error",
1280 "error": "LLM judge timed out",
1281 "llm_latency_ms": int((time.monotonic() - t0) * 1000),
1282 }
1283 )
1284 continue
1285 except FileNotFoundError:
1286 return EvaluationResult(
1287 verdict="error",
1288 details={
1289 "error": f"{invocation.binary} CLI not found. Install the active host CLI (see LL_HOST_CLI).",
1290 "missing_dependency": True,
1291 },
1292 )
1293 llm_latency_ms = int((time.monotonic() - t0) * 1000)
1295 if proc.returncode != 0:
1296 pair_results.append(
1297 {
1298 "producer": producer_path,
1299 "consumer": consumer_path,
1300 "verdict": "error",
1301 "error": f"CLI error: {proc.stderr.strip()}",
1302 "llm_latency_ms": llm_latency_ms,
1303 }
1304 )
1305 continue
1307 if not proc.stdout.strip():
1308 pair_results.append(
1309 {
1310 "producer": producer_path,
1311 "consumer": consumer_path,
1312 "verdict": "error",
1313 "error": "CLI returned empty output",
1314 "llm_latency_ms": llm_latency_ms,
1315 }
1316 )
1317 continue
1319 try:
1320 stdout = proc.stdout.strip()
1321 try:
1322 envelope = json.loads(stdout)
1323 except json.JSONDecodeError:
1324 lines = [line for line in stdout.split("\n") if line.strip()]
1325 if not lines:
1326 raise
1327 envelope = json.loads(lines[-1])
1329 if envelope.get("subtype") == "error_max_structured_output_retries":
1330 pair_results.append(
1331 {
1332 "producer": producer_path,
1333 "consumer": consumer_path,
1334 "verdict": "error",
1335 "error": "Claude CLI could not produce valid structured output after retries",
1336 "llm_latency_ms": llm_latency_ms,
1337 }
1338 )
1339 continue
1341 if envelope.get("is_error", False):
1342 err_text = str(envelope.get("result", "") or "")[:200]
1343 pair_results.append(
1344 {
1345 "producer": producer_path,
1346 "consumer": consumer_path,
1347 "verdict": "error",
1348 "error": f"Claude CLI reported error: {err_text}",
1349 "llm_latency_ms": llm_latency_ms,
1350 }
1351 )
1352 continue
1354 if isinstance(envelope.get("structured_output"), dict):
1355 llm_result: dict[str, Any] = envelope["structured_output"]
1356 else:
1357 raw_result = envelope.get("result", "")
1358 if isinstance(raw_result, dict):
1359 llm_result = raw_result
1360 elif raw_result:
1361 llm_result = json.loads(raw_result)
1362 elif "verdict" in envelope:
1363 llm_result = envelope
1364 else:
1365 pair_results.append(
1366 {
1367 "producer": producer_path,
1368 "consumer": consumer_path,
1369 "verdict": "error",
1370 "error": "empty result field in CLI response",
1371 "llm_latency_ms": llm_latency_ms,
1372 }
1373 )
1374 continue
1376 except (json.JSONDecodeError, TypeError, ValueError) as e:
1377 pair_results.append(
1378 {
1379 "producer": producer_path,
1380 "consumer": consumer_path,
1381 "verdict": "error",
1382 "error": f"failed to parse LLM response: {e}",
1383 "llm_latency_ms": llm_latency_ms,
1384 }
1385 )
1386 continue
1388 pair_results.append(
1389 {
1390 "producer": producer_path,
1391 "consumer": consumer_path,
1392 "verdict": str(llm_result.get("verdict", "error")),
1393 "confidence": float(llm_result.get("confidence", 1.0)),
1394 "reason": llm_result.get("reason", ""),
1395 "llm_latency_ms": llm_latency_ms,
1396 }
1397 )
1399 # Aggregate: yes only if all pairs aligned; error takes precedence over no
1400 if any(p["verdict"] == "error" for p in pair_results):
1401 overall = "error"
1402 elif all(p["verdict"] == "yes" for p in pair_results):
1403 overall = "yes"
1404 else:
1405 overall = "no"
1407 return EvaluationResult(
1408 verdict=overall,
1409 details={"pair_results": pair_results},
1410 )
1413def evaluate_comparator(
1414 config: EvaluateConfig,
1415 output: str,
1416 context: InterpolationContext,
1417) -> EvaluationResult:
1418 """Evaluate using blind A/B comparison against a stored baseline."""
1419 from pathlib import Path
1421 if config.baseline_path is None:
1422 return EvaluationResult(
1423 verdict="no_baseline",
1424 details={"reason": "No baseline_path configured"},
1425 )
1427 baseline_file = Path(config.baseline_path) / "output.txt"
1428 if not baseline_file.exists():
1429 if config.auto_promote:
1430 baseline_file.parent.mkdir(parents=True, exist_ok=True)
1431 baseline_file.write_text(output)
1432 return EvaluationResult(
1433 verdict="yes",
1434 details={
1435 "reason": "No baseline found; current output promoted as new baseline.",
1436 "bootstrapped": True,
1437 },
1438 )
1439 return EvaluationResult(
1440 verdict="no_baseline",
1441 details={"reason": f"Baseline file not found: {baseline_file}"},
1442 )
1444 baseline_text = baseline_file.read_text()
1445 min_pairs = max(1, config.min_pairs if config.min_pairs is not None else 1)
1446 harness_wins = 0
1447 baseline_wins = 0
1448 last_reason = ""
1449 last_raw: dict[str, Any] = {}
1451 for _ in range(min_pairs):
1452 result = evaluate_blind_comparator(output, baseline_text, prompt=config.prompt)
1453 if result.get("harness_pass"):
1454 harness_wins += 1
1455 if result.get("baseline_pass"):
1456 baseline_wins += 1
1457 last_reason = result.get("reason", "")
1458 last_raw = result.get("raw", {})
1460 if harness_wins > baseline_wins:
1461 verdict = "yes"
1462 elif baseline_wins > harness_wins:
1463 verdict = "no"
1464 else:
1465 verdict = "tie"
1467 if config.auto_promote and verdict == "yes":
1468 baseline_file.write_text(output)
1470 return EvaluationResult(
1471 verdict=verdict,
1472 details={
1473 "harness_wins": harness_wins,
1474 "baseline_wins": baseline_wins,
1475 "min_pairs": min_pairs,
1476 "reason": last_reason,
1477 "raw": last_raw,
1478 },
1479 )
1482def evaluate(
1483 config: EvaluateConfig,
1484 output: str,
1485 exit_code: int,
1486 context: InterpolationContext,
1487) -> EvaluationResult:
1488 """Dispatch to appropriate evaluator based on config type.
1490 Args:
1491 config: Evaluator configuration with type and parameters
1492 output: Action stdout
1493 exit_code: Action exit code
1494 context: Runtime context for variable interpolation
1496 Returns:
1497 EvaluationResult from the appropriate evaluator
1499 Raises:
1500 ValueError: If evaluator type is unknown
1501 """
1502 eval_type = config.type
1504 # BUG-1640: Action-level timeouts (exit_code=124) short-circuit to "error"
1505 # so loop authors' on_error: branches fire instead of being routed via
1506 # on_no: based on truncated output. mcp_result is exempted because it has
1507 # its own established "timeout" verdict (see evaluate_mcp_result).
1508 if exit_code == 124 and eval_type != "mcp_result":
1509 return EvaluationResult(
1510 verdict="error",
1511 details={"exit_code": exit_code, "error": "action timed out"},
1512 )
1514 # BUG-1815: Non-timeout non-zero exit codes short-circuit to "error" for
1515 # evaluator types that don't intrinsically check exit codes. Exit-code-aware
1516 # evaluators (exit_code, mcp_result, harbor_scorer, diff_stall, llm_structured)
1517 # are exempt because they handle exit codes via their own logic.
1518 _EXIT_CODE_AWARE_EVALUATORS: frozenset[str] = frozenset(
1519 {
1520 "exit_code",
1521 "mcp_result",
1522 "harbor_scorer",
1523 "diff_stall",
1524 "action_stall",
1525 "llm_structured",
1526 "contract",
1527 }
1528 )
1529 if exit_code != 0 and eval_type not in _EXIT_CODE_AWARE_EVALUATORS:
1530 return EvaluationResult(
1531 verdict="error",
1532 details={
1533 "exit_code": exit_code,
1534 "error": f"action exited with code {exit_code}",
1535 },
1536 )
1538 if eval_type == "exit_code":
1539 return evaluate_exit_code(exit_code)
1541 elif eval_type == "output_numeric":
1542 if config.target is None:
1543 raise ValueError("output_numeric evaluator requires 'target' to be set")
1544 elif isinstance(config.target, str):
1545 try:
1546 resolved = interpolate(config.target, context) if context else config.target
1547 numeric_target = float(resolved)
1548 except (InterpolationError, ValueError) as e:
1549 raise ValueError(
1550 f"output_numeric target must be numeric, got: {config.target!r}"
1551 ) from e
1552 else:
1553 numeric_target = float(config.target)
1554 return evaluate_output_numeric(
1555 output=output,
1556 operator=config.operator or "eq",
1557 target=numeric_target,
1558 )
1560 elif eval_type == "output_json":
1561 return evaluate_output_json(
1562 output=output,
1563 path=config.path or "",
1564 operator=config.operator or "eq",
1565 target=config.target,
1566 )
1568 elif eval_type == "output_contains":
1569 return evaluate_output_contains(
1570 output=output,
1571 pattern=config.pattern or "",
1572 negate=config.negate,
1573 )
1575 elif eval_type == "convergence":
1576 # Resolve previous value from interpolation if configured
1577 previous: float | None = None
1578 if config.previous:
1579 try:
1580 previous = float(interpolate(config.previous, context))
1581 except (InterpolationError, ValueError):
1582 # Previous unavailable on first iteration, continue with None
1583 pass
1585 # Parse current value from output
1586 try:
1587 current = float(output.strip())
1588 except ValueError:
1589 return EvaluationResult(
1590 verdict="error",
1591 details={"error": f"Cannot parse output as number: {output[:100]}"},
1592 )
1594 # Resolve target (may be interpolated string like "${context.target}")
1595 convergence_target: float
1596 if isinstance(config.target, str):
1597 try:
1598 convergence_target = float(interpolate(config.target, context))
1599 except (InterpolationError, ValueError) as e:
1600 return EvaluationResult(
1601 verdict="error",
1602 details={"error": f"Cannot resolve target: {e}"},
1603 )
1604 else:
1605 if config.target is None:
1606 raise ValueError("convergence evaluator requires 'target' to be set")
1607 convergence_target = float(config.target)
1609 # Resolve tolerance (may be interpolated string)
1610 tolerance: float = 0.0
1611 if config.tolerance is not None:
1612 if isinstance(config.tolerance, str):
1613 try:
1614 tolerance = float(interpolate(config.tolerance, context))
1615 except (InterpolationError, ValueError):
1616 tolerance = 0.0
1617 else:
1618 tolerance = float(config.tolerance)
1620 return evaluate_convergence(
1621 current=current,
1622 previous=previous,
1623 target=convergence_target,
1624 tolerance=tolerance,
1625 direction=config.direction,
1626 )
1628 elif eval_type == "diff_stall":
1629 return evaluate_diff_stall(
1630 scope=config.scope,
1631 max_stall=config.max_stall,
1632 )
1634 elif eval_type == "action_stall":
1635 return evaluate_action_stall(
1636 track=config.track,
1637 max_repeat=config.max_repeat,
1638 context=context,
1639 )
1641 elif eval_type == "llm_structured":
1642 prompt = config.prompt
1643 if prompt and context:
1644 try:
1645 prompt = interpolate(prompt, context)
1646 except InterpolationError:
1647 pass # Use raw prompt on resolution failure
1648 return evaluate_llm_structured(
1649 output=output,
1650 prompt=prompt,
1651 schema=config.schema,
1652 min_confidence=config.min_confidence,
1653 uncertain_suffix=config.uncertain_suffix,
1654 )
1656 elif eval_type == "mcp_result":
1657 return evaluate_mcp_result(output=output, exit_code=exit_code)
1659 elif eval_type == "harbor_scorer":
1660 return evaluate_harbor_scorer(output=output, exit_code=exit_code)
1662 elif eval_type == "comparator":
1663 return evaluate_comparator(config=config, output=output, context=context)
1665 elif eval_type == "contract":
1666 return evaluate_contract(config=config, context=context)
1668 elif eval_type == "classify":
1669 return evaluate_classify(output=output, line=config.line)
1671 else:
1672 raise ValueError(f"Unknown evaluator type: {eval_type}")