Coverage for little_loops / fsm / evaluators.py: 9%
516 statements
« prev ^ index » next coverage.py v7.12.0, created at 2026-06-08 15:34 -0500
1"""FSM Evaluators for loop execution.
3This module provides evaluators that interpret action output and produce
4verdicts for state transitions.
6Supported evaluator types:
8Tier 1 (Deterministic - no API calls):
9 exit_code: Map Unix exit codes to verdicts (0=success, 1=failure, 2+=error)
10 output_numeric: Compare numeric output to target value
11 output_json: Extract and compare JSON path values
12 output_contains: Pattern matching on stdout
13 convergence: Track progress toward a target value
14 diff_stall: Detect stalled iterations via git diff comparison
15 action_stall: Detect when the same action string or output repeats for N consecutive iterations
16 harbor_scorer: Interpret Harbor-format benchmark scorer exit code and float stdout
18Tier 2 (LLM-based):
19 llm_structured: Use LLM with structured output for natural language evaluation
20 contract: Read producer/consumer file pairs and assert contract alignment via LLM judge
22Tier 3 (External process):
23 mcp_result: Parse MCP tool call response envelope
24"""
26from __future__ import annotations
28import hashlib
29import json
30import random
31import re
32import subprocess
33import time
34from collections.abc import Callable
35from dataclasses import dataclass
36from pathlib import Path
37from typing import Any
39from little_loops.fsm.interpolation import (
40 InterpolationContext,
41 InterpolationError,
42 interpolate,
43)
44from little_loops.fsm.schema import DEFAULT_LLM_MODEL, EvaluateConfig
45from little_loops.host_runner import resolve_host
48@dataclass
49class EvaluationResult:
50 """Result from an evaluator.
52 Attributes:
53 verdict: The routing key for state transitions
54 details: Evaluator-specific metadata for debugging/logging
55 """
57 verdict: str
58 details: dict[str, Any]
61# Default schema for LLM structured evaluation
62DEFAULT_LLM_SCHEMA: dict[str, Any] = {
63 "type": "object",
64 "properties": {
65 "verdict": {
66 "type": "string",
67 "enum": ["yes", "no", "blocked", "partial"],
68 "description": (
69 "- yes: The condition/check evaluated to true\n"
70 "- no: The condition/check evaluated to false\n"
71 "- blocked: Cannot proceed without external help\n"
72 "- partial: Made progress but not complete"
73 ),
74 },
75 "confidence": {
76 "type": "number",
77 "minimum": 0,
78 "maximum": 1,
79 "description": "Confidence in this verdict (0-1)",
80 },
81 "reason": {
82 "type": "string",
83 "description": "Brief explanation",
84 },
85 },
86 "required": ["verdict", "confidence", "reason"],
87}
89DEFAULT_LLM_PROMPT = "Evaluate whether this action succeeded based on its output."
91# Schema for blind A/B comparator: evaluates two anonymized outputs
92BLIND_COMPARATOR_SCHEMA: dict[str, Any] = {
93 "type": "object",
94 "properties": {
95 "verdict_a": {
96 "type": "string",
97 "enum": ["yes", "no"],
98 "description": "Whether Output A meets the evaluation criteria",
99 },
100 "verdict_b": {
101 "type": "string",
102 "enum": ["yes", "no"],
103 "description": "Whether Output B meets the evaluation criteria",
104 },
105 "confidence": {
106 "type": "number",
107 "minimum": 0,
108 "maximum": 1,
109 "description": "Confidence in these verdicts (0-1)",
110 },
111 "reason": {
112 "type": "string",
113 "description": "Brief explanation comparing the two outputs",
114 },
115 },
116 "required": ["verdict_a", "verdict_b", "confidence", "reason"],
117}
119DEFAULT_BLIND_COMPARATOR_PROMPT = (
120 "You are evaluating two outputs (labeled 'Output A' and 'Output B') that were "
121 "produced by independent runs of the same task. Judge whether each output meets "
122 "the evaluation criteria below. Be objective and impartial — the labels 'A' and "
123 "'B' are arbitrary and do not indicate which is better."
124)
126_NUMERIC_OPERATORS: dict[str, Callable[[float, float], bool]] = {
127 "eq": lambda v, t: v == t,
128 "ne": lambda v, t: v != t,
129 "lt": lambda v, t: v < t,
130 "le": lambda v, t: v <= t,
131 "gt": lambda v, t: v > t,
132 "ge": lambda v, t: v >= t,
133}
136def evaluate_exit_code(exit_code: int) -> EvaluationResult:
137 """Map Unix exit code to verdict.
139 Args:
140 exit_code: The process exit code
142 Returns:
143 EvaluationResult with verdict:
144 - 0 -> yes
145 - 1 -> no
146 - 2+ -> error
147 """
148 if exit_code == 0:
149 verdict = "yes"
150 elif exit_code == 1:
151 verdict = "no"
152 else:
153 verdict = "error"
155 return EvaluationResult(verdict=verdict, details={"exit_code": exit_code})
158def evaluate_output_numeric(
159 output: str,
160 operator: str,
161 target: float,
162) -> EvaluationResult:
163 """Parse stdout as number and compare to target.
165 Args:
166 output: The action stdout to parse as a number
167 operator: Comparison operator (eq, ne, lt, le, gt, ge)
168 target: Target value to compare against
170 Returns:
171 EvaluationResult with verdict:
172 - Condition met -> yes
173 - Condition not met -> no
174 - Parse error -> error
175 """
176 try:
177 value = float(output.strip())
178 except ValueError:
179 return EvaluationResult(
180 verdict="error",
181 details={"error": f"Cannot parse as number: {output[:100]}"},
182 )
184 if operator not in _NUMERIC_OPERATORS:
185 return EvaluationResult(
186 verdict="error",
187 details={"error": f"Unknown operator: {operator}"},
188 )
190 condition_met = _NUMERIC_OPERATORS[operator](value, target)
191 return EvaluationResult(
192 verdict="yes" if condition_met else "no",
193 details={"value": value, "target": target, "operator": operator},
194 )
197def _extract_json_path(data: Any, path: str) -> Any:
198 """Extract value from dict using jq-style path like '.summary.failed'.
200 Args:
201 data: The parsed JSON data (dict or list)
202 path: Dot-separated path, optionally starting with '.'
204 Returns:
205 The value at the specified path
207 Raises:
208 KeyError: If path not found in data
209 """
210 if path.startswith("."):
211 path = path[1:]
212 parts = path.split(".")
213 current = data
214 for part in parts:
215 if isinstance(current, dict) and part in current:
216 current = current[part]
217 elif isinstance(current, list) and part.isdigit():
218 idx = int(part)
219 if 0 <= idx < len(current):
220 current = current[idx]
221 else:
222 raise KeyError(path)
223 else:
224 raise KeyError(path)
225 return current
228def _compare_values(
229 value: int | float, operator: str, target: int | float, path: str
230) -> EvaluationResult:
231 """Compare numeric values using operator.
233 Args:
234 value: The extracted value to compare
235 operator: Comparison operator
236 target: Target value
237 path: JSON path for details
239 Returns:
240 EvaluationResult with comparison result
241 """
242 if operator not in _NUMERIC_OPERATORS:
243 return EvaluationResult(
244 verdict="error",
245 details={"error": f"Unknown operator: {operator}"},
246 )
248 condition_met = _NUMERIC_OPERATORS[operator](value, target)
249 return EvaluationResult(
250 verdict="yes" if condition_met else "no",
251 details={"value": value, "path": path, "target": target, "operator": operator},
252 )
255def evaluate_output_json(
256 output: str,
257 path: str,
258 operator: str,
259 target: Any,
260) -> EvaluationResult:
261 """Parse JSON and extract value at path, then compare.
263 Args:
264 output: The action stdout containing JSON
265 path: jq-style dot notation path (e.g., '.summary.failed')
266 operator: Comparison operator (eq, ne, lt, le, gt, ge)
267 target: Target value for comparison
269 Returns:
270 EvaluationResult with verdict:
271 - Condition met -> yes
272 - Condition not met -> no
273 - Parse/path error -> error
274 """
275 try:
276 data = json.loads(output)
277 except json.JSONDecodeError as e:
278 return EvaluationResult(
279 verdict="error",
280 details={"error": f"Invalid JSON: {e}"},
281 )
283 try:
284 value = _extract_json_path(data, path)
285 except KeyError:
286 return EvaluationResult(
287 verdict="error",
288 details={"error": f"Path not found: {path}"},
289 )
291 # Use numeric comparison if both values are numeric
292 if isinstance(value, (int, float)) and isinstance(target, (int, float)):
293 return _compare_values(value, operator, target, path)
295 # For non-numeric values, only eq and ne are supported
296 if operator == "eq":
297 verdict = "yes" if value == target else "no"
298 elif operator == "ne":
299 verdict = "yes" if value != target else "no"
300 else:
301 return EvaluationResult(
302 verdict="error",
303 details={"error": f"Operator {operator} not supported for non-numeric values"},
304 )
306 return EvaluationResult(
307 verdict=verdict,
308 details={"value": value, "path": path, "target": target, "operator": operator},
309 )
312def evaluate_output_contains(
313 output: str,
314 pattern: str,
315 negate: bool = False,
316) -> EvaluationResult:
317 """Check if pattern exists in output.
319 Pattern can be regex or substring. If regex fails to compile,
320 falls back to substring matching.
322 Args:
323 output: The action stdout to search
324 pattern: Regex pattern or substring
325 negate: If True, invert the match result
327 Returns:
328 EvaluationResult with verdict:
329 - Found (negate=False) -> yes
330 - Found (negate=True) -> no
331 - Not found (negate=False) -> no
332 - Not found (negate=True) -> yes
333 """
334 # Try regex first, fall back to substring
335 try:
336 matched = bool(re.search(pattern, output))
337 except re.error:
338 matched = pattern in output
340 if negate:
341 verdict = "no" if matched else "yes"
342 else:
343 verdict = "yes" if matched else "no"
345 return EvaluationResult(
346 verdict=verdict,
347 details={"matched": matched, "pattern": pattern, "negate": negate},
348 )
351def evaluate_convergence(
352 current: float,
353 previous: float | None,
354 target: float,
355 tolerance: float = 0,
356 direction: str = "minimize",
357) -> EvaluationResult:
358 """Compare current value to target and previous.
360 Args:
361 current: Current metric value
362 previous: Previous metric value (None if first iteration)
363 target: Target value to reach
364 tolerance: Acceptable distance from target
365 direction: 'minimize' or 'maximize'
367 Returns:
368 EvaluationResult with verdict:
369 - Value within tolerance of target -> target
370 - Value improved toward target -> progress
371 - Value unchanged or worsened -> stall
372 """
373 # Check if target reached (within tolerance)
374 if abs(current - target) <= tolerance:
375 return EvaluationResult(
376 verdict="target",
377 details={"current": current, "target": target, "delta": 0},
378 )
380 # First iteration has no previous value
381 if previous is None:
382 return EvaluationResult(
383 verdict="progress",
384 details={
385 "current": current,
386 "previous": None,
387 "target": target,
388 "delta": None,
389 },
390 )
392 # Calculate progress
393 delta = current - previous
395 if direction == "minimize":
396 # For minimizing, negative delta is progress
397 made_progress = delta < 0
398 else:
399 # For maximizing, positive delta is progress
400 made_progress = delta > 0
402 verdict = "progress" if made_progress else "stall"
404 return EvaluationResult(
405 verdict=verdict,
406 details={
407 "current": current,
408 "previous": previous,
409 "target": target,
410 "delta": delta,
411 "direction": direction,
412 },
413 )
416def evaluate_diff_stall(
417 scope: list[str] | None = None,
418 max_stall: int = 1,
419) -> EvaluationResult:
420 """Detect stalled iterations by comparing git diff --stat between runs.
422 On first call, snapshots the current diff and returns 'yes'.
423 On subsequent calls, compares current diff to the previous snapshot.
424 If the diff is identical for max_stall consecutive iterations, returns
425 'no' (stalled). If different, resets the stall counter and returns
426 'yes' (progress).
428 State is persisted in /tmp using a key derived from the scope argument,
429 so different loops with different scopes maintain independent stall counters.
431 Args:
432 scope: Optional list of paths to limit the git diff to. Defaults to
433 the entire working tree.
434 max_stall: Number of consecutive no-change iterations before stall
435 verdict. Defaults to 1.
437 Returns:
438 EvaluationResult with verdict:
439 - yes: diff changed since last iteration (progress made)
440 - no: diff unchanged for max_stall iterations (stalled)
441 - error: git command failed or timed out
442 """
443 cmd = ["git", "diff", "--stat"]
444 if scope:
445 cmd += ["--"] + scope
447 try:
448 proc = subprocess.run(cmd, capture_output=True, text=True, timeout=30)
449 except subprocess.TimeoutExpired:
450 return EvaluationResult(verdict="error", details={"error": "git diff timed out"})
451 except FileNotFoundError:
452 return EvaluationResult(verdict="error", details={"error": "git not found in PATH"})
454 if proc.returncode != 0:
455 return EvaluationResult(
456 verdict="error",
457 details={"error": f"git diff failed: {proc.stderr[:200]}"},
458 )
460 current_diff = proc.stdout
462 # Derive a stable cache key from the scope so independent loops don't collide
463 scope_str = "|".join(sorted(scope)) if scope else "_root_"
464 cache_key = hashlib.md5(scope_str.encode()).hexdigest()[:12]
465 loops_tmp = Path.cwd() / ".loops" / "tmp"
466 loops_tmp.mkdir(parents=True, exist_ok=True)
467 state_file = loops_tmp / f"ll-diff-stall-{cache_key}.txt"
468 count_file = loops_tmp / f"ll-diff-stall-{cache_key}.count"
470 # Read previous snapshot and stall count
471 previous_diff: str | None = None
472 stall_count = 0
473 try:
474 previous_diff = state_file.read_text()
475 stall_count = int(count_file.read_text().strip())
476 except (FileNotFoundError, ValueError):
477 pass
479 # First iteration: save snapshot and report progress
480 if previous_diff is None:
481 state_file.write_text(current_diff)
482 count_file.write_text("0")
483 return EvaluationResult(
484 verdict="yes",
485 details={"stall_count": 0, "max_stall": max_stall, "diff_changed": True},
486 )
488 if current_diff == previous_diff:
489 stall_count += 1
490 count_file.write_text(str(stall_count))
491 if stall_count >= max_stall:
492 return EvaluationResult(
493 verdict="no",
494 details={"stall_count": stall_count, "max_stall": max_stall, "diff_changed": False},
495 )
496 # Not yet at max_stall threshold — still report yes so loop continues
497 return EvaluationResult(
498 verdict="yes",
499 details={"stall_count": stall_count, "max_stall": max_stall, "diff_changed": False},
500 )
501 else:
502 # Progress: update snapshot and reset counter
503 state_file.write_text(current_diff)
504 count_file.write_text("0")
505 return EvaluationResult(
506 verdict="yes",
507 details={"stall_count": 0, "max_stall": max_stall, "diff_changed": True},
508 )
511def evaluate_action_stall(
512 track: list[str] | None = None,
513 max_repeat: int = 2,
514 context: InterpolationContext | None = None,
515) -> EvaluationResult:
516 """Detect when the same action string or output repeats for N consecutive iterations.
518 On first call, snapshots the hashed values of the tracked context keys and returns
519 'yes'. On subsequent calls, compares the current hash to the previous snapshot.
520 If the hash is identical for max_repeat consecutive iterations, returns 'no'
521 (stalled). If different, resets the stall counter and returns 'yes' (progress).
523 State is persisted in .loops/tmp using a key derived from the tracked keys,
524 so different states/loops maintain independent stall counters.
526 Args:
527 track: Context keys to track. Defaults to ["action"] when None.
528 max_repeat: Number of consecutive identical-hash iterations before stall verdict.
529 Defaults to 2.
530 context: Runtime interpolation context for resolving tracked keys.
532 Returns:
533 EvaluationResult with verdict:
534 - yes: tracked values changed since last iteration (progress made)
535 - no: tracked values identical for max_repeat iterations (stalled)
536 """
537 effective_track: list[str] = track if track is not None else ["action"]
539 # Resolve each tracked key from context and hash the combined values.
540 # Keys may be bare names (e.g. "action") or namespaced (e.g. "context.action").
541 # Try namespaced forms first: context.<key>, captured.<key>, then bare ${key}.
542 parts: list[str] = []
543 for key in effective_track:
544 value: str = ""
545 if context is not None:
546 # If key already contains a dot it's already namespaced; use as-is.
547 if "." in key:
548 try:
549 value = str(interpolate(f"${{{key}}}", context))
550 except InterpolationError:
551 value = ""
552 else:
553 # Try context.<key> first, then captured.<key>, then give up.
554 resolved = False
555 for namespace in ("context", "captured", "prev", "result"):
556 try:
557 value = str(interpolate(f"${{{namespace}.{key}}}", context))
558 resolved = True
559 break
560 except InterpolationError:
561 continue
562 if not resolved:
563 value = ""
564 parts.append(f"{key}={value}")
566 combined = "|".join(parts)
567 current_hash = hashlib.md5(combined.encode()).hexdigest()
569 # Derive a stable cache key from the tracked keys
570 track_str = "|".join(sorted(effective_track))
571 cache_key = hashlib.md5(track_str.encode()).hexdigest()[:12]
572 loops_tmp = Path.cwd() / ".loops" / "tmp"
573 loops_tmp.mkdir(parents=True, exist_ok=True)
574 state_file = loops_tmp / f"ll-action-stall-{cache_key}.txt"
575 count_file = loops_tmp / f"ll-action-stall-{cache_key}.count"
577 # Read previous hash and stall count
578 previous_hash: str | None = None
579 stall_count = 0
580 try:
581 previous_hash = state_file.read_text().strip()
582 stall_count = int(count_file.read_text().strip())
583 except (FileNotFoundError, ValueError):
584 pass
586 # First iteration: save hash and report progress
587 if previous_hash is None:
588 state_file.write_text(current_hash)
589 count_file.write_text("0")
590 return EvaluationResult(
591 verdict="yes",
592 details={
593 "stall_count": 0,
594 "max_repeat": max_repeat,
595 "hash_changed": True,
596 "tracked_keys": effective_track,
597 },
598 )
600 hash_changed = current_hash != previous_hash
602 if hash_changed:
603 # Progress: update snapshot and reset counter
604 state_file.write_text(current_hash)
605 count_file.write_text("0")
606 return EvaluationResult(
607 verdict="yes",
608 details={
609 "stall_count": 0,
610 "max_repeat": max_repeat,
611 "hash_changed": True,
612 "tracked_keys": effective_track,
613 },
614 )
615 else:
616 # Same hash as last time
617 stall_count += 1
618 count_file.write_text(str(stall_count))
619 if stall_count >= max_repeat:
620 return EvaluationResult(
621 verdict="no",
622 details={
623 "stall_count": stall_count,
624 "max_repeat": max_repeat,
625 "hash_changed": False,
626 "tracked_keys": effective_track,
627 "repeated_hash": current_hash,
628 },
629 )
630 # Not yet at max_repeat threshold — still report yes so loop continues
631 return EvaluationResult(
632 verdict="yes",
633 details={
634 "stall_count": stall_count,
635 "max_repeat": max_repeat,
636 "hash_changed": False,
637 "tracked_keys": effective_track,
638 },
639 )
642def evaluate_mcp_result(output: str, exit_code: int) -> EvaluationResult:
643 """Evaluate an MCP tool call result from the mcp-call subprocess.
645 Maps exit codes and MCP response envelope fields to routing verdicts.
647 Exit code conventions (set by mcp-call):
648 0 → parse isError from JSON envelope
649 1 → tool_error (tool ran but isError: true)
650 124 → timeout (transport-level timeout)
651 127 → not_found (server or tool missing from .mcp.json)
653 Args:
654 output: stdout from mcp-call (MCP response envelope JSON)
655 exit_code: Exit code from mcp-call subprocess
657 Returns:
658 EvaluationResult with verdict:
659 - success → isError: false
660 - tool_error → isError: true
661 - not_found → server/tool not in .mcp.json (exit 127)
662 - timeout → transport-level timeout (exit 124)
663 """
664 if exit_code == 127:
665 return EvaluationResult(
666 verdict="not_found",
667 details={"exit_code": exit_code, "error": "Server or tool not found in .mcp.json"},
668 )
670 if exit_code == 124:
671 return EvaluationResult(
672 verdict="timeout",
673 details={"exit_code": exit_code, "error": "MCP tool call timed out"},
674 )
676 # Parse MCP envelope JSON from stdout
677 try:
678 envelope = json.loads(output.strip()) if output.strip() else {}
679 except json.JSONDecodeError:
680 return EvaluationResult(
681 verdict="tool_error",
682 details={
683 "exit_code": exit_code,
684 "error": f"Invalid JSON from mcp-call: {output[:200]}",
685 },
686 )
688 is_error = envelope.get("isError", exit_code != 0)
690 if is_error:
691 return EvaluationResult(
692 verdict="tool_error",
693 details={"exit_code": exit_code, "envelope": envelope},
694 )
696 return EvaluationResult(
697 verdict="success",
698 details={"exit_code": exit_code, "envelope": envelope},
699 )
702def evaluate_harbor_scorer(output: str, exit_code: int) -> EvaluationResult:
703 """Evaluate a Harbor-format benchmark scorer result.
705 The scorer is a shell command that prints a float score (0.0–1.0) to stdout
706 and exits 0 on success or non-zero on failure.
708 Args:
709 output: stdout from the scorer subprocess (expected: a bare float)
710 exit_code: Exit code from the scorer subprocess
712 Returns:
713 EvaluationResult with verdict:
714 - yes → exit 0 and stdout parses as a float
715 - no → exit non-zero (scorer determined failure)
716 - error → exit 0 but stdout is not parseable as a float
717 """
718 if exit_code != 0:
719 return EvaluationResult(
720 verdict="no",
721 details={"exit_code": exit_code},
722 )
724 try:
725 score = float(output.strip())
726 except (ValueError, AttributeError):
727 return EvaluationResult(
728 verdict="error",
729 details={
730 "exit_code": exit_code,
731 "error": f"Scorer stdout is not a float: {output[:200]}",
732 },
733 )
735 return EvaluationResult(
736 verdict="yes",
737 details={"score": score, "exit_code": 0},
738 )
741def evaluate_llm_structured(
742 output: str,
743 prompt: str | None = None,
744 schema: dict[str, Any] | None = None,
745 min_confidence: float = 0.5,
746 uncertain_suffix: bool = False,
747 model: str = DEFAULT_LLM_MODEL,
748 max_tokens: int = 256,
749 timeout: int = 1800,
750) -> EvaluationResult:
751 """Evaluate action output using LLM with structured output via Claude CLI.
753 This is the ONLY place in the FSM system that uses LLM structured output.
754 Requires the ``claude`` CLI to be installed and authenticated.
756 Args:
757 output: Action stdout to evaluate
758 prompt: Custom evaluation prompt (defaults to basic success check)
759 schema: Custom JSON schema for structured response
760 min_confidence: Minimum confidence threshold (0-1)
761 uncertain_suffix: If True, append _uncertain to low-confidence verdicts
762 model: Model identifier (CLI aliases like "sonnet" or full names)
763 max_tokens: Maximum tokens for response (passed to --max-turns is not
764 applicable; kept for signature compat)
765 timeout: Timeout in seconds
767 Returns:
768 EvaluationResult with verdict from LLM and confidence/reason in details
769 """
770 effective_schema = schema or DEFAULT_LLM_SCHEMA
771 effective_prompt = prompt or DEFAULT_LLM_PROMPT
773 # Truncate output to avoid context limits (keep last 4000 chars)
774 truncated = output[-4000:] if len(output) > 4000 else output
776 user_prompt = f"{effective_prompt}\n\n<action_output>\n{truncated}\n</action_output>"
778 invocation = resolve_host().build_blocking_json(prompt=user_prompt, model=model)
779 # Builder drops json_schema (Protocol surface only) and omits the
780 # claude-CLI-specific --no-session-persistence flag; augment at call site.
781 args = list(invocation.args) + [
782 "--json-schema",
783 json.dumps(effective_schema),
784 "--no-session-persistence",
785 ]
787 t0 = time.monotonic()
788 try:
789 proc = subprocess.run(
790 [invocation.binary, *args], capture_output=True, text=True, timeout=timeout
791 )
792 except subprocess.TimeoutExpired:
793 return EvaluationResult(
794 verdict="error",
795 details={"error": "LLM evaluation timeout", "timeout": True},
796 )
797 except FileNotFoundError:
798 return EvaluationResult(
799 verdict="error",
800 details={
801 "error": f"{invocation.binary} CLI not found. Install the active host CLI (see LL_HOST_CLI).",
802 "missing_dependency": True,
803 },
804 )
805 llm_latency_ms = int((time.monotonic() - t0) * 1000)
807 if proc.returncode != 0:
808 return EvaluationResult(
809 verdict="error",
810 details={
811 "error": f"{invocation.binary} CLI error: {proc.stderr.strip()}",
812 "api_error": True,
813 },
814 )
816 # Guard: empty stdout with exit 0 (API error not reflected in exit code)
817 if not proc.stdout.strip():
818 stderr_info = proc.stderr.strip()[:200] if proc.stderr else ""
819 error_msg = f"{invocation.binary} CLI returned empty output"
820 if stderr_info:
821 error_msg += f" (stderr: {stderr_info})"
822 return EvaluationResult(
823 verdict="error",
824 details={"error": error_msg, "empty_output": True},
825 )
827 # Parse the CLI JSON envelope and extract structured result.
828 # With --json-schema the envelope is:
829 # success: {"type":"result","subtype":"success","structured_output":{...},...}
830 # failure: {"type":"result","subtype":"error_max_structured_output_retries",...}
831 # If stdout is JSONL (multiple JSON objects), use the last non-empty line.
832 try:
833 stdout = proc.stdout.strip()
834 try:
835 envelope = json.loads(stdout)
836 except json.JSONDecodeError:
837 # Try JSONL: take the last non-empty line
838 lines = [line for line in stdout.split("\n") if line.strip()]
839 if not lines:
840 raise
841 envelope = json.loads(lines[-1])
843 # Check structured-output retry exhaustion (--json-schema failure mode)
844 if envelope.get("subtype") == "error_max_structured_output_retries":
845 return EvaluationResult(
846 verdict="error",
847 details={
848 "error": "Claude CLI could not produce valid structured output after retries",
849 "api_error": True,
850 },
851 )
853 # Check legacy is_error flag (some CLI versions exit 0 but report error in envelope)
854 if envelope.get("is_error", False):
855 err_text = str(envelope.get("result", "") or "")[:200]
856 return EvaluationResult(
857 verdict="error",
858 details={"error": f"Claude CLI reported error: {err_text}", "api_error": True},
859 )
861 # --json-schema mode returns validated dict in "structured_output"
862 if isinstance(envelope.get("structured_output"), dict):
863 llm_result: dict[str, Any] = envelope["structured_output"]
864 else:
865 raw_result = envelope.get("result", "")
866 if isinstance(raw_result, dict):
867 llm_result = raw_result
868 elif raw_result:
869 llm_result = json.loads(raw_result)
870 elif "verdict" in envelope:
871 llm_result = envelope
872 else:
873 raw_preview = proc.stdout[:300]
874 return EvaluationResult(
875 verdict="error",
876 details={
877 "error": "Empty result field in Claude CLI response",
878 "raw_preview": raw_preview,
879 },
880 )
881 except (json.JSONDecodeError, TypeError, ValueError) as e:
882 raw_preview = proc.stdout[:300] if proc.stdout else "(empty)"
883 return EvaluationResult(
884 verdict="error",
885 details={"error": f"Failed to parse LLM response: {e}", "raw_preview": raw_preview},
886 )
888 # Build result with confidence handling
889 verdict = str(llm_result.get("verdict", "error"))
890 confidence = float(llm_result.get("confidence", 1.0))
891 confident = confidence >= min_confidence
893 # Optionally modify verdict for low confidence
894 if uncertain_suffix and not confident:
895 verdict = f"{verdict}_uncertain"
897 return EvaluationResult(
898 verdict=verdict,
899 details={
900 "confidence": confidence,
901 "confident": confident,
902 "reason": llm_result.get("reason", ""),
903 "raw": llm_result,
904 "llm_model": model,
905 "llm_latency_ms": llm_latency_ms,
906 "llm_prompt": user_prompt[:500],
907 "llm_raw_output": proc.stdout[:500] if proc.stdout else "",
908 },
909 )
912def evaluate_blind_comparator(
913 output_harness: str,
914 output_baseline: str,
915 prompt: str | None = None,
916 model: str = DEFAULT_LLM_MODEL,
917 timeout: int = 1800,
918) -> dict[str, Any]:
919 """Blindly evaluate two outputs, returning pass/fail for each arm.
921 Outputs are randomly labeled "Output A" / "Output B" so the LLM judge
922 cannot distinguish the harness arm from the baseline arm. The mapping is
923 de-anonymized after judgment so callers receive harness/baseline verdicts.
925 Args:
926 output_harness: stdout from the harness (gated) arm
927 output_baseline: stdout from the baseline (ungated) arm
928 prompt: Custom evaluation prompt (appended to default framing)
929 model: Model identifier for the judge
930 timeout: Timeout in seconds
932 Returns:
933 Dict with keys: harness_pass (bool), baseline_pass (bool),
934 confidence (float), reason (str), raw (dict with A/B verdicts)
935 """
936 effective_prompt = prompt or DEFAULT_BLIND_COMPARATOR_PROMPT
938 # Truncate outputs to avoid context limits
939 truncated_harness = output_harness[-4000:] if len(output_harness) > 4000 else output_harness
940 truncated_baseline = output_baseline[-4000:] if len(output_baseline) > 4000 else output_baseline
942 # Randomize order: coin flip determines whether harness→A / baseline→B
943 harness_is_a = random.choice([True, False])
944 if harness_is_a:
945 output_a, output_b = truncated_harness, truncated_baseline
946 else:
947 output_a, output_b = truncated_baseline, truncated_harness
949 user_prompt = (
950 f"{effective_prompt}\n\n"
951 f"<output_a>\n{output_a}\n</output_a>\n\n"
952 f"<output_b>\n{output_b}\n</output_b>"
953 )
955 invocation = resolve_host().build_blocking_json(prompt=user_prompt, model=model)
956 args = list(invocation.args) + [
957 "--json-schema",
958 json.dumps(BLIND_COMPARATOR_SCHEMA),
959 "--no-session-persistence",
960 ]
962 try:
963 proc = subprocess.run(
964 [invocation.binary, *args], capture_output=True, text=True, timeout=timeout
965 )
966 except subprocess.TimeoutExpired:
967 # On timeout, both fail — conservative default
968 return {
969 "harness_pass": False,
970 "baseline_pass": False,
971 "confidence": 0.0,
972 "reason": "LLM evaluation timed out",
973 "raw": {"verdict_a": "timeout", "verdict_b": "timeout"},
974 "error": "timeout",
975 }
976 except FileNotFoundError:
977 return {
978 "harness_pass": False,
979 "baseline_pass": False,
980 "confidence": 0.0,
981 "reason": f"{invocation.binary} CLI not found",
982 "raw": {"verdict_a": "error", "verdict_b": "error"},
983 "error": "missing_cli",
984 }
986 if proc.returncode != 0:
987 return {
988 "harness_pass": False,
989 "baseline_pass": False,
990 "confidence": 0.0,
991 "reason": f"Judge CLI error: {proc.stderr.strip()[:200]}",
992 "raw": {"verdict_a": "error", "verdict_b": "error"},
993 "error": "api_error",
994 }
996 if not proc.stdout.strip():
997 return {
998 "harness_pass": False,
999 "baseline_pass": False,
1000 "confidence": 0.0,
1001 "reason": "Judge returned empty output",
1002 "raw": {"verdict_a": "error", "verdict_b": "error"},
1003 "error": "empty_output",
1004 }
1006 try:
1007 stdout = proc.stdout.strip()
1008 try:
1009 envelope = json.loads(stdout)
1010 except json.JSONDecodeError:
1011 lines = [line for line in stdout.split("\n") if line.strip()]
1012 if not lines:
1013 raise
1014 envelope = json.loads(lines[-1])
1016 if envelope.get("subtype") == "error_max_structured_output_retries":
1017 return {
1018 "harness_pass": False,
1019 "baseline_pass": False,
1020 "confidence": 0.0,
1021 "reason": "Judge could not produce valid structured output after retries",
1022 "raw": {"verdict_a": "error", "verdict_b": "error"},
1023 "error": "retry_exhausted",
1024 }
1026 if envelope.get("is_error", False):
1027 err_text = str(envelope.get("result", "") or "")[:200]
1028 return {
1029 "harness_pass": False,
1030 "baseline_pass": False,
1031 "confidence": 0.0,
1032 "reason": f"Judge reported error: {err_text}",
1033 "raw": {"verdict_a": "error", "verdict_b": "error"},
1034 "error": "api_error",
1035 }
1037 if isinstance(envelope.get("structured_output"), dict):
1038 result: dict[str, Any] = envelope["structured_output"]
1039 else:
1040 raw_result = envelope.get("result", "")
1041 if isinstance(raw_result, dict):
1042 result = raw_result
1043 elif raw_result:
1044 result = json.loads(raw_result)
1045 else:
1046 return {
1047 "harness_pass": False,
1048 "baseline_pass": False,
1049 "confidence": 0.0,
1050 "reason": "Empty result field in judge response",
1051 "raw": {"verdict_a": "error", "verdict_b": "error"},
1052 "error": "empty_result",
1053 }
1054 except (json.JSONDecodeError, TypeError, ValueError):
1055 return {
1056 "harness_pass": False,
1057 "baseline_pass": False,
1058 "confidence": 0.0,
1059 "reason": "Failed to parse judge response",
1060 "raw": {"verdict_a": "error", "verdict_b": "error"},
1061 "error": "parse_error",
1062 }
1064 # De-anonymize
1065 verdict_a = str(result.get("verdict_a", "no"))
1066 verdict_b = str(result.get("verdict_b", "no"))
1067 confidence = float(result.get("confidence", 0.0))
1068 reason = str(result.get("reason", ""))
1070 if harness_is_a:
1071 harness_pass = verdict_a == "yes"
1072 baseline_pass = verdict_b == "yes"
1073 else:
1074 harness_pass = verdict_b == "yes"
1075 baseline_pass = verdict_a == "yes"
1077 return {
1078 "harness_pass": harness_pass,
1079 "baseline_pass": baseline_pass,
1080 "confidence": confidence,
1081 "reason": reason,
1082 "raw": {"verdict_a": verdict_a, "verdict_b": verdict_b, "harness_is_a": harness_is_a},
1083 }
1086def evaluate_contract(
1087 config: EvaluateConfig,
1088 context: InterpolationContext,
1089 model: str = DEFAULT_LLM_MODEL,
1090 timeout: int = 1800,
1091) -> EvaluationResult:
1092 """Evaluate producer/consumer contract alignment using an LLM judge.
1094 Reads each producer/consumer file pair, applies optional regex extraction,
1095 then asks an LLM judge whether the producer satisfies the consumer contract.
1096 Returns yes only when all pairs align; any failure routes no/error.
1098 Args:
1099 config: EvaluateConfig with type="contract" and pairs list
1100 context: Interpolation context (unused by this evaluator directly)
1101 model: LLM model identifier
1102 timeout: Subprocess timeout in seconds
1104 Returns:
1105 EvaluationResult with verdict yes/no/error and pair_results in details
1106 """
1107 pairs = config.pairs
1108 if not pairs:
1109 return EvaluationResult(
1110 verdict="error",
1111 details={"error": "contract evaluator requires at least one pair in evaluate.pairs"},
1112 )
1114 contract_schema = {
1115 "type": "object",
1116 "properties": {
1117 "verdict": {"type": "string", "enum": ["yes", "no"]},
1118 "confidence": {"type": "number"},
1119 "reason": {"type": "string"},
1120 },
1121 "required": ["verdict", "confidence", "reason"],
1122 }
1124 pair_results: list[dict[str, Any]] = []
1126 for pair in pairs:
1127 producer_path = pair.get("producer", "")
1128 consumer_path = pair.get("consumer", "")
1129 producer_pattern = pair.get("producer_pattern")
1130 consumer_pattern = pair.get("consumer_pattern")
1131 contract_rule = pair.get("contract", "the producer and consumer must be compatible")
1133 # Read producer file
1134 try:
1135 producer_content = Path(producer_path).read_text()
1136 except OSError as e:
1137 pair_results.append(
1138 {
1139 "producer": producer_path,
1140 "consumer": consumer_path,
1141 "verdict": "error",
1142 "error": f"cannot read producer file: {e}",
1143 }
1144 )
1145 continue
1147 # Read consumer file
1148 try:
1149 consumer_content = Path(consumer_path).read_text()
1150 except OSError as e:
1151 pair_results.append(
1152 {
1153 "producer": producer_path,
1154 "consumer": consumer_path,
1155 "verdict": "error",
1156 "error": f"cannot read consumer file: {e}",
1157 }
1158 )
1159 continue
1161 # Apply optional regex extraction
1162 if producer_pattern:
1163 matches = re.findall(producer_pattern, producer_content, re.DOTALL)
1164 if not matches:
1165 pair_results.append(
1166 {
1167 "producer": producer_path,
1168 "consumer": consumer_path,
1169 "verdict": "error",
1170 "error": f"producer_pattern matched nothing in {producer_path}",
1171 }
1172 )
1173 continue
1174 producer_slice = "\n".join(matches)
1175 else:
1176 producer_slice = (
1177 producer_content[-4000:] if len(producer_content) > 4000 else producer_content
1178 )
1180 if consumer_pattern:
1181 matches = re.findall(consumer_pattern, consumer_content, re.DOTALL)
1182 if not matches:
1183 pair_results.append(
1184 {
1185 "producer": producer_path,
1186 "consumer": consumer_path,
1187 "verdict": "error",
1188 "error": f"consumer_pattern matched nothing in {consumer_path}",
1189 }
1190 )
1191 continue
1192 consumer_slice = "\n".join(matches)
1193 else:
1194 consumer_slice = (
1195 consumer_content[-4000:] if len(consumer_content) > 4000 else consumer_content
1196 )
1198 judge_prompt = (
1199 f"You are evaluating whether a producer output satisfies a consumer contract.\n\n"
1200 f"Contract rule: {contract_rule}\n\n"
1201 f'<producer path="{producer_path}">\n{producer_slice}\n</producer>\n\n'
1202 f'<consumer path="{consumer_path}">\n{consumer_slice}\n</consumer>\n\n'
1203 "Does the producer satisfy the consumer contract? "
1204 "Consider field names, types, casing, and structure. "
1205 "Answer yes if aligned, no if mismatched."
1206 )
1208 invocation = resolve_host().build_blocking_json(prompt=judge_prompt, model=model)
1209 args = list(invocation.args) + [
1210 "--json-schema",
1211 json.dumps(contract_schema),
1212 "--no-session-persistence",
1213 ]
1215 t0 = time.monotonic()
1216 try:
1217 proc = subprocess.run(
1218 [invocation.binary, *args], capture_output=True, text=True, timeout=timeout
1219 )
1220 except subprocess.TimeoutExpired:
1221 pair_results.append(
1222 {
1223 "producer": producer_path,
1224 "consumer": consumer_path,
1225 "verdict": "error",
1226 "error": "LLM judge timed out",
1227 "llm_latency_ms": int((time.monotonic() - t0) * 1000),
1228 }
1229 )
1230 continue
1231 except FileNotFoundError:
1232 return EvaluationResult(
1233 verdict="error",
1234 details={
1235 "error": f"{invocation.binary} CLI not found. Install the active host CLI (see LL_HOST_CLI).",
1236 "missing_dependency": True,
1237 },
1238 )
1239 llm_latency_ms = int((time.monotonic() - t0) * 1000)
1241 if proc.returncode != 0:
1242 pair_results.append(
1243 {
1244 "producer": producer_path,
1245 "consumer": consumer_path,
1246 "verdict": "error",
1247 "error": f"CLI error: {proc.stderr.strip()}",
1248 "llm_latency_ms": llm_latency_ms,
1249 }
1250 )
1251 continue
1253 if not proc.stdout.strip():
1254 pair_results.append(
1255 {
1256 "producer": producer_path,
1257 "consumer": consumer_path,
1258 "verdict": "error",
1259 "error": "CLI returned empty output",
1260 "llm_latency_ms": llm_latency_ms,
1261 }
1262 )
1263 continue
1265 try:
1266 stdout = proc.stdout.strip()
1267 try:
1268 envelope = json.loads(stdout)
1269 except json.JSONDecodeError:
1270 lines = [line for line in stdout.split("\n") if line.strip()]
1271 if not lines:
1272 raise
1273 envelope = json.loads(lines[-1])
1275 if envelope.get("subtype") == "error_max_structured_output_retries":
1276 pair_results.append(
1277 {
1278 "producer": producer_path,
1279 "consumer": consumer_path,
1280 "verdict": "error",
1281 "error": "Claude CLI could not produce valid structured output after retries",
1282 "llm_latency_ms": llm_latency_ms,
1283 }
1284 )
1285 continue
1287 if envelope.get("is_error", False):
1288 err_text = str(envelope.get("result", "") or "")[:200]
1289 pair_results.append(
1290 {
1291 "producer": producer_path,
1292 "consumer": consumer_path,
1293 "verdict": "error",
1294 "error": f"Claude CLI reported error: {err_text}",
1295 "llm_latency_ms": llm_latency_ms,
1296 }
1297 )
1298 continue
1300 if isinstance(envelope.get("structured_output"), dict):
1301 llm_result: dict[str, Any] = envelope["structured_output"]
1302 else:
1303 raw_result = envelope.get("result", "")
1304 if isinstance(raw_result, dict):
1305 llm_result = raw_result
1306 elif raw_result:
1307 llm_result = json.loads(raw_result)
1308 elif "verdict" in envelope:
1309 llm_result = envelope
1310 else:
1311 pair_results.append(
1312 {
1313 "producer": producer_path,
1314 "consumer": consumer_path,
1315 "verdict": "error",
1316 "error": "empty result field in CLI response",
1317 "llm_latency_ms": llm_latency_ms,
1318 }
1319 )
1320 continue
1322 except (json.JSONDecodeError, TypeError, ValueError) as e:
1323 pair_results.append(
1324 {
1325 "producer": producer_path,
1326 "consumer": consumer_path,
1327 "verdict": "error",
1328 "error": f"failed to parse LLM response: {e}",
1329 "llm_latency_ms": llm_latency_ms,
1330 }
1331 )
1332 continue
1334 pair_results.append(
1335 {
1336 "producer": producer_path,
1337 "consumer": consumer_path,
1338 "verdict": str(llm_result.get("verdict", "error")),
1339 "confidence": float(llm_result.get("confidence", 1.0)),
1340 "reason": llm_result.get("reason", ""),
1341 "llm_latency_ms": llm_latency_ms,
1342 }
1343 )
1345 # Aggregate: yes only if all pairs aligned; error takes precedence over no
1346 if any(p["verdict"] == "error" for p in pair_results):
1347 overall = "error"
1348 elif all(p["verdict"] == "yes" for p in pair_results):
1349 overall = "yes"
1350 else:
1351 overall = "no"
1353 return EvaluationResult(
1354 verdict=overall,
1355 details={"pair_results": pair_results},
1356 )
1359def evaluate_comparator(
1360 config: EvaluateConfig,
1361 output: str,
1362 context: InterpolationContext,
1363) -> EvaluationResult:
1364 """Evaluate using blind A/B comparison against a stored baseline."""
1365 from pathlib import Path
1367 if config.baseline_path is None:
1368 return EvaluationResult(
1369 verdict="no_baseline",
1370 details={"reason": "No baseline_path configured"},
1371 )
1373 baseline_file = Path(config.baseline_path) / "output.txt"
1374 if not baseline_file.exists():
1375 if config.auto_promote:
1376 baseline_file.parent.mkdir(parents=True, exist_ok=True)
1377 baseline_file.write_text(output)
1378 return EvaluationResult(
1379 verdict="yes",
1380 details={
1381 "reason": "No baseline found; current output promoted as new baseline.",
1382 "bootstrapped": True,
1383 },
1384 )
1385 return EvaluationResult(
1386 verdict="no_baseline",
1387 details={"reason": f"Baseline file not found: {baseline_file}"},
1388 )
1390 baseline_text = baseline_file.read_text()
1391 min_pairs = max(1, config.min_pairs if config.min_pairs is not None else 1)
1392 harness_wins = 0
1393 baseline_wins = 0
1394 last_reason = ""
1395 last_raw: dict[str, Any] = {}
1397 for _ in range(min_pairs):
1398 result = evaluate_blind_comparator(output, baseline_text, prompt=config.prompt)
1399 if result.get("harness_pass"):
1400 harness_wins += 1
1401 if result.get("baseline_pass"):
1402 baseline_wins += 1
1403 last_reason = result.get("reason", "")
1404 last_raw = result.get("raw", {})
1406 if harness_wins > baseline_wins:
1407 verdict = "yes"
1408 elif baseline_wins > harness_wins:
1409 verdict = "no"
1410 else:
1411 verdict = "tie"
1413 if config.auto_promote and verdict == "yes":
1414 baseline_file.write_text(output)
1416 return EvaluationResult(
1417 verdict=verdict,
1418 details={
1419 "harness_wins": harness_wins,
1420 "baseline_wins": baseline_wins,
1421 "min_pairs": min_pairs,
1422 "reason": last_reason,
1423 "raw": last_raw,
1424 },
1425 )
1428def evaluate(
1429 config: EvaluateConfig,
1430 output: str,
1431 exit_code: int,
1432 context: InterpolationContext,
1433) -> EvaluationResult:
1434 """Dispatch to appropriate evaluator based on config type.
1436 Args:
1437 config: Evaluator configuration with type and parameters
1438 output: Action stdout
1439 exit_code: Action exit code
1440 context: Runtime context for variable interpolation
1442 Returns:
1443 EvaluationResult from the appropriate evaluator
1445 Raises:
1446 ValueError: If evaluator type is unknown
1447 """
1448 eval_type = config.type
1450 # BUG-1640: Action-level timeouts (exit_code=124) short-circuit to "error"
1451 # so loop authors' on_error: branches fire instead of being routed via
1452 # on_no: based on truncated output. mcp_result is exempted because it has
1453 # its own established "timeout" verdict (see evaluate_mcp_result).
1454 if exit_code == 124 and eval_type != "mcp_result":
1455 return EvaluationResult(
1456 verdict="error",
1457 details={"exit_code": exit_code, "error": "action timed out"},
1458 )
1460 # BUG-1815: Non-timeout non-zero exit codes short-circuit to "error" for
1461 # evaluator types that don't intrinsically check exit codes. Exit-code-aware
1462 # evaluators (exit_code, mcp_result, harbor_scorer, diff_stall, llm_structured)
1463 # are exempt because they handle exit codes via their own logic.
1464 _EXIT_CODE_AWARE_EVALUATORS: frozenset[str] = frozenset(
1465 {
1466 "exit_code",
1467 "mcp_result",
1468 "harbor_scorer",
1469 "diff_stall",
1470 "action_stall",
1471 "llm_structured",
1472 "contract",
1473 }
1474 )
1475 if exit_code != 0 and eval_type not in _EXIT_CODE_AWARE_EVALUATORS:
1476 return EvaluationResult(
1477 verdict="error",
1478 details={
1479 "exit_code": exit_code,
1480 "error": f"action exited with code {exit_code}",
1481 },
1482 )
1484 if eval_type == "exit_code":
1485 return evaluate_exit_code(exit_code)
1487 elif eval_type == "output_numeric":
1488 if config.target is None:
1489 raise ValueError("output_numeric evaluator requires 'target' to be set")
1490 elif isinstance(config.target, str):
1491 try:
1492 resolved = interpolate(config.target, context) if context else config.target
1493 numeric_target = float(resolved)
1494 except (InterpolationError, ValueError) as e:
1495 raise ValueError(
1496 f"output_numeric target must be numeric, got: {config.target!r}"
1497 ) from e
1498 else:
1499 numeric_target = float(config.target)
1500 return evaluate_output_numeric(
1501 output=output,
1502 operator=config.operator or "eq",
1503 target=numeric_target,
1504 )
1506 elif eval_type == "output_json":
1507 return evaluate_output_json(
1508 output=output,
1509 path=config.path or "",
1510 operator=config.operator or "eq",
1511 target=config.target,
1512 )
1514 elif eval_type == "output_contains":
1515 return evaluate_output_contains(
1516 output=output,
1517 pattern=config.pattern or "",
1518 negate=config.negate,
1519 )
1521 elif eval_type == "convergence":
1522 # Resolve previous value from interpolation if configured
1523 previous: float | None = None
1524 if config.previous:
1525 try:
1526 previous = float(interpolate(config.previous, context))
1527 except (InterpolationError, ValueError):
1528 # Previous unavailable on first iteration, continue with None
1529 pass
1531 # Parse current value from output
1532 try:
1533 current = float(output.strip())
1534 except ValueError:
1535 return EvaluationResult(
1536 verdict="error",
1537 details={"error": f"Cannot parse output as number: {output[:100]}"},
1538 )
1540 # Resolve target (may be interpolated string like "${context.target}")
1541 convergence_target: float
1542 if isinstance(config.target, str):
1543 try:
1544 convergence_target = float(interpolate(config.target, context))
1545 except (InterpolationError, ValueError) as e:
1546 return EvaluationResult(
1547 verdict="error",
1548 details={"error": f"Cannot resolve target: {e}"},
1549 )
1550 else:
1551 if config.target is None:
1552 raise ValueError("convergence evaluator requires 'target' to be set")
1553 convergence_target = float(config.target)
1555 # Resolve tolerance (may be interpolated string)
1556 tolerance: float = 0.0
1557 if config.tolerance is not None:
1558 if isinstance(config.tolerance, str):
1559 try:
1560 tolerance = float(interpolate(config.tolerance, context))
1561 except (InterpolationError, ValueError):
1562 tolerance = 0.0
1563 else:
1564 tolerance = float(config.tolerance)
1566 return evaluate_convergence(
1567 current=current,
1568 previous=previous,
1569 target=convergence_target,
1570 tolerance=tolerance,
1571 direction=config.direction,
1572 )
1574 elif eval_type == "diff_stall":
1575 return evaluate_diff_stall(
1576 scope=config.scope,
1577 max_stall=config.max_stall,
1578 )
1580 elif eval_type == "action_stall":
1581 return evaluate_action_stall(
1582 track=config.track,
1583 max_repeat=config.max_repeat,
1584 context=context,
1585 )
1587 elif eval_type == "llm_structured":
1588 prompt = config.prompt
1589 if prompt and context:
1590 try:
1591 prompt = interpolate(prompt, context)
1592 except InterpolationError:
1593 pass # Use raw prompt on resolution failure
1594 return evaluate_llm_structured(
1595 output=output,
1596 prompt=prompt,
1597 schema=config.schema,
1598 min_confidence=config.min_confidence,
1599 uncertain_suffix=config.uncertain_suffix,
1600 )
1602 elif eval_type == "mcp_result":
1603 return evaluate_mcp_result(output=output, exit_code=exit_code)
1605 elif eval_type == "harbor_scorer":
1606 return evaluate_harbor_scorer(output=output, exit_code=exit_code)
1608 elif eval_type == "comparator":
1609 return evaluate_comparator(config=config, output=output, context=context)
1611 elif eval_type == "contract":
1612 return evaluate_contract(config=config, context=context)
1614 else:
1615 raise ValueError(f"Unknown evaluator type: {eval_type}")