Coverage for little_loops / fsm / evaluators.py: 11%
270 statements
« prev ^ index » next coverage.py v7.12.0, created at 2026-05-11 00:29 -0500
1"""FSM Evaluators for loop execution.
3This module provides evaluators that interpret action output and produce
4verdicts for state transitions.
6Supported evaluator types:
8Tier 1 (Deterministic - no API calls):
9 exit_code: Map Unix exit codes to verdicts (0=success, 1=failure, 2+=error)
10 output_numeric: Compare numeric output to target value
11 output_json: Extract and compare JSON path values
12 output_contains: Pattern matching on stdout
13 convergence: Track progress toward a target value
14 diff_stall: Detect stalled iterations via git diff comparison
15 harbor_scorer: Interpret Harbor-format benchmark scorer exit code and float stdout
17Tier 2 (LLM-based):
18 llm_structured: Use LLM with structured output for natural language evaluation
20Tier 3 (External process):
21 mcp_result: Parse MCP tool call response envelope
22"""
24from __future__ import annotations
26import hashlib
27import json
28import re
29import subprocess
30import time
31from collections.abc import Callable
32from dataclasses import dataclass
33from pathlib import Path
34from typing import Any
36from little_loops.fsm.interpolation import (
37 InterpolationContext,
38 InterpolationError,
39 interpolate,
40)
41from little_loops.fsm.schema import DEFAULT_LLM_MODEL, EvaluateConfig
44@dataclass
45class EvaluationResult:
46 """Result from an evaluator.
48 Attributes:
49 verdict: The routing key for state transitions
50 details: Evaluator-specific metadata for debugging/logging
51 """
53 verdict: str
54 details: dict[str, Any]
57# Default schema for LLM structured evaluation
58DEFAULT_LLM_SCHEMA: dict[str, Any] = {
59 "type": "object",
60 "properties": {
61 "verdict": {
62 "type": "string",
63 "enum": ["yes", "no", "blocked", "partial"],
64 "description": (
65 "- yes: The condition/check evaluated to true\n"
66 "- no: The condition/check evaluated to false\n"
67 "- blocked: Cannot proceed without external help\n"
68 "- partial: Made progress but not complete"
69 ),
70 },
71 "confidence": {
72 "type": "number",
73 "minimum": 0,
74 "maximum": 1,
75 "description": "Confidence in this verdict (0-1)",
76 },
77 "reason": {
78 "type": "string",
79 "description": "Brief explanation",
80 },
81 },
82 "required": ["verdict", "confidence", "reason"],
83}
85DEFAULT_LLM_PROMPT = "Evaluate whether this action succeeded based on its output."
87_NUMERIC_OPERATORS: dict[str, Callable[[float, float], bool]] = {
88 "eq": lambda v, t: v == t,
89 "ne": lambda v, t: v != t,
90 "lt": lambda v, t: v < t,
91 "le": lambda v, t: v <= t,
92 "gt": lambda v, t: v > t,
93 "ge": lambda v, t: v >= t,
94}
97def evaluate_exit_code(exit_code: int) -> EvaluationResult:
98 """Map Unix exit code to verdict.
100 Args:
101 exit_code: The process exit code
103 Returns:
104 EvaluationResult with verdict:
105 - 0 -> yes
106 - 1 -> no
107 - 2+ -> error
108 """
109 if exit_code == 0:
110 verdict = "yes"
111 elif exit_code == 1:
112 verdict = "no"
113 else:
114 verdict = "error"
116 return EvaluationResult(verdict=verdict, details={"exit_code": exit_code})
119def evaluate_output_numeric(
120 output: str,
121 operator: str,
122 target: float,
123) -> EvaluationResult:
124 """Parse stdout as number and compare to target.
126 Args:
127 output: The action stdout to parse as a number
128 operator: Comparison operator (eq, ne, lt, le, gt, ge)
129 target: Target value to compare against
131 Returns:
132 EvaluationResult with verdict:
133 - Condition met -> yes
134 - Condition not met -> no
135 - Parse error -> error
136 """
137 try:
138 value = float(output.strip())
139 except ValueError:
140 return EvaluationResult(
141 verdict="error",
142 details={"error": f"Cannot parse as number: {output[:100]}"},
143 )
145 if operator not in _NUMERIC_OPERATORS:
146 return EvaluationResult(
147 verdict="error",
148 details={"error": f"Unknown operator: {operator}"},
149 )
151 condition_met = _NUMERIC_OPERATORS[operator](value, target)
152 return EvaluationResult(
153 verdict="yes" if condition_met else "no",
154 details={"value": value, "target": target, "operator": operator},
155 )
158def _extract_json_path(data: Any, path: str) -> Any:
159 """Extract value from dict using jq-style path like '.summary.failed'.
161 Args:
162 data: The parsed JSON data (dict or list)
163 path: Dot-separated path, optionally starting with '.'
165 Returns:
166 The value at the specified path
168 Raises:
169 KeyError: If path not found in data
170 """
171 if path.startswith("."):
172 path = path[1:]
173 parts = path.split(".")
174 current = data
175 for part in parts:
176 if isinstance(current, dict) and part in current:
177 current = current[part]
178 elif isinstance(current, list) and part.isdigit():
179 idx = int(part)
180 if 0 <= idx < len(current):
181 current = current[idx]
182 else:
183 raise KeyError(path)
184 else:
185 raise KeyError(path)
186 return current
189def _compare_values(
190 value: int | float, operator: str, target: int | float, path: str
191) -> EvaluationResult:
192 """Compare numeric values using operator.
194 Args:
195 value: The extracted value to compare
196 operator: Comparison operator
197 target: Target value
198 path: JSON path for details
200 Returns:
201 EvaluationResult with comparison result
202 """
203 if operator not in _NUMERIC_OPERATORS:
204 return EvaluationResult(
205 verdict="error",
206 details={"error": f"Unknown operator: {operator}"},
207 )
209 condition_met = _NUMERIC_OPERATORS[operator](value, target)
210 return EvaluationResult(
211 verdict="yes" if condition_met else "no",
212 details={"value": value, "path": path, "target": target, "operator": operator},
213 )
216def evaluate_output_json(
217 output: str,
218 path: str,
219 operator: str,
220 target: Any,
221) -> EvaluationResult:
222 """Parse JSON and extract value at path, then compare.
224 Args:
225 output: The action stdout containing JSON
226 path: jq-style dot notation path (e.g., '.summary.failed')
227 operator: Comparison operator (eq, ne, lt, le, gt, ge)
228 target: Target value for comparison
230 Returns:
231 EvaluationResult with verdict:
232 - Condition met -> yes
233 - Condition not met -> no
234 - Parse/path error -> error
235 """
236 try:
237 data = json.loads(output)
238 except json.JSONDecodeError as e:
239 return EvaluationResult(
240 verdict="error",
241 details={"error": f"Invalid JSON: {e}"},
242 )
244 try:
245 value = _extract_json_path(data, path)
246 except KeyError:
247 return EvaluationResult(
248 verdict="error",
249 details={"error": f"Path not found: {path}"},
250 )
252 # Use numeric comparison if both values are numeric
253 if isinstance(value, (int, float)) and isinstance(target, (int, float)):
254 return _compare_values(value, operator, target, path)
256 # For non-numeric values, only eq and ne are supported
257 if operator == "eq":
258 verdict = "yes" if value == target else "no"
259 elif operator == "ne":
260 verdict = "yes" if value != target else "no"
261 else:
262 return EvaluationResult(
263 verdict="error",
264 details={"error": f"Operator {operator} not supported for non-numeric values"},
265 )
267 return EvaluationResult(
268 verdict=verdict,
269 details={"value": value, "path": path, "target": target, "operator": operator},
270 )
273def evaluate_output_contains(
274 output: str,
275 pattern: str,
276 negate: bool = False,
277) -> EvaluationResult:
278 """Check if pattern exists in output.
280 Pattern can be regex or substring. If regex fails to compile,
281 falls back to substring matching.
283 Args:
284 output: The action stdout to search
285 pattern: Regex pattern or substring
286 negate: If True, invert the match result
288 Returns:
289 EvaluationResult with verdict:
290 - Found (negate=False) -> yes
291 - Found (negate=True) -> no
292 - Not found (negate=False) -> no
293 - Not found (negate=True) -> yes
294 """
295 # Try regex first, fall back to substring
296 try:
297 matched = bool(re.search(pattern, output))
298 except re.error:
299 matched = pattern in output
301 if negate:
302 verdict = "no" if matched else "yes"
303 else:
304 verdict = "yes" if matched else "no"
306 return EvaluationResult(
307 verdict=verdict,
308 details={"matched": matched, "pattern": pattern, "negate": negate},
309 )
312def evaluate_convergence(
313 current: float,
314 previous: float | None,
315 target: float,
316 tolerance: float = 0,
317 direction: str = "minimize",
318) -> EvaluationResult:
319 """Compare current value to target and previous.
321 Args:
322 current: Current metric value
323 previous: Previous metric value (None if first iteration)
324 target: Target value to reach
325 tolerance: Acceptable distance from target
326 direction: 'minimize' or 'maximize'
328 Returns:
329 EvaluationResult with verdict:
330 - Value within tolerance of target -> target
331 - Value improved toward target -> progress
332 - Value unchanged or worsened -> stall
333 """
334 # Check if target reached (within tolerance)
335 if abs(current - target) <= tolerance:
336 return EvaluationResult(
337 verdict="target",
338 details={"current": current, "target": target, "delta": 0},
339 )
341 # First iteration has no previous value
342 if previous is None:
343 return EvaluationResult(
344 verdict="progress",
345 details={
346 "current": current,
347 "previous": None,
348 "target": target,
349 "delta": None,
350 },
351 )
353 # Calculate progress
354 delta = current - previous
356 if direction == "minimize":
357 # For minimizing, negative delta is progress
358 made_progress = delta < 0
359 else:
360 # For maximizing, positive delta is progress
361 made_progress = delta > 0
363 verdict = "progress" if made_progress else "stall"
365 return EvaluationResult(
366 verdict=verdict,
367 details={
368 "current": current,
369 "previous": previous,
370 "target": target,
371 "delta": delta,
372 "direction": direction,
373 },
374 )
377def evaluate_diff_stall(
378 scope: list[str] | None = None,
379 max_stall: int = 1,
380) -> EvaluationResult:
381 """Detect stalled iterations by comparing git diff --stat between runs.
383 On first call, snapshots the current diff and returns 'yes'.
384 On subsequent calls, compares current diff to the previous snapshot.
385 If the diff is identical for max_stall consecutive iterations, returns
386 'no' (stalled). If different, resets the stall counter and returns
387 'yes' (progress).
389 State is persisted in /tmp using a key derived from the scope argument,
390 so different loops with different scopes maintain independent stall counters.
392 Args:
393 scope: Optional list of paths to limit the git diff to. Defaults to
394 the entire working tree.
395 max_stall: Number of consecutive no-change iterations before stall
396 verdict. Defaults to 1.
398 Returns:
399 EvaluationResult with verdict:
400 - yes: diff changed since last iteration (progress made)
401 - no: diff unchanged for max_stall iterations (stalled)
402 - error: git command failed or timed out
403 """
404 cmd = ["git", "diff", "--stat"]
405 if scope:
406 cmd += ["--"] + scope
408 try:
409 proc = subprocess.run(cmd, capture_output=True, text=True, timeout=30)
410 except subprocess.TimeoutExpired:
411 return EvaluationResult(verdict="error", details={"error": "git diff timed out"})
412 except FileNotFoundError:
413 return EvaluationResult(verdict="error", details={"error": "git not found in PATH"})
415 if proc.returncode != 0:
416 return EvaluationResult(
417 verdict="error",
418 details={"error": f"git diff failed: {proc.stderr[:200]}"},
419 )
421 current_diff = proc.stdout
423 # Derive a stable cache key from the scope so independent loops don't collide
424 scope_str = "|".join(sorted(scope)) if scope else "_root_"
425 cache_key = hashlib.md5(scope_str.encode()).hexdigest()[:12]
426 loops_tmp = Path.cwd() / ".loops" / "tmp"
427 loops_tmp.mkdir(parents=True, exist_ok=True)
428 state_file = loops_tmp / f"ll-diff-stall-{cache_key}.txt"
429 count_file = loops_tmp / f"ll-diff-stall-{cache_key}.count"
431 # Read previous snapshot and stall count
432 previous_diff: str | None = None
433 stall_count = 0
434 try:
435 previous_diff = state_file.read_text()
436 stall_count = int(count_file.read_text().strip())
437 except (FileNotFoundError, ValueError):
438 pass
440 # First iteration: save snapshot and report progress
441 if previous_diff is None:
442 state_file.write_text(current_diff)
443 count_file.write_text("0")
444 return EvaluationResult(
445 verdict="yes",
446 details={"stall_count": 0, "max_stall": max_stall, "diff_changed": True},
447 )
449 if current_diff == previous_diff:
450 stall_count += 1
451 count_file.write_text(str(stall_count))
452 if stall_count >= max_stall:
453 return EvaluationResult(
454 verdict="no",
455 details={"stall_count": stall_count, "max_stall": max_stall, "diff_changed": False},
456 )
457 # Not yet at max_stall threshold — still report yes so loop continues
458 return EvaluationResult(
459 verdict="yes",
460 details={"stall_count": stall_count, "max_stall": max_stall, "diff_changed": False},
461 )
462 else:
463 # Progress: update snapshot and reset counter
464 state_file.write_text(current_diff)
465 count_file.write_text("0")
466 return EvaluationResult(
467 verdict="yes",
468 details={"stall_count": 0, "max_stall": max_stall, "diff_changed": True},
469 )
472def evaluate_mcp_result(output: str, exit_code: int) -> EvaluationResult:
473 """Evaluate an MCP tool call result from the mcp-call subprocess.
475 Maps exit codes and MCP response envelope fields to routing verdicts.
477 Exit code conventions (set by mcp-call):
478 0 → parse isError from JSON envelope
479 1 → tool_error (tool ran but isError: true)
480 124 → timeout (transport-level timeout)
481 127 → not_found (server or tool missing from .mcp.json)
483 Args:
484 output: stdout from mcp-call (MCP response envelope JSON)
485 exit_code: Exit code from mcp-call subprocess
487 Returns:
488 EvaluationResult with verdict:
489 - success → isError: false
490 - tool_error → isError: true
491 - not_found → server/tool not in .mcp.json (exit 127)
492 - timeout → transport-level timeout (exit 124)
493 """
494 if exit_code == 127:
495 return EvaluationResult(
496 verdict="not_found",
497 details={"exit_code": exit_code, "error": "Server or tool not found in .mcp.json"},
498 )
500 if exit_code == 124:
501 return EvaluationResult(
502 verdict="timeout",
503 details={"exit_code": exit_code, "error": "MCP tool call timed out"},
504 )
506 # Parse MCP envelope JSON from stdout
507 try:
508 envelope = json.loads(output.strip()) if output.strip() else {}
509 except json.JSONDecodeError:
510 return EvaluationResult(
511 verdict="tool_error",
512 details={
513 "exit_code": exit_code,
514 "error": f"Invalid JSON from mcp-call: {output[:200]}",
515 },
516 )
518 is_error = envelope.get("isError", exit_code != 0)
520 if is_error:
521 return EvaluationResult(
522 verdict="tool_error",
523 details={"exit_code": exit_code, "envelope": envelope},
524 )
526 return EvaluationResult(
527 verdict="success",
528 details={"exit_code": exit_code, "envelope": envelope},
529 )
532def evaluate_harbor_scorer(output: str, exit_code: int) -> EvaluationResult:
533 """Evaluate a Harbor-format benchmark scorer result.
535 The scorer is a shell command that prints a float score (0.0–1.0) to stdout
536 and exits 0 on success or non-zero on failure.
538 Args:
539 output: stdout from the scorer subprocess (expected: a bare float)
540 exit_code: Exit code from the scorer subprocess
542 Returns:
543 EvaluationResult with verdict:
544 - yes → exit 0 and stdout parses as a float
545 - no → exit non-zero (scorer determined failure)
546 - error → exit 0 but stdout is not parseable as a float
547 """
548 if exit_code != 0:
549 return EvaluationResult(
550 verdict="no",
551 details={"exit_code": exit_code},
552 )
554 try:
555 score = float(output.strip())
556 except (ValueError, AttributeError):
557 return EvaluationResult(
558 verdict="error",
559 details={
560 "exit_code": exit_code,
561 "error": f"Scorer stdout is not a float: {output[:200]}",
562 },
563 )
565 return EvaluationResult(
566 verdict="yes",
567 details={"score": score, "exit_code": 0},
568 )
571def evaluate_llm_structured(
572 output: str,
573 prompt: str | None = None,
574 schema: dict[str, Any] | None = None,
575 min_confidence: float = 0.5,
576 uncertain_suffix: bool = False,
577 model: str = DEFAULT_LLM_MODEL,
578 max_tokens: int = 256,
579 timeout: int = 1800,
580) -> EvaluationResult:
581 """Evaluate action output using LLM with structured output via Claude CLI.
583 This is the ONLY place in the FSM system that uses LLM structured output.
584 Requires the ``claude`` CLI to be installed and authenticated.
586 Args:
587 output: Action stdout to evaluate
588 prompt: Custom evaluation prompt (defaults to basic success check)
589 schema: Custom JSON schema for structured response
590 min_confidence: Minimum confidence threshold (0-1)
591 uncertain_suffix: If True, append _uncertain to low-confidence verdicts
592 model: Model identifier (CLI aliases like "sonnet" or full names)
593 max_tokens: Maximum tokens for response (passed to --max-turns is not
594 applicable; kept for signature compat)
595 timeout: Timeout in seconds
597 Returns:
598 EvaluationResult with verdict from LLM and confidence/reason in details
599 """
600 effective_schema = schema or DEFAULT_LLM_SCHEMA
601 effective_prompt = prompt or DEFAULT_LLM_PROMPT
603 # Truncate output to avoid context limits (keep last 4000 chars)
604 truncated = output[-4000:] if len(output) > 4000 else output
606 user_prompt = f"{effective_prompt}\n\n<action_output>\n{truncated}\n</action_output>"
608 cmd = [
609 "claude",
610 "-p",
611 user_prompt,
612 "--output-format",
613 "json",
614 "--json-schema",
615 json.dumps(effective_schema),
616 "--model",
617 model,
618 "--dangerously-skip-permissions",
619 "--no-session-persistence",
620 ]
622 t0 = time.monotonic()
623 try:
624 proc = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout)
625 except subprocess.TimeoutExpired:
626 return EvaluationResult(
627 verdict="error",
628 details={"error": "LLM evaluation timeout", "timeout": True},
629 )
630 except FileNotFoundError:
631 return EvaluationResult(
632 verdict="error",
633 details={
634 "error": "claude CLI not found. Install from https://docs.anthropic.com/en/docs/claude-code",
635 "missing_dependency": True,
636 },
637 )
638 llm_latency_ms = int((time.monotonic() - t0) * 1000)
640 if proc.returncode != 0:
641 return EvaluationResult(
642 verdict="error",
643 details={"error": f"Claude CLI error: {proc.stderr.strip()}", "api_error": True},
644 )
646 # Guard: empty stdout with exit 0 (API error not reflected in exit code)
647 if not proc.stdout.strip():
648 stderr_info = proc.stderr.strip()[:200] if proc.stderr else ""
649 error_msg = "Claude CLI returned empty output"
650 if stderr_info:
651 error_msg += f" (stderr: {stderr_info})"
652 return EvaluationResult(
653 verdict="error",
654 details={"error": error_msg, "empty_output": True},
655 )
657 # Parse the CLI JSON envelope and extract structured result.
658 # With --json-schema the envelope is:
659 # success: {"type":"result","subtype":"success","structured_output":{...},...}
660 # failure: {"type":"result","subtype":"error_max_structured_output_retries",...}
661 # If stdout is JSONL (multiple JSON objects), use the last non-empty line.
662 try:
663 stdout = proc.stdout.strip()
664 try:
665 envelope = json.loads(stdout)
666 except json.JSONDecodeError:
667 # Try JSONL: take the last non-empty line
668 lines = [line for line in stdout.split("\n") if line.strip()]
669 if not lines:
670 raise
671 envelope = json.loads(lines[-1])
673 # Check structured-output retry exhaustion (--json-schema failure mode)
674 if envelope.get("subtype") == "error_max_structured_output_retries":
675 return EvaluationResult(
676 verdict="error",
677 details={
678 "error": "Claude CLI could not produce valid structured output after retries",
679 "api_error": True,
680 },
681 )
683 # Check legacy is_error flag (some CLI versions exit 0 but report error in envelope)
684 if envelope.get("is_error", False):
685 err_text = str(envelope.get("result", "") or "")[:200]
686 return EvaluationResult(
687 verdict="error",
688 details={"error": f"Claude CLI reported error: {err_text}", "api_error": True},
689 )
691 # --json-schema mode returns validated dict in "structured_output"
692 if isinstance(envelope.get("structured_output"), dict):
693 llm_result: dict[str, Any] = envelope["structured_output"]
694 else:
695 raw_result = envelope.get("result", "")
696 if isinstance(raw_result, dict):
697 llm_result = raw_result
698 elif raw_result:
699 llm_result = json.loads(raw_result)
700 elif "verdict" in envelope:
701 llm_result = envelope
702 else:
703 raw_preview = proc.stdout[:300]
704 return EvaluationResult(
705 verdict="error",
706 details={
707 "error": "Empty result field in Claude CLI response",
708 "raw_preview": raw_preview,
709 },
710 )
711 except (json.JSONDecodeError, TypeError, ValueError) as e:
712 raw_preview = proc.stdout[:300] if proc.stdout else "(empty)"
713 return EvaluationResult(
714 verdict="error",
715 details={"error": f"Failed to parse LLM response: {e}", "raw_preview": raw_preview},
716 )
718 # Build result with confidence handling
719 verdict = str(llm_result.get("verdict", "error"))
720 confidence = float(llm_result.get("confidence", 1.0))
721 confident = confidence >= min_confidence
723 # Optionally modify verdict for low confidence
724 if uncertain_suffix and not confident:
725 verdict = f"{verdict}_uncertain"
727 return EvaluationResult(
728 verdict=verdict,
729 details={
730 "confidence": confidence,
731 "confident": confident,
732 "reason": llm_result.get("reason", ""),
733 "raw": llm_result,
734 "llm_model": model,
735 "llm_latency_ms": llm_latency_ms,
736 "llm_prompt": user_prompt[:500],
737 "llm_raw_output": proc.stdout[:500] if proc.stdout else "",
738 },
739 )
742def evaluate(
743 config: EvaluateConfig,
744 output: str,
745 exit_code: int,
746 context: InterpolationContext,
747) -> EvaluationResult:
748 """Dispatch to appropriate evaluator based on config type.
750 Args:
751 config: Evaluator configuration with type and parameters
752 output: Action stdout
753 exit_code: Action exit code
754 context: Runtime context for variable interpolation
756 Returns:
757 EvaluationResult from the appropriate evaluator
759 Raises:
760 ValueError: If evaluator type is unknown
761 """
762 eval_type = config.type
764 if eval_type == "exit_code":
765 return evaluate_exit_code(exit_code)
767 elif eval_type == "output_numeric":
768 if config.target is None:
769 raise ValueError("output_numeric evaluator requires 'target' to be set")
770 elif isinstance(config.target, str):
771 try:
772 resolved = interpolate(config.target, context) if context else config.target
773 numeric_target = float(resolved)
774 except (InterpolationError, ValueError) as e:
775 raise ValueError(
776 f"output_numeric target must be numeric, got: {config.target!r}"
777 ) from e
778 else:
779 numeric_target = float(config.target)
780 return evaluate_output_numeric(
781 output=output,
782 operator=config.operator or "eq",
783 target=numeric_target,
784 )
786 elif eval_type == "output_json":
787 return evaluate_output_json(
788 output=output,
789 path=config.path or "",
790 operator=config.operator or "eq",
791 target=config.target,
792 )
794 elif eval_type == "output_contains":
795 return evaluate_output_contains(
796 output=output,
797 pattern=config.pattern or "",
798 negate=config.negate,
799 )
801 elif eval_type == "convergence":
802 # Resolve previous value from interpolation if configured
803 previous: float | None = None
804 if config.previous:
805 try:
806 previous = float(interpolate(config.previous, context))
807 except (InterpolationError, ValueError):
808 # Previous unavailable on first iteration, continue with None
809 pass
811 # Parse current value from output
812 try:
813 current = float(output.strip())
814 except ValueError:
815 return EvaluationResult(
816 verdict="error",
817 details={"error": f"Cannot parse output as number: {output[:100]}"},
818 )
820 # Resolve target (may be interpolated string like "${context.target}")
821 convergence_target: float
822 if isinstance(config.target, str):
823 try:
824 convergence_target = float(interpolate(config.target, context))
825 except (InterpolationError, ValueError) as e:
826 return EvaluationResult(
827 verdict="error",
828 details={"error": f"Cannot resolve target: {e}"},
829 )
830 else:
831 if config.target is None:
832 raise ValueError("convergence evaluator requires 'target' to be set")
833 convergence_target = float(config.target)
835 # Resolve tolerance (may be interpolated string)
836 tolerance: float = 0.0
837 if config.tolerance is not None:
838 if isinstance(config.tolerance, str):
839 try:
840 tolerance = float(interpolate(config.tolerance, context))
841 except (InterpolationError, ValueError):
842 tolerance = 0.0
843 else:
844 tolerance = float(config.tolerance)
846 return evaluate_convergence(
847 current=current,
848 previous=previous,
849 target=convergence_target,
850 tolerance=tolerance,
851 direction=config.direction,
852 )
854 elif eval_type == "diff_stall":
855 return evaluate_diff_stall(
856 scope=config.scope,
857 max_stall=config.max_stall,
858 )
860 elif eval_type == "llm_structured":
861 prompt = config.prompt
862 if prompt and context:
863 try:
864 prompt = interpolate(prompt, context)
865 except InterpolationError:
866 pass # Use raw prompt on resolution failure
867 return evaluate_llm_structured(
868 output=output,
869 prompt=prompt,
870 schema=config.schema,
871 min_confidence=config.min_confidence,
872 uncertain_suffix=config.uncertain_suffix,
873 )
875 elif eval_type == "mcp_result":
876 return evaluate_mcp_result(output=output, exit_code=exit_code)
878 elif eval_type == "harbor_scorer":
879 return evaluate_harbor_scorer(output=output, exit_code=exit_code)
881 else:
882 raise ValueError(f"Unknown evaluator type: {eval_type}")