1"""FSM Evaluators for loop execution.

2

3 This module provides evaluators that interpret action output and produce

4 verdicts for state transitions.

5

6 Supported evaluator types:

7

8 Tier 1 (Deterministic - no API calls):

9 exit_code: Map Unix exit codes to verdicts (0=success, 1=failure, 2+=error)

10 output_numeric: Compare numeric output to target value

11 output_json: Extract and compare JSON path values

12 output_contains: Pattern matching on stdout

13 convergence: Track progress toward a target value

14 diff_stall: Detect stalled iterations via git diff comparison

15 action_stall: Detect when the same action string or output repeats for N consecutive iterations

16 harbor_scorer: Interpret Harbor-format benchmark scorer exit code and float stdout

17

18 Tier 2 (LLM-based):

19 llm_structured: Use LLM with structured output for natural language evaluation

20 contract: Read producer/consumer file pairs and assert contract alignment via LLM judge

21

22 Tier 3 (External process):

23 mcp_result: Parse MCP tool call response envelope

24"""

25

26 from __future__ import annotations

27

28 import hashlib

29 import json

30 import random

31 import re

32 import subprocess

33 import time

34 from collections.abc import Callable

35 from dataclasses import dataclass

36 from pathlib import Path

37 from typing import Any

38

39 from little_loops.fsm.interpolation import (

40 InterpolationContext,

41 InterpolationError,

42 interpolate,

43)

44 from little_loops.fsm.schema import DEFAULT_LLM_MODEL, EvaluateConfig

45 from little_loops.host_runner import resolve_host

46

47

48@dataclass

49 class EvaluationResult:

50 """Result from an evaluator.

51

52 Attributes:

53 verdict: The routing key for state transitions

54 details: Evaluator-specific metadata for debugging/logging

55 """

56

57 verdict: str

58 details: dict[str, Any]

59

60

61# Evidence contract injected into every LLM evaluator prompt (ENH-2342).

62# Requires the model to cite verbatim output text for each verdict; absent evidence

63# is coerced to "no" at the parsing layer so self-grade optimism can't slip through.

64 CHECK_SEMANTIC_EVIDENCE_CONTRACT = """

65 IMPORTANT: For each condition you evaluate:

66 1. State your verdict: Yes / No / Partial

67 2. Provide a VERBATIM quote from the output that supports your verdict (exact text, in quotes)

68 3. If you cannot quote specific text, your verdict is automatically No (or Partial if context suggests partial progress)

69

70 Do not assert a verdict without evidence. "The task appears complete" is not evidence.

71"""

72

73# Default schema for LLM structured evaluation

74 DEFAULT_LLM_SCHEMA: dict[str, Any] = {

75 "type": "object",

76 "properties": {

77 "verdict": {

78 "type": "string",

79 "enum": ["yes", "no", "blocked", "partial"],

80 "description": (

81 "- yes: The condition/check evaluated to true\n"

82 "- no: The condition/check evaluated to false\n"

83 "- blocked: Cannot proceed without external help\n"

84 "- partial: Made progress but not complete"

85 ),

86 },

87 "confidence": {

88 "type": "number",

89 "minimum": 0,

90 "maximum": 1,

91 "description": "Confidence in this verdict (0-1)",

92 },

93 "reason": {

94 "type": "string",

95 "description": "Brief explanation",

96 },

97 "evidence": {

98 "type": "string",

99 "description": (

100 "Verbatim quote from the action output that supports the verdict. "

101 "Empty string means no evidence was found; verdict will be coerced to 'no'."

102 ),

103 },

104 },

105 "required": ["verdict", "confidence", "reason", "evidence"],

106}

107

108 DEFAULT_LLM_PROMPT = "Evaluate whether this action succeeded based on its output."

109

110# Schema for blind A/B comparator: evaluates two anonymized outputs

111 BLIND_COMPARATOR_SCHEMA: dict[str, Any] = {

112 "type": "object",

113 "properties": {

114 "verdict_a": {

115 "type": "string",

116 "enum": ["yes", "no"],

117 "description": "Whether Output A meets the evaluation criteria",

118 },

119 "verdict_b": {

120 "type": "string",

121 "enum": ["yes", "no"],

122 "description": "Whether Output B meets the evaluation criteria",

123 },

124 "confidence": {

125 "type": "number",

126 "minimum": 0,

127 "maximum": 1,

128 "description": "Confidence in these verdicts (0-1)",

129 },

130 "reason": {

131 "type": "string",

132 "description": "Brief explanation comparing the two outputs",

133 },

134 },

135 "required": ["verdict_a", "verdict_b", "confidence", "reason"],

136}

137

138 DEFAULT_BLIND_COMPARATOR_PROMPT = (

139 "You are evaluating two outputs (labeled 'Output A' and 'Output B') that were "

140 "produced by independent runs of the same task. Judge whether each output meets "

141 "the evaluation criteria below. Be objective and impartial — the labels 'A' and "

142 "'B' are arbitrary and do not indicate which is better."

143)

144

145_NUMERIC_OPERATORS: dict[str, Callable[[float, float], bool]] = {

146 "eq": lambda v, t: v == t,

147 "ne": lambda v, t: v != t,

148 "lt": lambda v, t: v < t,

149 "le": lambda v, t: v <= t,

150 "gt": lambda v, t: v > t,

151 "ge": lambda v, t: v >= t,

152}

153

154

155 def evaluate_exit_code(exit_code: int) -> EvaluationResult:

156 """Map Unix exit code to verdict.

157

158 Args:

159 exit_code: The process exit code

160

161 Returns:

162 EvaluationResult with verdict:

163 - 0 -> yes

164 - 1 -> no

165 - 2+ -> error

166 """

167 if exit_code == 0:

168 verdict = "yes"

169 elif exit_code == 1:

170 verdict = "no"

171 else:

172 verdict = "error"

173

174 return EvaluationResult(verdict=verdict, details={"exit_code": exit_code})

175

176

177 def evaluate_output_numeric(

178 output: str,

179 operator: str,

180 target: float,

181) -> EvaluationResult:

182 """Parse stdout as number and compare to target.

183

184 Args:

185 output: The action stdout to parse as a number

186 operator: Comparison operator (eq, ne, lt, le, gt, ge)

187 target: Target value to compare against

188

189 Returns:

190 EvaluationResult with verdict:

191 - Condition met -> yes

192 - Condition not met -> no

193 - Parse error -> error

194 """

195 try:

196 value = float(output.strip())

197 except ValueError:

198 return EvaluationResult(

199 verdict="error",

200 details={"error": f"Cannot parse as number: {output[:100]}"},

201 )

202

203 if operator not in _NUMERIC_OPERATORS:

204 return EvaluationResult(

205 verdict="error",

206 details={"error": f"Unknown operator: {operator}"},

207 )

208

209 condition_met = _NUMERIC_OPERATORS[operator](value, target)

210 return EvaluationResult(

211 verdict="yes" if condition_met else "no",

212 details={"value": value, "target": target, "operator": operator},

213 )

214

215

216 def _extract_json_path(data: Any, path: str) -> Any:

217 """Extract value from dict using jq-style path like '.summary.failed'.

218

219 Args:

220 data: The parsed JSON data (dict or list)

221 path: Dot-separated path, optionally starting with '.'

222

223 Returns:

224 The value at the specified path

225

226 Raises:

227 KeyError: If path not found in data

228 """

229 if path.startswith("."):

230 path = path[1:]

231 parts = path.split(".")

232 current = data

233 for part in parts:

234 if isinstance(current, dict) and part in current:

235 current = current[part]

236 elif isinstance(current, list) and part.isdigit():

237 idx = int(part)

238 if 0 <= idx < len(current):

239 current = current[idx]

240 else:

241 raise KeyError(path)

242 else:

243 raise KeyError(path)

244 return current

245

246

247 def _compare_values(

248 value: int | float, operator: str, target: int | float, path: str

249) -> EvaluationResult:

250 """Compare numeric values using operator.

251

252 Args:

253 value: The extracted value to compare

254 operator: Comparison operator

255 target: Target value

256 path: JSON path for details

257

258 Returns:

259 EvaluationResult with comparison result

260 """

261 if operator not in _NUMERIC_OPERATORS:

262 return EvaluationResult(

263 verdict="error",

264 details={"error": f"Unknown operator: {operator}"},

265 )

266

267 condition_met = _NUMERIC_OPERATORS[operator](value, target)

268 return EvaluationResult(

269 verdict="yes" if condition_met else "no",

270 details={"value": value, "path": path, "target": target, "operator": operator},

271 )

272

273

274 def evaluate_output_json(

275 output: str,

276 path: str,

277 operator: str,

278 target: Any,

279) -> EvaluationResult:

280 """Parse JSON and extract value at path, then compare.

281

282 Args:

283 output: The action stdout containing JSON

284 path: jq-style dot notation path (e.g., '.summary.failed')

285 operator: Comparison operator (eq, ne, lt, le, gt, ge)

286 target: Target value for comparison

287

288 Returns:

289 EvaluationResult with verdict:

290 - Condition met -> yes

291 - Condition not met -> no

292 - Parse/path error -> error

293 """

294 try:

295 data = json.loads(output)

296 except json.JSONDecodeError as e:

297 return EvaluationResult(

298 verdict="error",

299 details={"error": f"Invalid JSON: {e}"},

300 )

301

302 try:

303 value = _extract_json_path(data, path)

304 except KeyError:

305 return EvaluationResult(

306 verdict="error",

307 details={"error": f"Path not found: {path}"},

308 )

309

310 # Use numeric comparison if both values are numeric

311 if isinstance(value, (int, float)) and isinstance(target, (int, float)):

312 return _compare_values(value, operator, target, path)

313

314 # For non-numeric values, only eq and ne are supported

315 if operator == "eq":

316 verdict = "yes" if value == target else "no"

317 elif operator == "ne":

318 verdict = "yes" if value != target else "no"

319 else:

320 return EvaluationResult(

321 verdict="error",

322 details={"error": f"Operator {operator} not supported for non-numeric values"},

323 )

324

325 return EvaluationResult(

326 verdict=verdict,

327 details={"value": value, "path": path, "target": target, "operator": operator},

328 )

329

330

331 def evaluate_output_contains(

332 output: str,

333 pattern: str,

334 negate: bool = False,

335 error_patterns: list[str] | None = None,

336) -> EvaluationResult:

337 """Check if pattern exists in output.

338

339 Pattern can be regex or substring. If regex fails to compile,

340 falls back to substring matching.

341

342 Args:

343 output: The action stdout to search

344 pattern: Regex pattern or substring

345 negate: If True, invert the match result

346 error_patterns: Optional list of substrings that, when matched in output

347 and the main pattern is not found, yield verdict="error". This allows

348 loops to route auth/error output to on_error without raising an exception.

349

350 Returns:

351 EvaluationResult with verdict:

352 - Found (negate=False) -> yes

353 - Found (negate=True) -> no

354 - Not found (negate=False) -> no (or "error" if error_patterns matched)

355 - Not found (negate=True) -> yes

356 """

357 # Try regex first, fall back to substring

358 try:

359 matched = bool(re.search(pattern, output))

360 except re.error:

361 matched = pattern in output

362

363 if negate:

364 verdict = "no" if matched else "yes"

365 else:

366 verdict = "yes" if matched else "no"

367

368 # Check error_patterns before returning "no" — only when main pattern didn't match

369 if verdict == "no" and not negate and error_patterns:

370 for ep in error_patterns:

371 if ep in output:

372 return EvaluationResult(

373 verdict="error",

374 details={

375 "matched": False,

376 "pattern": pattern,

377 "negate": negate,

378 "error_pattern": ep,

379 },

380 )

381

382 return EvaluationResult(

383 verdict=verdict,

384 details={"matched": matched, "pattern": pattern, "negate": negate},

385 )

386

387

388 def evaluate_convergence(

389 current: float,

390 previous: float | None,

391 target: float,

392 tolerance: float = 0,

393 direction: str = "minimize",

394) -> EvaluationResult:

395 """Compare current value to target and previous.

396

397 Args:

398 current: Current metric value

399 previous: Previous metric value (None if first iteration)

400 target: Target value to reach

401 tolerance: Acceptable distance from target

402 direction: 'minimize' or 'maximize'

403

404 Returns:

405 EvaluationResult with verdict:

406 - Value within tolerance of target -> target

407 - Value improved toward target -> progress

408 - Value unchanged or worsened -> stall

409 """

410 # Check if target reached (within tolerance)

411 if abs(current - target) <= tolerance:

412 return EvaluationResult(

413 verdict="target",

414 details={"current": current, "target": target, "delta": 0},

415 )

416

417 # First iteration has no previous value

418 if previous is None:

419 return EvaluationResult(

420 verdict="progress",

421 details={

422 "current": current,

423 "previous": None,

424 "target": target,

425 "delta": None,

426 },

427 )

428

429 # Calculate progress

430 delta = current - previous

431

432 if direction == "minimize":

433 # For minimizing, negative delta is progress

434 made_progress = delta < 0

435 else:

436 # For maximizing, positive delta is progress

437 made_progress = delta > 0

438

439 verdict = "progress" if made_progress else "stall"

440

441 return EvaluationResult(

442 verdict=verdict,

443 details={

444 "current": current,

445 "previous": previous,

446 "target": target,

447 "delta": delta,

448 "direction": direction,

449 },

450 )

451

452

453 def evaluate_classify(

454 output: str,

455 line: str | int | None = None,

456) -> EvaluationResult:

457 """Read a token from stdout and return it as the verdict.

458

459 Intended for single-state multi-way routing: the action prints exactly one

460 token to stdout and the route: table maps that token to the next state.

461

462 Args:

463 output: The action stdout to read the token from

464 line: Which line to select. 'last' (default) picks the last non-empty

465 line; 'first' picks the first non-empty line; an integer index

466 selects that line (0-based, negative indices supported).

467

468 Returns:

469 EvaluationResult with verdict = trimmed token, or "" when output is

470 empty (which _route() maps to the route.default fallback).

471 """

472 lines = [ln for ln in output.splitlines() if ln.strip()]

473 if not lines:

474 return EvaluationResult(

475 verdict="",

476 details={"token": "", "line": line, "source_lines": 0},

477 )

478

479 selector = line if line is not None else "last"

480 if selector == "last":

481 selected = lines[-1]

482 elif selector == "first":

483 selected = lines[0]

484 elif isinstance(selector, int):

485 try:

486 selected = lines[selector]

487 except IndexError:

488 return EvaluationResult(

489 verdict="",

490 details={

491 "token": "",

492 "line": line,

493 "source_lines": len(lines),

494 "error": "index out of range",

495 },

496 )

497 else:

498 selected = lines[-1]

499

500 token = selected.strip()

501 return EvaluationResult(

502 verdict=token,

503 details={"token": token, "line": line},

504 )

505

506

507 def evaluate_diff_stall(

508 scope: list[str] | None = None,

509 max_stall: int = 1,

510) -> EvaluationResult:

511 """Detect stalled iterations by comparing git diff --stat between runs.

512

513 On first call, snapshots the current diff and returns 'yes'.

514 On subsequent calls, compares current diff to the previous snapshot.

515 If the diff is identical for max_stall consecutive iterations, returns

516 'no' (stalled). If different, resets the stall counter and returns

517 'yes' (progress).

518

519 State is persisted in /tmp using a key derived from the scope argument,

520 so different loops with different scopes maintain independent stall counters.

521

522 Args:

523 scope: Optional list of paths to limit the git diff to. Defaults to

524 the entire working tree.

525 max_stall: Number of consecutive no-change iterations before stall

526 verdict. Defaults to 1.

527

528 Returns:

529 EvaluationResult with verdict:

530 - yes: diff changed since last iteration (progress made)

531 - no: diff unchanged for max_stall iterations (stalled)

532 - error: git command failed or timed out

533 """

534 cmd = ["git", "diff", "--stat"]

535 if scope:

536 cmd += ["--"] + scope

537

538 try:

539 proc = subprocess.run(cmd, capture_output=True, text=True, timeout=30)

540 except subprocess.TimeoutExpired:

541 return EvaluationResult(verdict="error", details={"error": "git diff timed out"})

542 except FileNotFoundError:

543 return EvaluationResult(verdict="error", details={"error": "git not found in PATH"})

544

545 if proc.returncode != 0:

546 return EvaluationResult(

547 verdict="error",

548 details={"error": f"git diff failed: {proc.stderr[:200]}"},

549 )

550

551 current_diff = proc.stdout

552

553 # Derive a stable cache key from the scope so independent loops don't collide

554 scope_str = "|".join(sorted(scope)) if scope else "_root_"

555 cache_key = hashlib.md5(scope_str.encode()).hexdigest()[:12]

556 loops_tmp = Path.cwd() / ".loops" / "tmp"

557 loops_tmp.mkdir(parents=True, exist_ok=True)

558 state_file = loops_tmp / f"ll-diff-stall-{cache_key}.txt"

559 count_file = loops_tmp / f"ll-diff-stall-{cache_key}.count"

560

561 # Read previous snapshot and stall count

562 previous_diff: str | None = None

563 stall_count = 0

564 try:

565 previous_diff = state_file.read_text()

566 stall_count = int(count_file.read_text().strip())

567 except (FileNotFoundError, ValueError):

568 pass

569

570 # First iteration: save snapshot and report progress

571 if previous_diff is None:

572 state_file.write_text(current_diff)

573 count_file.write_text("0")

574 return EvaluationResult(

575 verdict="yes",

576 details={"stall_count": 0, "max_stall": max_stall, "diff_changed": True},

577 )

578

579 if current_diff == previous_diff:

580 stall_count += 1

581 count_file.write_text(str(stall_count))

582 if stall_count >= max_stall:

583 return EvaluationResult(

584 verdict="no",

585 details={"stall_count": stall_count, "max_stall": max_stall, "diff_changed": False},

586 )

587 # Not yet at max_stall threshold — still report yes so loop continues

588 return EvaluationResult(

589 verdict="yes",

590 details={"stall_count": stall_count, "max_stall": max_stall, "diff_changed": False},

591 )

592 else:

593 # Progress: update snapshot and reset counter

594 state_file.write_text(current_diff)

595 count_file.write_text("0")

596 return EvaluationResult(

597 verdict="yes",

598 details={"stall_count": 0, "max_stall": max_stall, "diff_changed": True},

599 )

600

601

602 def evaluate_action_stall(

603 track: list[str] | None = None,

604 max_repeat: int = 2,

605 context: InterpolationContext | None = None,

606) -> EvaluationResult:

607 """Detect when the same action string or output repeats for N consecutive iterations.

608

609 On first call, snapshots the hashed values of the tracked context keys and returns

610 'yes'. On subsequent calls, compares the current hash to the previous snapshot.

611 If the hash is identical for max_repeat consecutive iterations, returns 'no'

612 (stalled). If different, resets the stall counter and returns 'yes' (progress).

613

614 State is persisted in .loops/tmp using a key derived from the tracked keys,

615 so different states/loops maintain independent stall counters.

616

617 Args:

618 track: Context keys to track. Defaults to ["action"] when None.

619 max_repeat: Number of consecutive identical-hash iterations before stall verdict.

620 Defaults to 2.

621 context: Runtime interpolation context for resolving tracked keys.

622

623 Returns:

624 EvaluationResult with verdict:

625 - yes: tracked values changed since last iteration (progress made)

626 - no: tracked values identical for max_repeat iterations (stalled)

627 """

628 effective_track: list[str] = track if track is not None else ["action"]

629

630 # Resolve each tracked key from context and hash the combined values.

631 # Keys may be bare names (e.g. "action") or namespaced (e.g. "context.action").

632 # Try namespaced forms first: context.<key>, captured.<key>, then bare ${key}.

633 parts: list[str] = []

634 for key in effective_track:

635 value: str = ""

636 if context is not None:

637 # If key already contains a dot it's already namespaced; use as-is.

638 if "." in key:

639 try:

640 value = str(interpolate(f"${{{key}}}", context))

641 except InterpolationError:

642 value = ""

643 else:

644 # Try context.<key> first, then captured.<key>, then give up.

645 resolved = False

646 for namespace in ("context", "captured", "prev", "result"):

647 try:

648 value = str(interpolate(f"${{{namespace}.{key}}}", context))

649 resolved = True

650 break

651 except InterpolationError:

652 continue

653 if not resolved:

654 value = ""

655 parts.append(f"{key}={value}")

656

657 combined = "|".join(parts)

658 current_hash = hashlib.md5(combined.encode()).hexdigest()

659

660 # Derive a stable cache key from the tracked keys

661 track_str = "|".join(sorted(effective_track))

662 cache_key = hashlib.md5(track_str.encode()).hexdigest()[:12]

663 loops_tmp = Path.cwd() / ".loops" / "tmp"

664 loops_tmp.mkdir(parents=True, exist_ok=True)

665 state_file = loops_tmp / f"ll-action-stall-{cache_key}.txt"

666 count_file = loops_tmp / f"ll-action-stall-{cache_key}.count"

667

668 # Read previous hash and stall count

669 previous_hash: str | None = None

670 stall_count = 0

671 try:

672 previous_hash = state_file.read_text().strip()

673 stall_count = int(count_file.read_text().strip())

674 except (FileNotFoundError, ValueError):

675 pass

676

677 # First iteration: save hash and report progress

678 if previous_hash is None:

679 state_file.write_text(current_hash)

680 count_file.write_text("0")

681 return EvaluationResult(

682 verdict="yes",

683 details={

684 "stall_count": 0,

685 "max_repeat": max_repeat,

686 "hash_changed": True,

687 "tracked_keys": effective_track,

688 },

689 )

690

691 hash_changed = current_hash != previous_hash

692

693 if hash_changed:

694 # Progress: update snapshot and reset counter

695 state_file.write_text(current_hash)

696 count_file.write_text("0")

697 return EvaluationResult(

698 verdict="yes",

699 details={

700 "stall_count": 0,

701 "max_repeat": max_repeat,

702 "hash_changed": True,

703 "tracked_keys": effective_track,

704 },

705 )

706 else:

707 # Same hash as last time

708 stall_count += 1

709 count_file.write_text(str(stall_count))

710 if stall_count >= max_repeat:

711 return EvaluationResult(

712 verdict="no",

713 details={

714 "stall_count": stall_count,

715 "max_repeat": max_repeat,

716 "hash_changed": False,

717 "tracked_keys": effective_track,

718 "repeated_hash": current_hash,

719 },

720 )

721 # Not yet at max_repeat threshold — still report yes so loop continues

722 return EvaluationResult(

723 verdict="yes",

724 details={

725 "stall_count": stall_count,

726 "max_repeat": max_repeat,

727 "hash_changed": False,

728 "tracked_keys": effective_track,

729 },

730 )

731

732

733 def evaluate_mcp_result(output: str, exit_code: int) -> EvaluationResult:

734 """Evaluate an MCP tool call result from the mcp-call subprocess.

735

736 Maps exit codes and MCP response envelope fields to routing verdicts.

737

738 Exit code conventions (set by mcp-call):

739 0 → parse isError from JSON envelope

740 1 → tool_error (tool ran but isError: true)

741 124 → timeout (transport-level timeout)

742 127 → not_found (server or tool missing from .mcp.json)

743

744 Args:

745 output: stdout from mcp-call (MCP response envelope JSON)

746 exit_code: Exit code from mcp-call subprocess

747

748 Returns:

749 EvaluationResult with verdict:

750 - success → isError: false

751 - tool_error → isError: true

752 - not_found → server/tool not in .mcp.json (exit 127)

753 - timeout → transport-level timeout (exit 124)

754 """

755 if exit_code == 127:

756 return EvaluationResult(

757 verdict="not_found",

758 details={"exit_code": exit_code, "error": "Server or tool not found in .mcp.json"},

759 )

760

761 if exit_code == 124:

762 return EvaluationResult(

763 verdict="timeout",

764 details={"exit_code": exit_code, "error": "MCP tool call timed out"},

765 )

766

767 # Parse MCP envelope JSON from stdout

768 try:

769 envelope = json.loads(output.strip()) if output.strip() else {}

770 except json.JSONDecodeError:

771 return EvaluationResult(

772 verdict="tool_error",

773 details={

774 "exit_code": exit_code,

775 "error": f"Invalid JSON from mcp-call: {output[:200]}",

776 },

777 )

778

779 is_error = envelope.get("isError", exit_code != 0)

780

781 if is_error:

782 return EvaluationResult(

783 verdict="tool_error",

784 details={"exit_code": exit_code, "envelope": envelope},

785 )

786

787 return EvaluationResult(

788 verdict="success",

789 details={"exit_code": exit_code, "envelope": envelope},

790 )

791

792

793 def evaluate_harbor_scorer(output: str, exit_code: int) -> EvaluationResult:

794 """Evaluate a Harbor-format benchmark scorer result.

795

796 The scorer is a shell command that prints a float score (0.0–1.0) to stdout

797 and exits 0 on success or non-zero on failure.

798

799 Args:

800 output: stdout from the scorer subprocess (expected: a bare float)

801 exit_code: Exit code from the scorer subprocess

802

803 Returns:

804 EvaluationResult with verdict:

805 - yes → exit 0 and stdout parses as a float

806 - no → exit non-zero (scorer determined failure)

807 - error → exit 0 but stdout is not parseable as a float

808 """

809 if exit_code != 0:

810 return EvaluationResult(

811 verdict="no",

812 details={"exit_code": exit_code},

813 )

814

815 try:

816 score = float(output.strip())

817 except (ValueError, AttributeError):

818 return EvaluationResult(

819 verdict="error",

820 details={

821 "exit_code": exit_code,

822 "error": f"Scorer stdout is not a float: {output[:200]}",

823 },

824 )

825

826 return EvaluationResult(

827 verdict="yes",

828 details={"score": score, "exit_code": 0},

829 )

830

831

832 def evaluate_llm_structured(

833 output: str,

834 prompt: str | None = None,

835 schema: dict[str, Any] | None = None,

836 min_confidence: float = 0.5,

837 uncertain_suffix: bool = False,

838 model: str = DEFAULT_LLM_MODEL,

839 max_tokens: int = 256,

840 timeout: int = 1800,

841) -> EvaluationResult:

842 """Evaluate action output using LLM with structured output via Claude CLI.

843

844 This is the ONLY place in the FSM system that uses LLM structured output.

845 Requires the ``claude`` CLI to be installed and authenticated.

846

847 Args:

848 output: Action stdout to evaluate

849 prompt: Custom evaluation prompt (defaults to basic success check)

850 schema: Custom JSON schema for structured response

851 min_confidence: Minimum confidence threshold (0-1)

852 uncertain_suffix: If True, append _uncertain to low-confidence verdicts

853 model: Model identifier (CLI aliases like "sonnet" or full names)

854 max_tokens: Maximum tokens for response (passed to --max-turns is not

855 applicable; kept for signature compat)

856 timeout: Timeout in seconds

857

858 Returns:

859 EvaluationResult with verdict from LLM and confidence/reason in details

860 """

861 effective_schema = schema or DEFAULT_LLM_SCHEMA

862 effective_prompt = (prompt or DEFAULT_LLM_PROMPT) + "\n\n" + CHECK_SEMANTIC_EVIDENCE_CONTRACT

863

864 # Truncate output to avoid context limits (keep last 4000 chars)

865 truncated = output[-4000:] if len(output) > 4000 else output

866

867 user_prompt = f"{effective_prompt}\n\n<action_output>\n{truncated}\n</action_output>"

868

869 invocation = resolve_host().build_blocking_json(prompt=user_prompt, model=model)

870 # Builder drops json_schema (Protocol surface only) and omits the

871 # claude-CLI-specific --no-session-persistence flag; augment at call site.

872 args = list(invocation.args) + [

873 "--json-schema",

874 json.dumps(effective_schema),

875 "--no-session-persistence",

876 ]

877

878 t0 = time.monotonic()

879 try:

880 proc = subprocess.run(

881 [invocation.binary, *args], capture_output=True, text=True, timeout=timeout

882 )

883 except subprocess.TimeoutExpired:

884 return EvaluationResult(

885 verdict="error",

886 details={"error": "LLM evaluation timeout", "timeout": True},

887 )

888 except FileNotFoundError:

889 return EvaluationResult(

890 verdict="error",

891 details={

892 "error": f"{invocation.binary} CLI not found. Install the active host CLI (see LL_HOST_CLI).",

893 "missing_dependency": True,

894 },

895 )

896 llm_latency_ms = int((time.monotonic() - t0) * 1000)

897

898 if proc.returncode != 0:

899 return EvaluationResult(

900 verdict="error",

901 details={

902 "error": f"{invocation.binary} CLI error: {proc.stderr.strip()}",

903 "api_error": True,

904 },

905 )

906

907 # Guard: empty stdout with exit 0 (API error not reflected in exit code)

908 if not proc.stdout.strip():

909 stderr_info = proc.stderr.strip()[:200] if proc.stderr else ""

910 error_msg = f"{invocation.binary} CLI returned empty output"

911 if stderr_info:

912 error_msg += f" (stderr: {stderr_info})"

913 return EvaluationResult(

914 verdict="error",

915 details={"error": error_msg, "empty_output": True},

916 )

917

918 # Parse the CLI JSON envelope and extract structured result.

919 # With --json-schema the envelope is:

920 # success: {"type":"result","subtype":"success","structured_output":{...},...}

921 # failure: {"type":"result","subtype":"error_max_structured_output_retries",...}

922 # If stdout is JSONL (multiple JSON objects), use the last non-empty line.

923 try:

924 stdout = proc.stdout.strip()

925 try:

926 envelope = json.loads(stdout)

927 except json.JSONDecodeError:

928 # Try JSONL: take the last non-empty line

929 lines = [line for line in stdout.split("\n") if line.strip()]

930 if not lines:

931 raise

932 envelope = json.loads(lines[-1])

933

934 # Check structured-output retry exhaustion (--json-schema failure mode)

935 if envelope.get("subtype") == "error_max_structured_output_retries":

936 return EvaluationResult(

937 verdict="error",

938 details={

939 "error": "Claude CLI could not produce valid structured output after retries",

940 "api_error": True,

941 },

942 )

943

944 # Check legacy is_error flag (some CLI versions exit 0 but report error in envelope)

945 if envelope.get("is_error", False):

946 err_text = str(envelope.get("result", "") or "")[:200]

947 return EvaluationResult(

948 verdict="error",

949 details={"error": f"Claude CLI reported error: {err_text}", "api_error": True},

950 )

951

952 # --json-schema mode returns validated dict in "structured_output"

953 if isinstance(envelope.get("structured_output"), dict):

954 llm_result: dict[str, Any] = envelope["structured_output"]

955 else:

956 raw_result = envelope.get("result", "")

957 if isinstance(raw_result, dict):

958 llm_result = raw_result

959 elif raw_result:

960 llm_result = json.loads(raw_result)

961 elif "verdict" in envelope:

962 llm_result = envelope

963 else:

964 raw_preview = proc.stdout[:300]

965 return EvaluationResult(

966 verdict="error",

967 details={

968 "error": "Empty result field in Claude CLI response",

969 "raw_preview": raw_preview,

970 },

971 )

972 except (json.JSONDecodeError, TypeError, ValueError) as e:

973 raw_preview = proc.stdout[:300] if proc.stdout else "(empty)"

974 return EvaluationResult(

975 verdict="error",

976 details={"error": f"Failed to parse LLM response: {e}", "raw_preview": raw_preview},

977 )

978

979 # Build result with confidence handling

980 verdict = str(llm_result.get("verdict", "error"))

981 confidence = float(llm_result.get("confidence", 1.0))

982 confident = confidence >= min_confidence

983

984 # Evidence coercion (ENH-2342): when using the default schema, a verdict without

985 # verbatim evidence is treated as "no" — prevents LLM optimism from passing silently.

986 # Custom schemas bypass this; callers that supply their own schema control the contract.

987 evidence = llm_result.get("evidence", "")

988 evidence_coerced = schema is None and not evidence.strip() and verdict not in ("error",)

989 if evidence_coerced:

990 verdict = "no"

991

992 # Optionally modify verdict for low confidence

993 if uncertain_suffix and not confident:

994 verdict = f"{verdict}_uncertain"

995

996 return EvaluationResult(

997 verdict=verdict,

998 details={

999 "confidence": confidence,

1000 "confident": confident,

1001 "reason": llm_result.get("reason", ""),

1002 "evidence": evidence,

1003 "evidence_coerced": evidence_coerced,

1004 "raw": llm_result,

1005 "llm_model": model,

1006 "llm_latency_ms": llm_latency_ms,

1007 "llm_prompt": user_prompt[:500],

1008 "llm_raw_output": proc.stdout[:500] if proc.stdout else "",

1009 },

1010 )

1011

1012

1013 def evaluate_blind_comparator(

1014 output_harness: str,

1015 output_baseline: str,

1016 prompt: str | None = None,

1017 model: str = DEFAULT_LLM_MODEL,

1018 timeout: int = 1800,

1019) -> dict[str, Any]:

1020 """Blindly evaluate two outputs, returning pass/fail for each arm.

1021

1022 Outputs are randomly labeled "Output A" / "Output B" so the LLM judge

1023 cannot distinguish the harness arm from the baseline arm. The mapping is

1024 de-anonymized after judgment so callers receive harness/baseline verdicts.

1025

1026 Args:

1027 output_harness: stdout from the harness (gated) arm

1028 output_baseline: stdout from the baseline (ungated) arm

1029 prompt: Custom evaluation prompt (appended to default framing)

1030 model: Model identifier for the judge

1031 timeout: Timeout in seconds

1032

1033 Returns:

1034 Dict with keys: harness_pass (bool), baseline_pass (bool),

1035 confidence (float), reason (str), raw (dict with A/B verdicts)

1036 """

1037 effective_prompt = prompt or DEFAULT_BLIND_COMPARATOR_PROMPT

1038

1039 # Truncate outputs to avoid context limits

1040 truncated_harness = output_harness[-4000:] if len(output_harness) > 4000 else output_harness

1041 truncated_baseline = output_baseline[-4000:] if len(output_baseline) > 4000 else output_baseline

1042

1043 # Randomize order: coin flip determines whether harness→A / baseline→B

1044 harness_is_a = random.choice([True, False])

1045 if harness_is_a:

1046 output_a, output_b = truncated_harness, truncated_baseline

1047 else:

1048 output_a, output_b = truncated_baseline, truncated_harness

1049

1050 user_prompt = (

1051 f"{effective_prompt}\n\n"

1052 f"<output_a>\n{output_a}\n</output_a>\n\n"

1053 f"<output_b>\n{output_b}\n</output_b>"

1054 )

1055

1056 invocation = resolve_host().build_blocking_json(prompt=user_prompt, model=model)

1057 args = list(invocation.args) + [

1058 "--json-schema",

1059 json.dumps(BLIND_COMPARATOR_SCHEMA),

1060 "--no-session-persistence",

1061 ]

1062

1063 try:

1064 proc = subprocess.run(

1065 [invocation.binary, *args], capture_output=True, text=True, timeout=timeout

1066 )

1067 except subprocess.TimeoutExpired:

1068 # On timeout, both fail — conservative default

1069 return {

1070 "harness_pass": False,

1071 "baseline_pass": False,

1072 "confidence": 0.0,

1073 "reason": "LLM evaluation timed out",

1074 "raw": {"verdict_a": "timeout", "verdict_b": "timeout"},

1075 "error": "timeout",

1076 }

1077 except FileNotFoundError:

1078 return {

1079 "harness_pass": False,

1080 "baseline_pass": False,

1081 "confidence": 0.0,

1082 "reason": f"{invocation.binary} CLI not found",

1083 "raw": {"verdict_a": "error", "verdict_b": "error"},

1084 "error": "missing_cli",

1085 }

1086

1087 if proc.returncode != 0:

1088 return {

1089 "harness_pass": False,

1090 "baseline_pass": False,

1091 "confidence": 0.0,

1092 "reason": f"Judge CLI error: {proc.stderr.strip()[:200]}",

1093 "raw": {"verdict_a": "error", "verdict_b": "error"},

1094 "error": "api_error",

1095 }

1096

1097 if not proc.stdout.strip():

1098 return {

1099 "harness_pass": False,

1100 "baseline_pass": False,

1101 "confidence": 0.0,

1102 "reason": "Judge returned empty output",

1103 "raw": {"verdict_a": "error", "verdict_b": "error"},

1104 "error": "empty_output",

1105 }

1106

1107 try:

1108 stdout = proc.stdout.strip()

1109 try:

1110 envelope = json.loads(stdout)

1111 except json.JSONDecodeError:

1112 lines = [line for line in stdout.split("\n") if line.strip()]

1113 if not lines:

1114 raise

1115 envelope = json.loads(lines[-1])

1116

1117 if envelope.get("subtype") == "error_max_structured_output_retries":

1118 return {

1119 "harness_pass": False,

1120 "baseline_pass": False,

1121 "confidence": 0.0,

1122 "reason": "Judge could not produce valid structured output after retries",

1123 "raw": {"verdict_a": "error", "verdict_b": "error"},

1124 "error": "retry_exhausted",

1125 }

1126

1127 if envelope.get("is_error", False):

1128 err_text = str(envelope.get("result", "") or "")[:200]

1129 return {

1130 "harness_pass": False,

1131 "baseline_pass": False,

1132 "confidence": 0.0,

1133 "reason": f"Judge reported error: {err_text}",

1134 "raw": {"verdict_a": "error", "verdict_b": "error"},

1135 "error": "api_error",

1136 }

1137

1138 if isinstance(envelope.get("structured_output"), dict):

1139 result: dict[str, Any] = envelope["structured_output"]

1140 else:

1141 raw_result = envelope.get("result", "")

1142 if isinstance(raw_result, dict):

1143 result = raw_result

1144 elif raw_result:

1145 result = json.loads(raw_result)

1146 else:

1147 return {

1148 "harness_pass": False,

1149 "baseline_pass": False,

1150 "confidence": 0.0,

1151 "reason": "Empty result field in judge response",

1152 "raw": {"verdict_a": "error", "verdict_b": "error"},

1153 "error": "empty_result",

1154 }

1155 except (json.JSONDecodeError, TypeError, ValueError):

1156 return {

1157 "harness_pass": False,

1158 "baseline_pass": False,

1159 "confidence": 0.0,

1160 "reason": "Failed to parse judge response",

1161 "raw": {"verdict_a": "error", "verdict_b": "error"},

1162 "error": "parse_error",

1163 }

1164

1165 # De-anonymize

1166 verdict_a = str(result.get("verdict_a", "no"))

1167 verdict_b = str(result.get("verdict_b", "no"))

1168 confidence = float(result.get("confidence", 0.0))

1169 reason = str(result.get("reason", ""))

1170

1171 if harness_is_a:

1172 harness_pass = verdict_a == "yes"

1173 baseline_pass = verdict_b == "yes"

1174 else:

1175 harness_pass = verdict_b == "yes"

1176 baseline_pass = verdict_a == "yes"

1177

1178 return {

1179 "harness_pass": harness_pass,

1180 "baseline_pass": baseline_pass,

1181 "confidence": confidence,

1182 "reason": reason,

1183 "raw": {"verdict_a": verdict_a, "verdict_b": verdict_b, "harness_is_a": harness_is_a},

1184 }

1185

1186

1187 def evaluate_contract(

1188 config: EvaluateConfig,

1189 context: InterpolationContext,

1190 model: str = DEFAULT_LLM_MODEL,

1191 timeout: int = 1800,

1192) -> EvaluationResult:

1193 """Evaluate producer/consumer contract alignment using an LLM judge.

1194

1195 Reads each producer/consumer file pair, applies optional regex extraction,

1196 then asks an LLM judge whether the producer satisfies the consumer contract.

1197 Returns yes only when all pairs align; any failure routes no/error.

1198

1199 Args:

1200 config: EvaluateConfig with type="contract" and pairs list

1201 context: Interpolation context (unused by this evaluator directly)

1202 model: LLM model identifier

1203 timeout: Subprocess timeout in seconds

1204

1205 Returns:

1206 EvaluationResult with verdict yes/no/error and pair_results in details

1207 """

1208 pairs = config.pairs

1209 if not pairs:

1210 return EvaluationResult(

1211 verdict="error",

1212 details={"error": "contract evaluator requires at least one pair in evaluate.pairs"},

1213 )

1214

1215 contract_schema = {

1216 "type": "object",

1217 "properties": {

1218 "verdict": {"type": "string", "enum": ["yes", "no"]},

1219 "confidence": {"type": "number"},

1220 "reason": {"type": "string"},

1221 },

1222 "required": ["verdict", "confidence", "reason"],

1223 }

1224

1225 pair_results: list[dict[str, Any]] = []

1226

1227 for pair in pairs:

1228 producer_path = pair.get("producer", "")

1229 consumer_path = pair.get("consumer", "")

1230 producer_pattern = pair.get("producer_pattern")

1231 consumer_pattern = pair.get("consumer_pattern")

1232 contract_rule = pair.get("contract", "the producer and consumer must be compatible")

1233

1234 # Read producer file

1235 try:

1236 producer_content = Path(producer_path).read_text()

1237 except OSError as e:

1238 pair_results.append(

1239 {

1240 "producer": producer_path,

1241 "consumer": consumer_path,

1242 "verdict": "error",

1243 "error": f"cannot read producer file: {e}",

1244 }

1245 )

1246 continue

1247

1248 # Read consumer file

1249 try:

1250 consumer_content = Path(consumer_path).read_text()

1251 except OSError as e:

1252 pair_results.append(

1253 {

1254 "producer": producer_path,

1255 "consumer": consumer_path,

1256 "verdict": "error",

1257 "error": f"cannot read consumer file: {e}",

1258 }

1259 )

1260 continue

1261

1262 # Apply optional regex extraction

1263 if producer_pattern:

1264 matches = re.findall(producer_pattern, producer_content, re.DOTALL)

1265 if not matches:

1266 pair_results.append(

1267 {

1268 "producer": producer_path,

1269 "consumer": consumer_path,

1270 "verdict": "error",

1271 "error": f"producer_pattern matched nothing in {producer_path}",

1272 }

1273 )

1274 continue

1275 producer_slice = "\n".join(matches)

1276 else:

1277 producer_slice = (

1278 producer_content[-4000:] if len(producer_content) > 4000 else producer_content

1279 )

1280

1281 if consumer_pattern:

1282 matches = re.findall(consumer_pattern, consumer_content, re.DOTALL)

1283 if not matches:

1284 pair_results.append(

1285 {

1286 "producer": producer_path,

1287 "consumer": consumer_path,

1288 "verdict": "error",

1289 "error": f"consumer_pattern matched nothing in {consumer_path}",

1290 }

1291 )

1292 continue

1293 consumer_slice = "\n".join(matches)

1294 else:

1295 consumer_slice = (

1296 consumer_content[-4000:] if len(consumer_content) > 4000 else consumer_content

1297 )

1298

1299 judge_prompt = (

1300 f"You are evaluating whether a producer output satisfies a consumer contract.\n\n"

1301 f"Contract rule: {contract_rule}\n\n"

1302 f'<producer path="{producer_path}">\n{producer_slice}\n</producer>\n\n'

1303 f'<consumer path="{consumer_path}">\n{consumer_slice}\n</consumer>\n\n'

1304 "Does the producer satisfy the consumer contract? "

1305 "Consider field names, types, casing, and structure. "

1306 "Answer yes if aligned, no if mismatched."

1307 )

1308

1309 invocation = resolve_host().build_blocking_json(prompt=judge_prompt, model=model)

1310 args = list(invocation.args) + [

1311 "--json-schema",

1312 json.dumps(contract_schema),

1313 "--no-session-persistence",

1314 ]

1315

1316 t0 = time.monotonic()

1317 try:

1318 proc = subprocess.run(

1319 [invocation.binary, *args], capture_output=True, text=True, timeout=timeout

1320 )

1321 except subprocess.TimeoutExpired:

1322 pair_results.append(

1323 {

1324 "producer": producer_path,

1325 "consumer": consumer_path,

1326 "verdict": "error",

1327 "error": "LLM judge timed out",

1328 "llm_latency_ms": int((time.monotonic() - t0) * 1000),

1329 }

1330 )

1331 continue

1332 except FileNotFoundError:

1333 return EvaluationResult(

1334 verdict="error",

1335 details={

1336 "error": f"{invocation.binary} CLI not found. Install the active host CLI (see LL_HOST_CLI).",

1337 "missing_dependency": True,

1338 },

1339 )

1340 llm_latency_ms = int((time.monotonic() - t0) * 1000)

1341

1342 if proc.returncode != 0:

1343 pair_results.append(

1344 {

1345 "producer": producer_path,

1346 "consumer": consumer_path,

1347 "verdict": "error",

1348 "error": f"CLI error: {proc.stderr.strip()}",

1349 "llm_latency_ms": llm_latency_ms,

1350 }

1351 )

1352 continue

1353

1354 if not proc.stdout.strip():

1355 pair_results.append(

1356 {

1357 "producer": producer_path,

1358 "consumer": consumer_path,

1359 "verdict": "error",

1360 "error": "CLI returned empty output",

1361 "llm_latency_ms": llm_latency_ms,

1362 }

1363 )

1364 continue

1365

1366 try:

1367 stdout = proc.stdout.strip()

1368 try:

1369 envelope = json.loads(stdout)

1370 except json.JSONDecodeError:

1371 lines = [line for line in stdout.split("\n") if line.strip()]

1372 if not lines:

1373 raise

1374 envelope = json.loads(lines[-1])

1375

1376 if envelope.get("subtype") == "error_max_structured_output_retries":

1377 pair_results.append(

1378 {

1379 "producer": producer_path,

1380 "consumer": consumer_path,

1381 "verdict": "error",

1382 "error": "Claude CLI could not produce valid structured output after retries",

1383 "llm_latency_ms": llm_latency_ms,

1384 }

1385 )

1386 continue

1387

1388 if envelope.get("is_error", False):

1389 err_text = str(envelope.get("result", "") or "")[:200]

1390 pair_results.append(

1391 {

1392 "producer": producer_path,

1393 "consumer": consumer_path,

1394 "verdict": "error",

1395 "error": f"Claude CLI reported error: {err_text}",

1396 "llm_latency_ms": llm_latency_ms,

1397 }

1398 )

1399 continue

1400

1401 if isinstance(envelope.get("structured_output"), dict):

1402 llm_result: dict[str, Any] = envelope["structured_output"]

1403 else:

1404 raw_result = envelope.get("result", "")

1405 if isinstance(raw_result, dict):

1406 llm_result = raw_result

1407 elif raw_result:

1408 llm_result = json.loads(raw_result)

1409 elif "verdict" in envelope:

1410 llm_result = envelope

1411 else:

1412 pair_results.append(

1413 {

1414 "producer": producer_path,

1415 "consumer": consumer_path,

1416 "verdict": "error",

1417 "error": "empty result field in CLI response",

1418 "llm_latency_ms": llm_latency_ms,

1419 }

1420 )

1421 continue

1422

1423 except (json.JSONDecodeError, TypeError, ValueError) as e:

1424 pair_results.append(

1425 {

1426 "producer": producer_path,

1427 "consumer": consumer_path,

1428 "verdict": "error",

1429 "error": f"failed to parse LLM response: {e}",

1430 "llm_latency_ms": llm_latency_ms,

1431 }

1432 )

1433 continue

1434

1435 pair_results.append(

1436 {

1437 "producer": producer_path,

1438 "consumer": consumer_path,

1439 "verdict": str(llm_result.get("verdict", "error")),

1440 "confidence": float(llm_result.get("confidence", 1.0)),

1441 "reason": llm_result.get("reason", ""),

1442 "llm_latency_ms": llm_latency_ms,

1443 }

1444 )

1445

1446 # Aggregate: yes only if all pairs aligned; error takes precedence over no

1447 if any(p["verdict"] == "error" for p in pair_results):

1448 overall = "error"

1449 elif all(p["verdict"] == "yes" for p in pair_results):

1450 overall = "yes"

1451 else:

1452 overall = "no"

1453

1454 return EvaluationResult(

1455 verdict=overall,

1456 details={"pair_results": pair_results},

1457 )

1458

1459

1460 def evaluate_comparator(

1461 config: EvaluateConfig,

1462 output: str,

1463 context: InterpolationContext,

1464) -> EvaluationResult:

1465 """Evaluate using blind A/B comparison against a stored baseline."""

1466 from pathlib import Path

1467

1468 if config.baseline_path is None:

1469 return EvaluationResult(

1470 verdict="no_baseline",

1471 details={"reason": "No baseline_path configured"},

1472 )

1473

1474 baseline_file = Path(config.baseline_path) / "output.txt"

1475 if not baseline_file.exists():

1476 if config.auto_promote:

1477 baseline_file.parent.mkdir(parents=True, exist_ok=True)

1478 baseline_file.write_text(output)

1479 return EvaluationResult(

1480 verdict="yes",

1481 details={

1482 "reason": "No baseline found; current output promoted as new baseline.",

1483 "bootstrapped": True,

1484 },

1485 )

1486 return EvaluationResult(

1487 verdict="no_baseline",

1488 details={"reason": f"Baseline file not found: {baseline_file}"},

1489 )

1490

1491 baseline_text = baseline_file.read_text()

1492 min_pairs = max(1, config.min_pairs if config.min_pairs is not None else 1)

1493 harness_wins = 0

1494 baseline_wins = 0

1495 last_reason = ""

1496 last_raw: dict[str, Any] = {}

1497

1498 for _ in range(min_pairs):

1499 result = evaluate_blind_comparator(output, baseline_text, prompt=config.prompt)

1500 if result.get("harness_pass"):

1501 harness_wins += 1

1502 if result.get("baseline_pass"):

1503 baseline_wins += 1

1504 last_reason = result.get("reason", "")

1505 last_raw = result.get("raw", {})

1506

1507 if harness_wins > baseline_wins:

1508 verdict = "yes"

1509 elif baseline_wins > harness_wins:

1510 verdict = "no"

1511 else:

1512 verdict = "tie"

1513

1514 if config.auto_promote and verdict == "yes":

1515 baseline_file.write_text(output)

1516

1517 return EvaluationResult(

1518 verdict=verdict,

1519 details={

1520 "harness_wins": harness_wins,

1521 "baseline_wins": baseline_wins,

1522 "min_pairs": min_pairs,

1523 "reason": last_reason,

1524 "raw": last_raw,

1525 },

1526 )

1527

1528

1529 def evaluate(

1530 config: EvaluateConfig,

1531 output: str,

1532 exit_code: int,

1533 context: InterpolationContext,

1534) -> EvaluationResult:

1535 """Dispatch to appropriate evaluator based on config type.

1536

1537 Args:

1538 config: Evaluator configuration with type and parameters

1539 output: Action stdout

1540 exit_code: Action exit code

1541 context: Runtime context for variable interpolation

1542

1543 Returns:

1544 EvaluationResult from the appropriate evaluator

1545

1546 Raises:

1547 ValueError: If evaluator type is unknown

1548 """

1549 eval_type = config.type

1550

1551 # BUG-1640: Action-level timeouts (exit_code=124) short-circuit to "error"

1552 # so loop authors' on_error: branches fire instead of being routed via

1553 # on_no: based on truncated output. mcp_result is exempted because it has

1554 # its own established "timeout" verdict (see evaluate_mcp_result).

1555 if exit_code == 124 and eval_type != "mcp_result":

1556 return EvaluationResult(

1557 verdict="error",

1558 details={"exit_code": exit_code, "error": "action timed out"},

1559 )

1560

1561 # BUG-1815: Non-timeout non-zero exit codes short-circuit to "error" for

1562 # evaluator types that don't intrinsically check exit codes. Exit-code-aware

1563 # evaluators (exit_code, mcp_result, harbor_scorer, diff_stall, llm_structured)

1564 # are exempt because they handle exit codes via their own logic.

1565 _EXIT_CODE_AWARE_EVALUATORS: frozenset[str] = frozenset(

1566 {

1567 "exit_code",

1568 "mcp_result",

1569 "harbor_scorer",

1570 "diff_stall",

1571 "action_stall",

1572 "llm_structured",

1573 "contract",

1574 }

1575 )

1576 if exit_code != 0 and eval_type not in _EXIT_CODE_AWARE_EVALUATORS:

1577 return EvaluationResult(

1578 verdict="error",

1579 details={

1580 "exit_code": exit_code,

1581 "error": f"action exited with code {exit_code}",

1582 },

1583 )

1584

1585 if eval_type == "exit_code":

1586 return evaluate_exit_code(exit_code)

1587

1588 elif eval_type == "output_numeric":

1589 if config.target is None:

1590 raise ValueError("output_numeric evaluator requires 'target' to be set")

1591 elif isinstance(config.target, str):

1592 try:

1593 resolved = interpolate(config.target, context) if context else config.target

1594 numeric_target = float(resolved)

1595 except (InterpolationError, ValueError) as e:

1596 raise ValueError(

1597 f"output_numeric target must be numeric, got: {config.target!r}"

1598 ) from e

1599 else:

1600 numeric_target = float(config.target)

1601 return evaluate_output_numeric(

1602 output=output,

1603 operator=config.operator or "eq",

1604 target=numeric_target,

1605 )

1606

1607 elif eval_type == "output_json":

1608 return evaluate_output_json(

1609 output=output,

1610 path=config.path or "",

1611 operator=config.operator or "eq",

1612 target=config.target,

1613 )

1614

1615 elif eval_type == "output_contains":

1616 return evaluate_output_contains(

1617 output=output,

1618 pattern=config.pattern or "",

1619 negate=config.negate,

1620 error_patterns=config.error_patterns,

1621 )

1622

1623 elif eval_type == "convergence":

1624 # Resolve previous value from interpolation if configured

1625 previous: float | None = None

1626 if config.previous:

1627 try:

1628 previous = float(interpolate(config.previous, context))

1629 except (InterpolationError, ValueError):

1630 # Previous unavailable on first iteration, continue with None

1631 pass

1632

1633 # Parse current value from output

1634 try:

1635 current = float(output.strip())

1636 except ValueError:

1637 return EvaluationResult(

1638 verdict="error",

1639 details={"error": f"Cannot parse output as number: {output[:100]}"},

1640 )

1641

1642 # Resolve target (may be interpolated string like "${context.target}")

1643 convergence_target: float

1644 if isinstance(config.target, str):

1645 try:

1646 convergence_target = float(interpolate(config.target, context))

1647 except (InterpolationError, ValueError) as e:

1648 return EvaluationResult(

1649 verdict="error",

1650 details={"error": f"Cannot resolve target: {e}"},

1651 )

1652 else:

1653 if config.target is None:

1654 raise ValueError("convergence evaluator requires 'target' to be set")

1655 convergence_target = float(config.target)

1656

1657 # Resolve tolerance (may be interpolated string)

1658 tolerance: float = 0.0

1659 if config.tolerance is not None:

1660 if isinstance(config.tolerance, str):

1661 try:

1662 tolerance = float(interpolate(config.tolerance, context))

1663 except (InterpolationError, ValueError):

1664 tolerance = 0.0

1665 else:

1666 tolerance = float(config.tolerance)

1667

1668 return evaluate_convergence(

1669 current=current,

1670 previous=previous,

1671 target=convergence_target,

1672 tolerance=tolerance,

1673 direction=config.direction,

1674 )

1675

1676 elif eval_type == "diff_stall":

1677 return evaluate_diff_stall(

1678 scope=config.scope,

1679 max_stall=config.max_stall,

1680 )

1681

1682 elif eval_type == "action_stall":

1683 return evaluate_action_stall(

1684 track=config.track,

1685 max_repeat=config.max_repeat,

1686 context=context,

1687 )

1688

1689 elif eval_type == "llm_structured":

1690 prompt = config.prompt

1691 if prompt and context:

1692 try:

1693 prompt = interpolate(prompt, context)

1694 except InterpolationError:

1695 pass # Use raw prompt on resolution failure

1696 return evaluate_llm_structured(

1697 output=output,

1698 prompt=prompt,

1699 schema=config.schema,

1700 min_confidence=config.min_confidence,

1701 uncertain_suffix=config.uncertain_suffix,

1702 )

1703

1704 elif eval_type == "mcp_result":

1705 return evaluate_mcp_result(output=output, exit_code=exit_code)

1706

1707 elif eval_type == "harbor_scorer":

1708 return evaluate_harbor_scorer(output=output, exit_code=exit_code)

1709

1710 elif eval_type == "comparator":

1711 return evaluate_comparator(config=config, output=output, context=context)

1712

1713 elif eval_type == "contract":

1714 return evaluate_contract(config=config, context=context)

1715

1716 elif eval_type == "classify":

1717 return evaluate_classify(output=output, line=config.line)

1718

1719 else:

1720 raise ValueError(f"Unknown evaluator type: {eval_type}")