1"""FSM loop validation logic.

2

3 This module provides validation for FSM loop definitions, ensuring

4 structural correctness and catching common configuration errors.

5

6 Validation checks:

7- Initial state exists in states dict

8- All referenced states exist

9- At least one terminal state

10- Evaluator configs have required fields for their type

11- No conflicting routing (shorthand vs full route)

12- Numeric fields in valid ranges (max_iterations > 0, backoff >= 0, timeout > 0)

13"""

14

15 from __future__ import annotations

16

17 import logging

18 import re

19 from collections import deque

20 from dataclasses import dataclass

21 from enum import Enum

22 from pathlib import Path

23 from typing import Any

24

25 import yaml

26

27 from little_loops.fsm.evaluators import _NUMERIC_OPERATORS

28 from little_loops.fsm.fragments import resolve_flow, resolve_fragments, resolve_inheritance

29 from little_loops.fsm.schema import EvaluateConfig, FSMLoop, ParameterSpec, StateConfig

30

31 logger = logging.getLogger(__name__)

32

33

34 class ValidationSeverity(Enum):

35 """Severity level for validation issues."""

36

37 ERROR = "error"

38 WARNING = "warning"

39

40

41@dataclass

42 class ValidationError:

43 """Structured validation error.

44

45 Attributes:

46 message: Human-readable error description

47 path: Path to the problematic element (e.g., "states.check.route")

48 severity: Error severity (error or warning)

49 """

50

51 message: str

52 path: str | None = None

53 severity: ValidationSeverity = ValidationSeverity.ERROR

54

55 def __str__(self) -> str:

56 """Format error for display."""

57 prefix = f"[{self.severity.value.upper()}]"

58 if self.path:

59 return f"{prefix} {self.path}: {self.message}"

60 return f"{prefix} {self.message}"

61

62

63# Evaluator type to required fields mapping

64 EVALUATOR_REQUIRED_FIELDS: dict[str, list[str]] = {

65 "exit_code": [],

66 "output_numeric": ["operator", "target"],

67 "output_json": ["path", "operator", "target"],

68 "output_contains": ["pattern"],

69 "convergence": ["target"],

70 "diff_stall": [],

71 "action_stall": [],

72 "llm_structured": [],

73 "mcp_result": [],

74 "harbor_scorer": [],

75 "comparator": ["baseline_path"],

76 "contract": ["pairs"],

77 "classify": [],

78}

79

80# Non-LLM evaluator types: all evaluator types except llm_structured

81# Derived from EVALUATOR_REQUIRED_FIELDS so new types are automatically included

82 NON_LLM_EVALUATOR_TYPES: frozenset[str] = frozenset(EVALUATOR_REQUIRED_FIELDS.keys()) - {

83 "llm_structured",

84 "comparator",

85 "contract",

86}

87

88# Meta-loop detector: action string patterns that indicate harness artifact writes

89_META_LOOP_ACTION_PATTERNS: tuple[re.Pattern[str], ...] = (

90 re.compile(r"loops/[\w-]+\.yaml"),

91 re.compile(r"skills/[\w-]+/SKILL\.md"),

92 re.compile(r"agents/[\w-]+\.md"),

93 re.compile(r"commands/[\w-]+\.md"),

94 re.compile(r"\.claude/(CLAUDE\.md|settings)"),

95)

96

97# Action string tokens that indicate meta-loop behavior

98_META_LOOP_ACTION_TOKENS: frozenset[str] = frozenset({"yaml_state_editor", "replace_action"})

99

100# Import paths that identify a loop as a meta-loop (harness optimization framework)

101_META_LOOP_IMPORT_TRIGGERS: frozenset[str] = frozenset({"lib/benchmark.yaml"})

102

103# MR-3: shared-tmp path detector. The runner injects ${context.run_dir} resolving

104# to .loops/runs/<loop>-<timestamp>/; loops that hardcode .loops/tmp/ instead

105# cause state corruption under concurrent runs (ll-parallel, retries, etc.).

106_SHARED_TMP_PATH_RE = re.compile(r"\.loops/tmp/[\w./-]+")

107

108# MR-7: bash-default interpolation detector. Matches ${namespace.path:-default}

109# (unescaped bash `:- ` default form) that the FSM interpolator does not support.

110# Negative lookbehind exempts the legitimate escaped form $${VAR:-value}.

111_BASH_DEFAULT_RE = re.compile(r"(?<!\$)\$\{[^}]*:-[^}]*\}")

112

113# MR-9: over-escaped shell `$$` detector. The FSM interpolator only rewrites the

114# brace form `$${...}` → `${...}`; bare `$(...)` and `$VAR` are passed to the shell

115# untouched (interpolation.py). Doubling them — `$$(` or `$$VAR` — is NOT an escape:

116# the runner's `bash -c` expands the leading `$$` to the PID, so `$$(pwd)/$$DIR`

117# becomes `<pid>(pwd)/<pid>DIR`. The lookahead matches `$$(` (command substitution)

118# and `$$<identifier-start>` (variable) while exempting the legitimate `$${...}`

119# brace escape (next char `{`) and a standalone PID `$$` (followed by `/`, `.`, space,

120# or end). See ENH for the interactive-/html-/svg-generator over-escape bug.

121_OVERESCAPED_SHELL_RE = re.compile(r"\$\$(?=\(|[A-Za-z_])")

122

123# ENH-1961: Regex for extracting captured variable names from ${captured.<var>.*} references

124_CAPTURED_REF_RE = re.compile(r"\$\{captured\.(\w+)")

125

126# Full-reference form, capturing the var name and the remainder up to the closing

127# brace so we can detect a `:default=` guard. A reference written as

128# `${captured.x.output:default=...}` is provably safe even on paths that bypass

129# the capturing state — the interpolation engine (interpolation.py) substitutes

130# the default when the path is missing — so it must NOT be flagged by the

131# capture-reachability check. _CAPTURED_REF_RE alone can't see the guard.

132_CAPTURED_REF_FULL_RE = re.compile(r"\$\{captured\.(\w+)([^}]*)\}")

133

134

135 def _unguarded_captured_refs(text: str) -> set[str]:

136 """Return captured var names that have at least one reference WITHOUT a

137 `:default=` guard. Vars referenced only via `${captured.x...:default=...}`

138 are omitted: the default makes a missing value safe, so they should not

139 trigger missing-capture or bypass-path diagnostics.

140 """

141 refs: set[str] = set()

142 for var_name, remainder in _CAPTURED_REF_FULL_RE.findall(text):

143 if ":default=" not in remainder:

144 refs.add(var_name)

145 return refs

146

147

148# ENH-1819: Regex patterns for detecting multimodal evaluation in prompt actions

149_MULTIMODAL_EVAL_PATTERNS: tuple[re.Pattern[str], ...] = (

150 re.compile(r"Read the screenshot", re.IGNORECASE),

151 re.compile(r"view the (generated )?(website|page|image)", re.IGNORECASE),

152 re.compile(r"screenshot\.(png|jpg|jpeg|webp)"),

153 re.compile(r"\.(png|jpg|jpeg|webp)\b.*\b(read|view|evaluate|score|judge)", re.IGNORECASE),

154)

155

156# Valid comparison operators

157 VALID_OPERATORS = {"eq", "ne", "lt", "le", "gt", "ge"}

158

159# Valid values for the top-level `visibility:` field (audience axis for

160# `ll-loop list` filtering). "public" is the default when the field is absent.

161 VALID_VISIBILITY: frozenset[str] = frozenset({"public", "internal", "example"})

162

163# All top-level keys recognized by FSMLoop.from_dict()

164 KNOWN_TOP_LEVEL_KEYS: frozenset[str] = frozenset(

165 {

166 "name",

167 "description",

168 "initial",

169 "states",

170 "context",

171 "parameters",

172 "scope",

173 "max_steps",

174 "on_max_steps",

175 "max_iterations",

176 "on_max_iterations",

177 "max_edge_revisits",

178 "backoff",

179 "timeout",

180 "default_timeout",

181 "maintain",

182 "llm",

183 "on_handoff",

184 "input_key",

185 "required_inputs",

186 "config",

187 "category",

188 "labels",

189 "visibility",

190 "commands",

191 "targets",

192 "circuit",

193 "meta_self_eval_ok",

194 "shared_state_ok",

195 "partial_route_ok",

196 "artifact_versioning",

197 "artifact_versioning_ok",

198 "generator_fix_ok",

199 "bash_default_ok",

200 "shell_pid_ok",

201 "import",

202 "fragments",

203 "from",

204 "flow",

205 "state_defs",

206 }

207)

208

209# Special tokens accepted as the `on_repeated_failure` target on the

210# stall detector — `"abort"` means terminate via _finish("stall_detected").

211 STALL_SPECIAL_TOKENS: frozenset[str] = frozenset({"abort"})

212

213# Valid parameter types for the 'parameters:' block

214 VALID_PARAMETER_TYPES: frozenset[str] = frozenset(

215 {"string", "integer", "number", "boolean", "enum", "path"}

216)

217

218

219 def _validate_evaluator(state_name: str, evaluate: EvaluateConfig) -> list[ValidationError]:

220 """Validate evaluator configuration for type-specific requirements.

221

222 Args:

223 state_name: Name of the state containing this evaluator

224 evaluate: The evaluator configuration to validate

225

226 Returns:

227 List of validation errors found

228 """

229 errors: list[ValidationError] = []

230 path = f"states.{state_name}.evaluate"

231

232 # Check that evaluator type is recognized

233 valid_types = set(EVALUATOR_REQUIRED_FIELDS.keys())

234 if evaluate.type not in valid_types:

235 errors.append(

236 ValidationError(

237 message=f"Unknown evaluator type '{evaluate.type}'. "

238 f"Must be one of: {', '.join(sorted(valid_types))}",

239 path=path,

240 )

241 )

242 return errors # Can't check required fields for unknown type

243

244 # Check required fields for evaluator type

245 required = EVALUATOR_REQUIRED_FIELDS.get(evaluate.type, [])

246 for field_name in required:

247 value = getattr(evaluate, field_name, None)

248 if value is None:

249 errors.append(

250 ValidationError(

251 message=f"Evaluator type '{evaluate.type}' requires '{field_name}' field",

252 path=path,

253 )

254 )

255

256 # Validate operator if present

257 if evaluate.operator is not None and evaluate.operator not in VALID_OPERATORS:

258 errors.append(

259 ValidationError(

260 message=f"Invalid operator '{evaluate.operator}'. "

261 f"Must be one of: {', '.join(sorted(VALID_OPERATORS))}",

262 path=f"{path}.operator",

263 )

264 )

265

266 # Validate convergence-specific fields

267 if evaluate.type == "convergence":

268 if evaluate.direction not in ("minimize", "maximize"):

269 errors.append(

270 ValidationError(

271 message=f"Invalid direction '{evaluate.direction}'. "

272 "Must be 'minimize' or 'maximize'",

273 path=f"{path}.direction",

274 )

275 )

276 # Only validate tolerance if it's a numeric value (not an interpolation string)

277 if (

278 evaluate.tolerance is not None

279 and isinstance(evaluate.tolerance, (int, float))

280 and evaluate.tolerance < 0

281 ):

282 errors.append(

283 ValidationError(

284 message="Tolerance cannot be negative",

285 path=f"{path}.tolerance",

286 )

287 )

288

289 # Validate llm_structured-specific fields

290 if evaluate.type == "llm_structured":

291 if evaluate.min_confidence < 0 or evaluate.min_confidence > 1:

292 errors.append(

293 ValidationError(

294 message="min_confidence must be between 0 and 1",

295 path=f"{path}.min_confidence",

296 )

297 )

298

299 # Validate diff_stall-specific fields

300 if evaluate.type == "diff_stall":

301 if evaluate.max_stall < 1:

302 errors.append(

303 ValidationError(

304 message="max_stall must be >= 1",

305 path=f"{path}.max_stall",

306 )

307 )

308

309 # Validate action_stall-specific fields

310 if evaluate.type == "action_stall":

311 if evaluate.max_repeat < 1:

312 errors.append(

313 ValidationError(

314 message="max_repeat must be >= 1",

315 path=f"{path}.max_repeat",

316 )

317 )

318

319 return errors

320

321

322 def _validate_parameters(fsm: FSMLoop) -> list[ValidationError]:

323 """Validate the loop's top-level parameters: block.

324

325 Args:

326 fsm: The FSM loop to validate

327

328 Returns:

329 List of validation errors found

330 """

331 errors: list[ValidationError] = []

332

333 for param_name, param_spec in fsm.parameters.items():

334 path = f"parameters.{param_name}"

335

336 if param_spec.type not in VALID_PARAMETER_TYPES:

337 errors.append(

338 ValidationError(

339 message=(

340 f"Unknown parameter type '{param_spec.type}'. "

341 f"Must be one of: {', '.join(sorted(VALID_PARAMETER_TYPES))}"

342 ),

343 path=path,

344 )

345 )

346

347 if param_spec.type == "enum" and not param_spec.values:

348 errors.append(

349 ValidationError(

350 message="Parameter type 'enum' requires a 'values' list",

351 path=path,

352 )

353 )

354

355 if param_spec.required and param_spec.default is not None:

356 errors.append(

357 ValidationError(

358 message="Parameter cannot be both 'required: true' and have a 'default' value",

359 path=path,

360 )

361 )

362

363 return errors

364

365

366 def _check_param_type(value: Any, spec: ParameterSpec) -> str | None:

367 """Return an error message if value does not match spec.type, else None."""

368 if spec.type == "string" and not isinstance(value, str):

369 return f"expected string, got {type(value).__name__}"

370 if spec.type == "integer" and not isinstance(value, int):

371 return f"expected integer, got {type(value).__name__}"

372 if spec.type == "number" and not isinstance(value, (int, float)):

373 return f"expected number, got {type(value).__name__}"

374 if spec.type == "boolean" and not isinstance(value, bool):

375 return f"expected boolean, got {type(value).__name__}"

376 if spec.type == "enum" and spec.values and value not in spec.values:

377 return f"expected one of {spec.values!r}, got {value!r}"

378 return None

379

380

381 def _validate_with_bindings(fsm: FSMLoop, loop_dir: Path) -> list[ValidationError]:

382 """Validate with: bindings against child loop parameter contracts.

383

384 Called from load_and_validate (not validate_fsm) because resolving child loops

385 requires file-system access via the loop directory path.

386

387 Args:

388 fsm: The parent FSM loop

389 loop_dir: Directory to resolve child loop paths from

390

391 Returns:

392 List of validation errors found

393 """

394 errors: list[ValidationError] = []

395

396 for state_name, state in fsm.states.items():

397 if state.loop is None or not state.with_:

398 continue

399

400 # Try to resolve and load the child loop; skip if unavailable

401 try:

402 from little_loops.cli.loop._helpers import resolve_loop_path

403

404 loop_path = resolve_loop_path(state.loop, loop_dir)

405 child_fsm, _ = load_and_validate(loop_path)

406 except Exception:

407 continue

408

409 if not child_fsm.parameters:

410 continue # Child has no declared contract — nothing to cross-validate

411

412 path = f"states.{state_name}"

413

414 # Unknown with: keys (not declared by child)

415 for key in state.with_:

416 if key not in child_fsm.parameters:

417 errors.append(

418 ValidationError(

419 message=(

420 f"'with.{key}' is not a declared parameter of loop '{state.loop}'. "

421 f"Declared: {', '.join(sorted(child_fsm.parameters))}"

422 ),

423 path=f"{path}.with.{key}",

424 )

425 )

426

427 # Required parameters not bound

428 for param_name, param_spec in child_fsm.parameters.items():

429 if param_spec.required and param_name not in state.with_:

430 errors.append(

431 ValidationError(

432 message=(

433 f"Required parameter '{param_name}' of loop '{state.loop}' "

434 f"is not bound in 'with'"

435 ),

436 path=f"{path}.with",

437 )

438 )

439

440 # Statically-detectable type mismatches (skip interpolation strings)

441 for param_name, value in state.with_.items():

442 if param_name not in child_fsm.parameters:

443 continue

444 if isinstance(value, str) and "${" in value:

445 continue

446 type_error = _check_param_type(value, child_fsm.parameters[param_name])

447 if type_error:

448 errors.append(

449 ValidationError(

450 message=f"Parameter '{param_name}': {type_error}",

451 path=f"{path}.with.{param_name}",

452 )

453 )

454

455 return errors

456

457

458 def _validate_loop_references(fsm: FSMLoop, loop_dir: Path) -> list[ValidationError]:

459 """Validate that every state's loop: reference resolves to an actual loop file.

460

461 Called from load_and_validate (not validate_fsm) because resolving child loops

462 requires file-system access via the loop directory path.

463 """

464 errors: list[ValidationError] = []

465 for state_name, state in fsm.states.items():

466 if state.loop is None:

467 continue

468 # Skip dynamically interpolated loop names — they can only be checked at runtime

469 if "${" in state.loop:

470 continue

471 try:

472 from little_loops.cli.loop._helpers import resolve_loop_path

473

474 resolve_loop_path(state.loop, loop_dir)

475 except FileNotFoundError:

476 errors.append(

477 ValidationError(

478 message=f"Loop reference '{state.loop}' does not resolve to any file.",

479 path=f"states.{state_name}.loop",

480 severity=ValidationSeverity.WARNING,

481 )

482 )

483 return errors

484

485

486 def _validate_fragment_bindings(fsm: FSMLoop, loop_dir: Path) -> list[ValidationError]:

487 """Validate fragment with: bindings against fragment parameter contracts.

488

489 Called from load_and_validate (not validate_fsm) because fragment parameters

490 are populated by resolve_fragments which runs before dataclass parsing.

491

492 Args:

493 fsm: The FSM loop to validate

494 loop_dir: Directory containing the loop file (unused; kept for API symmetry with

495 _validate_with_bindings)

496

497 Returns:

498 List of validation errors found

499 """

500 # Runner-injected vars available at runtime but not at static analysis time

501 RUNNER_INJECTED = {"run_dir", "loop_name", "started_at", "input_hash"}

502

503 errors: list[ValidationError] = []

504

505 for state_name, state in fsm.states.items():

506 if not state.fragment_parameters:

507 continue # No declared contract — nothing to cross-validate

508

509 path = f"states.{state_name}"

510

511 # Unknown with: keys (not declared by fragment)

512 for key in state.fragment_bindings:

513 if key not in state.fragment_parameters:

514 errors.append(

515 ValidationError(

516 message=(

517 f"'with.{key}' is not a declared parameter of fragment "

518 f"'{state.fragment_name}'. "

519 f"Declared: {', '.join(sorted(state.fragment_parameters))}"

520 ),

521 path=f"{path}.with.{key}",

522 )

523 )

524

525 # Required parameters not bound (whitelist runner-injected vars)

526 for param_name, param_spec in state.fragment_parameters.items():

527 if param_spec.required and param_name not in state.fragment_bindings:

528 if param_name in RUNNER_INJECTED:

529 continue # Available at runtime; not a static error

530 errors.append(

531 ValidationError(

532 message=(

533 f"Required parameter '{param_name}' of fragment "

534 f"'{state.fragment_name}' is not bound in 'with'"

535 ),

536 path=f"{path}.with",

537 )

538 )

539

540 # Statically-detectable type mismatches (skip interpolation strings)

541 for param_name, value in state.fragment_bindings.items():

542 if param_name not in state.fragment_parameters:

543 continue

544 if isinstance(value, str) and "${" in value:

545 continue

546 type_error = _check_param_type(value, state.fragment_parameters[param_name])

547 if type_error:

548 errors.append(

549 ValidationError(

550 message=f"Parameter '{param_name}': {type_error}",

551 path=f"{path}.with.{param_name}",

552 )

553 )

554

555 return errors

556

557

558 def _validate_state_action(state_name: str, state: StateConfig) -> list[ValidationError]:

559 """Validate state action configuration.

560

561 Args:

562 state_name: Name of the state to validate

563 state: The state configuration to validate

564

565 Returns:

566 List of validation errors found

567 """

568 errors: list[ValidationError] = []

569 path = f"states.{state_name}"

570

571 # append_to_messages must contain at least one ${...} interpolation expression

572 if state.append_to_messages is not None:

573 if "${" not in state.append_to_messages:

574 errors.append(

575 ValidationError(

576 message=(

577 "'append_to_messages' must contain a ${...} interpolation expression "

578 f"(e.g. '${{captured.{state_name}.output}}')"

579 ),

580 path=f"{path}.append_to_messages",

581 )

582 )

583

584 # model: override is silently ignored for non-prompt states (host CLI is not invoked)

585 if state.model is not None and state.action_type not in ("prompt", "slash_command", None):

586 errors.append(

587 ValidationError(

588 message="model: override is ignored for shell/mcp_tool/contract states",

589 path=f"{path}.model",

590 severity=ValidationSeverity.WARNING,

591 )

592 )

593

594 # params field is only valid for mcp_tool states

595 if state.params and state.action_type != "mcp_tool":

596 errors.append(

597 ValidationError(

598 message="'params' field is only valid when action_type is 'mcp_tool'",

599 path=f"{path}.params",

600 )

601 )

602

603 # loop and action are mutually exclusive

604 if state.loop is not None and state.action is not None:

605 errors.append(

606 ValidationError(

607 message="'loop' and 'action' are mutually exclusive — "

608 "a sub-loop state cannot also have an action",

609 path=f"{path}",

610 )

611 )

612

613 # with: requires loop: to be set

614 if state.with_ and state.loop is None:

615 errors.append(

616 ValidationError(

617 message="'with' is only valid when 'loop' is set",

618 path=f"{path}.with",

619 )

620 )

621

622 # FEAT-1283: type=learning requires a populated LearningConfig

623 if state.type == "learning" and state.learning is not None:

624 if not state.learning.targets and not state.learning.targets_csv:

625 errors.append(

626 ValidationError(

627 message="type=learning requires non-empty 'learning.targets' or 'learning.targets_csv'",

628 path=f"{path}.learning.targets",

629 )

630 )

631 if state.learning.max_retries < 0:

632 errors.append(

633 ValidationError(

634 message=(

635 f"learning.max_retries must be >= 0, got {state.learning.max_retries}"

636 ),

637 path=f"{path}.learning.max_retries",

638 )

639 )

640 if state.on_yes is None:

641 errors.append(

642 ValidationError(

643 message="type=learning requires 'on_yes' (target for all-proven)",

644 path=f"{path}.on_yes",

645 )

646 )

647 if state.on_blocked is None and state.on_no is None:

648 errors.append(

649 ValidationError(

650 message=(

651 "type=learning requires 'on_blocked' or 'on_no' "

652 "(target for refuted / retries_exhausted)"

653 ),

654 path=f"{path}",

655 )

656 )

657

658 # with: and context_passthrough are mutually exclusive

659 if state.with_ and state.context_passthrough:

660 errors.append(

661 ValidationError(

662 message=(

663 "'with' and 'context_passthrough' are mutually exclusive — "

664 "use 'with' for explicit parameter bindings or 'context_passthrough' "

665 "for legacy bulk passthrough, not both"

666 ),

667 path=f"{path}",

668 )

669 )

670

671 return errors

672

673

674 def _validate_state_routing(state_name: str, state: StateConfig) -> list[ValidationError]:

675 """Validate state routing configuration.

676

677 Checks for conflicting routing definitions (shorthand vs full route).

678

679 Args:

680 state_name: Name of the state to validate

681 state: The state configuration to validate

682

683 Returns:

684 List of validation errors/warnings found

685 """

686 errors: list[ValidationError] = []

687 path = f"states.{state_name}"

688

689 has_shorthand = (

690 state.on_yes is not None

691 or state.on_no is not None

692 or state.on_error is not None

693 or state.on_partial is not None

694 or state.on_blocked is not None

695 or bool(state.extra_routes)

696 )

697 has_route = state.route is not None

698

699 # Warn about conflicting definitions

700 if has_shorthand and has_route:

701 errors.append(

702 ValidationError(

703 message="Both shorthand routing (on_yes/on_no/on_error) "

704 "and full route table defined. Route table will take precedence.",

705 path=path,

706 severity=ValidationSeverity.WARNING,

707 )

708 )

709

710 # Check for no valid transition definition

711 has_next = state.next is not None

712 has_terminal = state.terminal

713 has_loop = state.loop is not None

714

715 if not has_shorthand and not has_route and not has_next and not has_terminal and not has_loop:

716 errors.append(

717 ValidationError(

718 message="State has no transition defined. Add routing, 'next', "

719 "or mark as 'terminal: true'",

720 path=path,

721 )

722 )

723

724 # Validate retry field pairing: max_retries requires on_retry_exhausted and vice versa

725 if state.max_retries is not None and state.on_retry_exhausted is None:

726 errors.append(

727 ValidationError(

728 message="'max_retries' requires 'on_retry_exhausted' to also be set",

729 path=path,

730 )

731 )

732 if state.on_retry_exhausted is not None and state.max_retries is None:

733 errors.append(

734 ValidationError(

735 message="'on_retry_exhausted' requires 'max_retries' to also be set",

736 path=path,

737 )

738 )

739 if state.max_retries is not None and state.max_retries < 1:

740 errors.append(

741 ValidationError(

742 message=f"'max_retries' must be >= 1, got {state.max_retries}",

743 path=path,

744 )

745 )

746

747 # Validate retryable_exit_codes: requires on_error; all codes must be positive ints

748 if state.retryable_exit_codes is not None:

749 if state.on_error is None:

750 errors.append(

751 ValidationError(

752 message="'retryable_exit_codes' requires 'on_error' to also be set",

753 path=path,

754 )

755 )

756 for code in state.retryable_exit_codes:

757 if not isinstance(code, int) or code < 1:

758 errors.append(

759 ValidationError(

760 message=(

761 f"'retryable_exit_codes' entries must be positive "

762 f"integers, got {code!r}"

763 ),

764 path=f"{path}.retryable_exit_codes",

765 )

766 )

767 break

768

769 # Validate rate-limit retry field pairing (mirrors max_retries/on_retry_exhausted)

770 if state.max_rate_limit_retries is not None and state.on_rate_limit_exhausted is None:

771 errors.append(

772 ValidationError(

773 message="'max_rate_limit_retries' requires 'on_rate_limit_exhausted' to also be set",

774 path=path,

775 )

776 )

777 if state.on_rate_limit_exhausted is not None and state.max_rate_limit_retries is None:

778 errors.append(

779 ValidationError(

780 message="'on_rate_limit_exhausted' requires 'max_rate_limit_retries' to also be set",

781 path=path,

782 )

783 )

784 if state.max_rate_limit_retries is not None and state.max_rate_limit_retries < 1:

785 errors.append(

786 ValidationError(

787 message=f"'max_rate_limit_retries' must be >= 1, got {state.max_rate_limit_retries}",

788 path=path,

789 )

790 )

791 if (

792 state.rate_limit_backoff_base_seconds is not None

793 and state.rate_limit_backoff_base_seconds < 1

794 ):

795 errors.append(

796 ValidationError(

797 message=(

798 f"'rate_limit_backoff_base_seconds' must be >= 1, "

799 f"got {state.rate_limit_backoff_base_seconds}"

800 ),

801 path=path,

802 )

803 )

804 if state.rate_limit_max_wait_seconds is not None and state.rate_limit_max_wait_seconds < 1:

805 errors.append(

806 ValidationError(

807 message=(

808 f"'rate_limit_max_wait_seconds' must be >= 1, "

809 f"got {state.rate_limit_max_wait_seconds}"

810 ),

811 path=path,

812 )

813 )

814 if state.rate_limit_long_wait_ladder is not None:

815 if len(state.rate_limit_long_wait_ladder) == 0:

816 errors.append(

817 ValidationError(

818 message="'rate_limit_long_wait_ladder' must be non-empty if specified",

819 path=path,

820 )

821 )

822 else:

823 for idx, value in enumerate(state.rate_limit_long_wait_ladder):

824 if not isinstance(value, int) or value < 1:

825 errors.append(

826 ValidationError(

827 message=(

828 f"'rate_limit_long_wait_ladder[{idx}]' must be a "

829 f"positive integer, got {value!r}"

830 ),

831 path=path,

832 )

833 )

834

835 # Validate throttle config when present

836 if state.throttle is not None:

837 t = state.throttle

838 fields = {

839 "normal_max": t.normal_max,

840 "warn_max": t.warn_max,

841 "hard_max": t.hard_max,

842 }

843 for field_name, val in fields.items():

844 if val is not None and (not isinstance(val, int) or val < 1):

845 errors.append(

846 ValidationError(

847 message=f"'throttle.{field_name}' must be a positive integer, got {val!r}",

848 path=path,

849 )

850 )

851 # Enforce ordering when all three are set

852 if t.normal_max is not None and t.warn_max is not None and t.normal_max >= t.warn_max:

853 errors.append(

854 ValidationError(

855 message=(

856 f"'throttle.normal_max' ({t.normal_max}) must be less than "

857 f"'throttle.warn_max' ({t.warn_max})"

858 ),

859 path=path,

860 )

861 )

862 if t.warn_max is not None and t.hard_max is not None and t.warn_max >= t.hard_max:

863 errors.append(

864 ValidationError(

865 message=(

866 f"'throttle.warn_max' ({t.warn_max}) must be less than "

867 f"'throttle.hard_max' ({t.hard_max})"

868 ),

869 path=path,

870 )

871 )

872

873 return errors

874

875

876 def _validate_targets(fsm: FSMLoop) -> list[ValidationError]:

877 """Validate top-level targets[] entries (ENH-1552).

878

879 Rejects any targets[].states[] entry whose sibling file: value does not

880 end with a .yaml extension.

881 """

882 errors: list[ValidationError] = []

883 for i, target in enumerate(fsm.targets):

884 if target.file is not None and not target.file.endswith(".yaml"):

885 errors.append(

886 ValidationError(

887 message=(f"targets[{i}].file must be a .yaml file, got '{target.file}'"),

888 path=f"targets[{i}].file",

889 )

890 )

891 return errors

892

893

894 def _validate_failure_terminal_action(fsm: FSMLoop) -> list[ValidationError]:

895 """Warn when a failure-named terminal state has no diagnostic predecessor.

896

897 Failure terminals (failed, error, aborted) should have at least one

898 predecessor state with an action or sub-loop that provides diagnostic

899 output before termination. Otherwise the failure is silent — the

900 executor calls _finish("terminal") before any action on the terminal

901 itself can execute.

902

903 Severity is WARNING (not ERROR) so that existing loops with bare

904 failure terminals continue to load, and test_terminal_only_state_valid

905 (which filters by ERROR) passes without modification.

906 """

907 FAILURE_TERMINAL_NAMES: frozenset[str] = frozenset({"failed", "error", "aborted"})

908 errors: list[ValidationError] = []

909

910 terminal_states = fsm.get_terminal_states()

911 failure_terminals = terminal_states & FAILURE_TERMINAL_NAMES

912

913 for ft_name in failure_terminals:

914 has_diagnostic_predecessor = False

915 for state_name, state in fsm.states.items():

916 if state_name == ft_name:

917 continue

918 if ft_name in state.get_referenced_states():

919 if state.action is not None or state.loop is not None:

920 has_diagnostic_predecessor = True

921 break

922

923 if not has_diagnostic_predecessor:

924 errors.append(

925 ValidationError(

926 message=(

927 f"Failure-named terminal state '{ft_name}' has no predecessor "

928 "state with a diagnostic action. Add a non-terminal diagnostic "

929 "state (e.g. 'diagnose') with an action or sub-loop that routes "

930 f"to '{ft_name}'."

931 ),

932 path=f"states.{ft_name}",

933 severity=ValidationSeverity.WARNING,

934 )

935 )

936

937 return errors

938

939

940 def validate_fsm(fsm: FSMLoop) -> list[ValidationError]:

941 """Validate FSM structure and return list of errors.

942

943 Performs comprehensive validation:

944 - Initial state exists

945 - All referenced states exist

946 - At least one terminal state

947 - Evaluator configurations are valid

948 - Routing configurations are valid

949 - Numeric fields are in valid ranges (max_iterations > 0, backoff >= 0, timeout > 0)

950

951 Args:

952 fsm: The FSM loop to validate

953

954 Returns:

955 List of validation errors (empty if valid)

956 """

957 errors: list[ValidationError] = []

958 defined_states = fsm.get_all_state_names()

959

960 # Warn when no top-level description: field is set. The field is optional

961 # for FSM execution but required for goal-alignment skills (debug-loop-run,

962 # audit-loop-run) and for ll-loop show --json to surface intent text.

963 if not fsm.description:

964 errors.append(

965 ValidationError(

966 path="<root>",

967 message=("No 'description' field defined. Add a top-level description: key."),

968 severity=ValidationSeverity.WARNING,

969 )

970 )

971

972 # Validate parameters block

973 errors.extend(_validate_parameters(fsm))

974

975 # Validate targets block (ENH-1552)

976 errors.extend(_validate_targets(fsm))

977

978 # Check initial state exists

979 if fsm.initial not in defined_states:

980 errors.append(

981 ValidationError(

982 message=f"Initial state '{fsm.initial}' not found in states",

983 path="initial",

984 )

985 )

986

987 # Check at least one terminal state

988 terminal_states = fsm.get_terminal_states()

989 if not terminal_states:

990 errors.append(

991 ValidationError(

992 message="No terminal state defined. At least one state must have 'terminal: true'",

993 path="states",

994 )

995 )

996

997 # Validate each state

998 for state_name, state in fsm.states.items():

999 # Check all referenced states exist

1000 refs = state.get_referenced_states()

1001 for ref in refs:

1002 # $current is a special token for retry

1003 if ref != "$current" and ref not in defined_states:

1004 errors.append(

1005 ValidationError(

1006 message=f"References unknown state '{ref}'",

1007 path=f"states.{state_name}",

1008 )

1009 )

1010

1011 # Validate action configuration

1012 errors.extend(_validate_state_action(state_name, state))

1013

1014 # Validate evaluator if present

1015 if state.evaluate is not None:

1016 errors.extend(_validate_evaluator(state_name, state.evaluate))

1017

1018 # Validate routing configuration

1019 errors.extend(_validate_state_routing(state_name, state))

1020

1021 # Check numeric field ranges

1022 if fsm.max_steps <= 0:

1023 errors.append(

1024 ValidationError(

1025 message=f"max_steps must be > 0, got {fsm.max_steps}",

1026 path="max_steps",

1027 )

1028 )

1029 if fsm.max_iterations is not None and fsm.max_iterations <= 0:

1030 errors.append(

1031 ValidationError(

1032 message=f"max_iterations must be > 0, got {fsm.max_iterations}",

1033 path="max_iterations",

1034 )

1035 )

1036 if fsm.max_edge_revisits <= 0:

1037 errors.append(

1038 ValidationError(

1039 message=f"max_edge_revisits must be > 0, got {fsm.max_edge_revisits}",

1040 path="max_edge_revisits",

1041 )

1042 )

1043 if fsm.backoff is not None and fsm.backoff < 0:

1044 errors.append(

1045 ValidationError(

1046 message=f"backoff must be >= 0, got {fsm.backoff}",

1047 path="backoff",

1048 )

1049 )

1050 if fsm.timeout is not None and fsm.timeout <= 0:

1051 errors.append(

1052 ValidationError(

1053 message=f"timeout must be > 0, got {fsm.timeout}",

1054 path="timeout",

1055 )

1056 )

1057 if fsm.llm.max_tokens <= 0:

1058 errors.append(

1059 ValidationError(

1060 message=f"llm.max_tokens must be > 0, got {fsm.llm.max_tokens}",

1061 path="llm.max_tokens",

1062 )

1063 )

1064 if fsm.llm.timeout <= 0:

1065 errors.append(

1066 ValidationError(

1067 message=f"llm.timeout must be > 0, got {fsm.llm.timeout}",

1068 path="llm.timeout",

1069 )

1070 )

1071

1072 # Check for unreachable states (warning only)

1073 reachable = _find_reachable_states(fsm)

1074 unreachable = defined_states - reachable

1075 for state_name in unreachable:

1076 errors.append(

1077 ValidationError(

1078 message="State is not reachable from initial state",

1079 path=f"states.{state_name}",

1080 severity=ValidationSeverity.WARNING,

1081 )

1082 )

1083

1084 errors.extend(_validate_failure_terminal_action(fsm))

1085

1086 errors.extend(_validate_meta_loop_evaluation(fsm))

1087

1088 errors.extend(_validate_input_key_without_guard(fsm))

1089

1090 errors.extend(_validate_artifact_isolation(fsm))

1091

1092 errors.extend(_validate_harness_multimodal_evaluator_blind_spot(fsm))

1093

1094 errors.extend(_validate_partial_route_dead_end(fsm))

1095

1096 errors.extend(_validate_artifact_overwrite(fsm))

1097

1098 errors.extend(_validate_generator_fix_discipline(fsm))

1099

1100 errors.extend(_validate_bash_default_interpolation(fsm))

1101

1102 errors.extend(_validate_overescaped_shell(fsm))

1103

1104 errors.extend(_validate_classify_route_default(fsm))

1105

1106 errors.extend(_validate_zero_retry_counter(fsm))

1107

1108 errors.extend(_validate_on_max_steps(fsm, defined_states))

1109 errors.extend(_validate_on_max_iterations(fsm, defined_states))

1110

1111 errors.extend(_validate_circuit(fsm, defined_states))

1112

1113 errors.extend(_validate_progress_paths_isolation(fsm))

1114

1115 errors.extend(_validate_capture_reachability(fsm))

1116

1117 errors.extend(_validate_llm_evidence_contract(fsm))

1118

1119 return errors

1120

1121

1122 def _is_meta_loop(fsm: FSMLoop) -> bool:

1123 """Return True if fsm is classified as a meta-loop.

1124

1125 A loop is meta if ANY of the following match:

1126 1. Any state action string matches a harness-artifact path regex

1127 (writes another loop YAML, skill, agent, command, or project config)

1128 2. The loop's import list contains lib/benchmark.yaml

1129 3. Any state action references yaml_state_editor or replace_action

1130 """

1131 # Condition 2: imports lib/benchmark.yaml

1132 if any(imp in _META_LOOP_IMPORT_TRIGGERS for imp in fsm.imports):

1133 return True

1134 # Conditions 1 and 3: scan action strings

1135 for state in fsm.states.values():

1136 if state.action is None:

1137 continue

1138 for pattern in _META_LOOP_ACTION_PATTERNS:

1139 if pattern.search(state.action):

1140 return True

1141 for token in _META_LOOP_ACTION_TOKENS:

1142 if token in state.action:

1143 return True

1144 return False

1145

1146

1147 def _validate_meta_loop_evaluation(fsm: FSMLoop) -> list[ValidationError]:

1148 """Validate meta-loop evaluation rules MR-1 and MR-2.

1149

1150 MR-1 (ERROR): meta-loop must have at least one non-LLM evaluator.

1151 MR-2 (WARNING): meta-loop should reference a captured baseline in an evaluator.

1152

1153 Both rules are suppressed by ``meta_self_eval_ok: true`` at the loop top-level.

1154 """

1155 errors: list[ValidationError] = []

1156 if fsm.meta_self_eval_ok or not _is_meta_loop(fsm):

1157 return errors

1158

1159 # Collect all evaluator types used across all states

1160 evaluator_types: set[str] = set()

1161 for state in fsm.states.values():

1162 if state.evaluate is not None:

1163 evaluator_types.add(state.evaluate.type)

1164

1165 # MR-1: must have at least one non-LLM evaluator

1166 if not evaluator_types & NON_LLM_EVALUATOR_TYPES:

1167 errors.append(

1168 ValidationError(

1169 message=(

1170 "Loop modifies harness artifacts but has no non-LLM evaluator. "

1171 "LLM self-grades on harness updates are unreliable (SHOR Table 1: "

1172 "33-55% accuracy). Pair every check_semantic state with at least one "

1173 "of: exit_code, output_numeric, convergence, diff_stall, action_stall, mcp_result. "

1174 "Note: llm_structured and comparator both use the LLM and do not satisfy MR-1. "

1175 "To suppress with justification, set `meta_self_eval_ok: true` at the "

1176 "loop top-level."

1177 ),

1178 path="<root>",

1179 severity=ValidationSeverity.ERROR,

1180 )

1181 )

1182

1183 # MR-2: should reference a captured baseline in a later evaluator

1184 capture_names: set[str] = {state.capture for state in fsm.states.values() if state.capture}

1185 if capture_names and not _has_baseline_reference(fsm, capture_names):

1186 errors.append(

1187 ValidationError(

1188 message=(

1189 "Meta-loop appears to lack a measure→propose→apply→re-measure "

1190 "spine: no captured baseline value is referenced by a later evaluator. "

1191 "Meta-loops should compare a post-change score against a pre-change "

1192 "baseline (see loops/harness-optimize.yaml as reference template). "

1193 "To suppress, set `meta_self_eval_ok: true`."

1194 ),

1195 path="<root>",

1196 severity=ValidationSeverity.WARNING,

1197 )

1198 )

1199

1200 return errors

1201

1202

1203# Regex patterns for detecting counter-increment actions.

1204# Must contain a printf/echo writing to a file AND an arithmetic increment.

1205_COUNTER_FILE_WRITE_RE = re.compile(r"(?:printf|echo)\s+.*>")

1206_COUNTER_INCREMENT_RE = re.compile(

1207 r"\$\(\(.*\+\s*1\s*\)\)" # $((N + 1)) or $((N+1))

1208 r"|\+\+" # C-style increment

1209 r"|\+=1" # compound assignment

1210 r"|awk\s+.*\+\+" # awk with increment

1211)

1212

1213

1214 def _validate_zero_retry_counter(fsm: FSMLoop) -> list[ValidationError]:

1215 """Detect counter + output_numeric combos that yield zero effective retries.

1216

1217 A common loop-authoring footgun: a state increments a counter file and then

1218 evaluates ``output_numeric`` with ``operator: lt, target: 1`` against it.

1219 After the first increment the counter is 1, ``1 < 1 == false``, so the

1220 retry budget is 0 by construction. Author almost always intended target=2.

1221 """

1222 errors: list[ValidationError] = []

1223

1224 for state_name, state in fsm.states.items():

1225 if not state.action or not state.evaluate:

1226 continue

1227

1228 ev = state.evaluate

1229 if ev.type != "output_numeric":

1230 continue

1231 if ev.operator is None or ev.target is None:

1232 continue

1233

1234 # Must be a number-like target for numeric comparison

1235 try:

1236 target = float(ev.target)

1237 except (ValueError, TypeError):

1238 continue

1239

1240 if not _is_counter_action(state.action):

1241 continue

1242

1243 # Check: after first increment (0→1), does operator(1, target) already fail?

1244 op_fn = _NUMERIC_OPERATORS.get(ev.operator)

1245 if op_fn is None:

1246 continue

1247

1248 if not op_fn(1.0, target):

1249 suggested_target = _suggested_target(ev.operator, target)

1250 errors.append(

1251 ValidationError(

1252 message=(

1253 f"Zero retry budget: operator={ev.operator} target={target} "

1254 f"means the first post-increment value (1) already fails "

1255 f"({ev.operator}(1, {target}) == False). "

1256 f"Did you mean target={suggested_target}?"

1257 ),

1258 path=f"states.{state_name}.evaluate",

1259 severity=ValidationSeverity.WARNING,

1260 )

1261 )

1262

1263 return errors

1264

1265

1266 def _is_counter_action(action: str) -> bool:

1267 """Return True if the action string contains a counter-increment pattern."""

1268 return bool(_COUNTER_FILE_WRITE_RE.search(action) and _COUNTER_INCREMENT_RE.search(action))

1269

1270

1271 def _suggested_target(operator: str, target: float) -> str:

1272 """Suggest a target value that allows at least one retry."""

1273 # For lt/le with a too-low target, suggest target+1 so first post-increment passes

1274 if operator in ("lt", "le"):

1275 return str(int(target) + 1)

1276 # For eq with target=0, suggest 1 so the counter can eventually match

1277 if operator == "eq" and target == 0:

1278 return "1"

1279 # For other cases, suggest target+1 as a default nudge

1280 return str(int(target) + 1)

1281

1282

1283 def _validate_harness_multimodal_evaluator_blind_spot(fsm: FSMLoop) -> list[ValidationError]:

1284 """Warn when harness loops use LLM multimodal eval as sole gate to terminal.

1285

1286 LLMs can silently fall back to text-only analysis when reading images,

1287 producing verdicts based on incomplete information. The output_contains

1288 evaluator can verify the LLM wrote the pass string but not that it

1289 actually processed the image. This is the same class of failure as MR-1

1290 (LLM self-evaluation bias) applied to artifact evaluation rather than

1291 harness modification.

1292

1293 Suppressed by ``meta_self_eval_ok: true`` at the loop top-level.

1294 """

1295 errors: list[ValidationError] = []

1296 if fsm.meta_self_eval_ok or fsm.category != "harness":

1297 return errors

1298

1299 terminal_states = fsm.get_terminal_states()

1300

1301 for state_name, state in fsm.states.items():

1302 if state.action_type != "prompt" or not state.action:

1303 continue

1304 if state.evaluate is None or state.evaluate.type != "output_contains":

1305 continue

1306 if not any(p.search(state.action) for p in _MULTIMODAL_EVAL_PATTERNS):

1307 continue

1308 if state.on_yes not in terminal_states:

1309 continue

1310 errors.append(

1311 ValidationError(

1312 message=(

1313 f"State '{state_name}' evaluates a screenshot/image via LLM "

1314 "prompt and routes directly to a terminal on success. The "

1315 "output_contains evaluator can verify the LLM wrote the pass "

1316 "string but not that the LLM actually processed the image. "

1317 "Consider adding a shell-action verification state (e.g., "

1318 "functional smoke test) between scoring and the terminal."

1319 ),

1320 path=f"states.{state_name}",

1321 severity=ValidationSeverity.WARNING,

1322 )

1323 )

1324

1325 return errors

1326

1327

1328 def _find_shared_tmp_writes(fsm: FSMLoop) -> list[tuple[str, str]]:

1329 """Return (state_name, matched_path) for every action referencing shared .loops/tmp/.

1330

1331 Scans `state.action` only. Prompts and sub-loop bindings can also reference

1332 paths, but those are out of static-scan reach: action strings are the

1333 place where loop YAMLs directly encode artifact paths.

1334 """

1335 findings: list[tuple[str, str]] = []

1336 for state_name, state in fsm.states.items():

1337 if not state.action:

1338 continue

1339 for match in _SHARED_TMP_PATH_RE.finditer(state.action):

1340 findings.append((state_name, match.group(0)))

1341 return findings

1342

1343

1344 def _validate_input_key_without_guard(fsm: FSMLoop) -> list[ValidationError]:

1345 """Warn when a loop sets a custom input_key but omits required_inputs.

1346

1347 A loop that accepts a runtime input via a named key (e.g. input_key: description)

1348 but doesn't declare required_inputs will silently proceed with an empty value if the

1349 user forgets to pass one. Declaring required_inputs shifts that failure to start-time.

1350 """

1351 if fsm.input_key == "input":

1352 return []

1353 if fsm.required_inputs:

1354 return []

1355 return [

1356 ValidationError(

1357 message=(

1358 f"Loop sets input_key: '{fsm.input_key}' but does not declare "

1359 f"required_inputs. If this input is mandatory, add "

1360 f"'required_inputs: [\"{fsm.input_key}\"]' to make the runner "

1361 f"abort when no input is provided."

1362 ),

1363 path="required_inputs",

1364 severity=ValidationSeverity.WARNING,

1365 )

1366 ]

1367

1368

1369 def _validate_artifact_isolation(fsm: FSMLoop) -> list[ValidationError]:

1370 """Validate rule MR-3: loops must isolate artifacts to ${context.run_dir}.

1371

1372 The runner injects ${context.run_dir} pointing at .loops/runs/<loop>-<ts>/

1373 and creates the folder before execution. Loops that write intermediate

1374 state (queues, checkpoints, generated files) to shared .loops/tmp/ paths

1375 will corrupt each other under concurrent runs (ll-parallel workers, retries,

1376 repeated invocations).

1377

1378 Suppressed by `shared_state_ok: true` at the loop top-level for loops that

1379 intentionally share state across runs.

1380 """

1381 if fsm.shared_state_ok:

1382 return []

1383 errors: list[ValidationError] = []

1384 for state_name, path in _find_shared_tmp_writes(fsm):

1385 errors.append(

1386 ValidationError(

1387 message=(

1388 f"State writes to shared '{path}' instead of "

1389 "'${context.run_dir}/...'. Concurrent runs of this loop "

1390 "(e.g., under ll-parallel) will corrupt each other's state. "

1391 "Use the runner-injected `${context.run_dir}` for per-run "

1392 "artifact paths, or set `shared_state_ok: true` at the loop "

1393 "top-level if cross-run sharing is intentional."

1394 ),

1395 path=f"states.{state_name}.action",

1396 severity=ValidationSeverity.WARNING,

1397 )

1398 )

1399 return errors

1400

1401

1402 def _is_llm_judged(state: StateConfig) -> bool:

1403 """Return True if this state will be graded by the default LLM judge.

1404

1405 Mirrors the action-mode detection in executor._action_mode() (not imported

1406 here because that is runtime code). A state is LLM-judged when:

1407 - it has no explicit evaluate block AND its action is a prompt or slash_command, OR

1408 - it has an explicit evaluate block of type llm_structured or check_semantic.

1409 """

1410 if state.evaluate is None:

1411 # Heuristic: explicit action_type wins; fall back to leading "/" on action string.

1412 action_type = state.action_type

1413 if action_type in ("prompt", "slash_command"):

1414 return True

1415 if action_type is None and state.action and state.action.lstrip().startswith("/"):

1416 return True

1417 return False

1418 return state.evaluate.type in ("llm_structured", "check_semantic")

1419

1420

1421 def _validate_partial_route_dead_end(fsm: FSMLoop) -> list[ValidationError]:

1422 """Validate rule MR-4: LLM-judged states with only on_yes have a partial/no dead-end.

1423

1424 A state gated by the default LLM judge can receive yes/no/partial verdicts.

1425 If only on_yes is mapped (and no on_no, on_partial, next, or route table with

1426 a default exist), a partial or no verdict returns None from _route and silently

1427 terminates the loop — the parent treats this as failed.

1428

1429 Suppressed by `partial_route_ok: true` at the loop top-level for the rare

1430 case where dead-ending on a non-yes verdict is intentional.

1431 """

1432 if fsm.partial_route_ok:

1433 return []

1434 errors: list[ValidationError] = []

1435 for state_name, state in fsm.states.items():

1436 if not _is_llm_judged(state):

1437 continue

1438 # States with an unconditional next: or a full route: table are safe.

1439 if state.next is not None or state.route is not None:

1440 continue

1441 # Only flag when on_yes is set but at least one of on_no/on_partial is missing.

1442 if state.on_yes is None:

1443 continue

1444 missing = [v for v in ("no", "partial") if getattr(state, f"on_{v}") is None]

1445 if not missing:

1446 continue

1447 unrouted = " or ".join(f"`{v}`" for v in missing)

1448 errors.append(

1449 ValidationError(

1450 message=(

1451 f"[state: {state_name}] LLM-judged prompt routes only on_yes; "

1452 f"a {unrouted} verdict has no route and will dead-end the loop "

1453 "(parent reads this as failed). Add on_no/on_partial, use `next:` "

1454 "for an unconditional handoff, or a `route:` table with a default. "

1455 "Set `partial_route_ok: true` at the loop top-level to suppress "

1456 "if intentional. (ENH-1917)"

1457 ),

1458 path=f"states.{state_name}",

1459 severity=ValidationSeverity.WARNING,

1460 )

1461 )

1462 return errors

1463

1464

1465 def _validate_artifact_overwrite(fsm: FSMLoop) -> list[ValidationError]:

1466 """Validate rule MR-5 (ENH-1957): harness loops should version artifacts per iteration.

1467

1468 A harness-category loop that iteratively generates and overwrites a flat artifact

1469 path (e.g. ``${context.run_dir}/image.svg``) loses all intermediate versions.

1470 Only the final iteration survives. This rule flags iterative generate→evaluate→generate

1471 cycles that write to artifact paths without declaring versioning intent.

1472

1473 Suppressed by ``artifact_versioning: true`` (loop snapshots per-iteration artifacts)

1474 or ``artifact_versioning_ok: true`` (intentional overwrite, e.g. artifact varies

1475 by task).

1476 """

1477 if fsm.artifact_versioning or fsm.artifact_versioning_ok:

1478 return []

1479 if fsm.category not in ("harness",):

1480 return []

1481

1482 errors: list[ValidationError] = []

1483

1484 # Find states that write to artifact paths (shell actions with file output)

1485 writers: dict[str, set[str]] = {} # state_name -> set of artifact paths

1486 for state_name, state in fsm.states.items():

1487 if not state.action or state.action_type not in ("shell", None):

1488 continue

1489 # Skip sub-loop delegation states

1490 if state.action_type == "loop":

1491 continue

1492 action = state.action

1493 # Find run_dir-based artifact writes: ${context.run_dir}/<path> or $RUNDIR/<path>

1494 import re

1495

1496 # Match patterns like: ${context.run_dir}/output.svg, $RUNDIR/image.png, > path

1497 # We look for output redirections or cp/mv commands writing to run_dir

1498 artifact_refs = set()

1499 for pattern in (

1500 r'\$\{context\.run_dir\}/([^\s"\';&]+)',

1501 r'\$\{captured\.[^}]+\}/([^\s"\';&]+)',

1502 ):

1503 for m in re.finditer(pattern, action):

1504 artifact_refs.add(m.group(1))

1505 # Also detect explicit cp/mv writing to run_dir paths

1506 for m in re.finditer(r'(?:cp|mv)\s+.*\s+\$\{context\.run_dir\}/([^\s"\';&]+)', action):

1507 artifact_refs.add(m.group(1))

1508 if artifact_refs:

1509 writers[state_name] = artifact_refs

1510

1511 if not writers:

1512 return []

1513

1514 # Detect iterative cycles: a writer state that is reachable from itself

1515 # via a non-trivial path through other states (generate → evaluate → generate)

1516 for state_name in writers:

1517 refs = fsm.states[state_name].get_referenced_states()

1518 # Check if this writer or its downstream states loop back to this writer

1519 visited: set[str] = set()

1520 to_visit = list(refs - {"$current"})

1521 while to_visit:

1522 target = to_visit.pop()

1523 if target in visited or target not in fsm.states:

1524 continue

1525 visited.add(target)

1526 if target == state_name:

1527 # Found a cycle: this writer state is reachable from itself

1528 artifact_list = ", ".join(sorted(writers[state_name]))

1529 errors.append(

1530 ValidationError(

1531 message=(

1532 f"[state: {state_name}] Harness loop writes artifact(s) "

1533 f"({artifact_list}) to a flat path in an iterative cycle "

1534 f"({state_name} → ... → {state_name}). Per-iteration versions "

1535 "are lost; only the final output survives. Add per-iteration "

1536 "snapshots (see oracle generator-evaluator for pattern) and "

1537 "declare `artifact_versioning: true`, or set "

1538 "`artifact_versioning_ok: true` if intentional. (ENH-1957)"

1539 ),

1540 path=f"states.{state_name}",

1541 severity=ValidationSeverity.WARNING,

1542 )

1543 )

1544 break

1545 target_state = fsm.states.get(target)

1546 if target_state is not None:

1547 target_refs = target_state.get_referenced_states()

1548 for r in target_refs:

1549 if r != "$current" and r not in visited:

1550 to_visit.append(r)

1551

1552 return errors

1553

1554

1555 def _validate_generator_fix_discipline(fsm: FSMLoop) -> list[ValidationError]:

1556 """Validate rule MR-6 (ENH-2079): meta-loops should not hand-patch generator artifacts.

1557

1558 Detects the hand-patching anti-pattern: a ``shell``-type state that writes to the

1559 same file path as a non-shell (LLM-type) generator state in the same loop.

1560 Hand-patching creates fragile output that diverges from the generator on the next

1561 run; the stable fix is to update the generator action instead.

1562

1563 Suppressed by ``generator_fix_ok: true`` at the loop top-level for intentional

1564 post-processing cases.

1565 """

1566 if fsm.generator_fix_ok or not _is_meta_loop(fsm):

1567 return []

1568

1569 # Markers that indicate a prompt/slash_command state is generating file artifacts

1570 _GENERATOR_MARKERS = ("yaml_state_editor", "replace_action", "to_file:")

1571

1572 _PATH_PATTERNS = (

1573 re.compile(r'\$\{context\.run_dir\}/([^\s"\';&|]+)'),

1574 re.compile(r'\$\{captured\.[^}]+\}/([^\s"\';&|]+)'),

1575 )

1576

1577 def _extract_paths(action: str) -> set[str]:

1578 paths: set[str] = set()

1579 for pat in _PATH_PATTERNS:

1580 for m in pat.finditer(action):

1581 paths.add(m.group(1).rstrip("/"))

1582 return paths

1583

1584 shell_targets: dict[str, set[str]] = {} # state_name -> set of file paths

1585 generator_targets: dict[str, set[str]] = {} # state_name -> set of file paths

1586

1587 for state_name, state in fsm.states.items():

1588 if not state.action:

1589 continue

1590 action = state.action

1591 paths = _extract_paths(action)

1592 if not paths:

1593 continue

1594 action_type = state.action_type

1595 if action_type in ("shell", None):

1596 shell_targets[state_name] = paths

1597 elif action_type in ("prompt", "slash_command"):

1598 if any(marker in action for marker in _GENERATOR_MARKERS):

1599 generator_targets[state_name] = paths

1600

1601 errors: list[ValidationError] = []

1602 for gen_name, gen_paths in generator_targets.items():

1603 for shell_name, shell_paths in shell_targets.items():

1604 overlap = gen_paths & shell_paths

1605 if overlap:

1606 artifact_list = ", ".join(sorted(overlap))

1607 errors.append(

1608 ValidationError(

1609 message=(

1610 f"[states: {gen_name}, {shell_name}] Hand-patching anti-pattern: "

1611 f"LLM-generator state '{gen_name}' and shell state '{shell_name}' "

1612 f"both write to ({artifact_list}). Move the fix into the generator "

1613 "action so every run produces correct output automatically. "

1614 "Set `generator_fix_ok: true` to suppress for intentional "

1615 "post-processing. (ENH-2079)"

1616 ),

1617 path=f"states.{shell_name}",

1618 severity=ValidationSeverity.WARNING,

1619 )

1620 )

1621

1622 return errors

1623

1624

1625 def _find_bash_default_tokens(fsm: FSMLoop) -> list[tuple[str, str]]:

1626 """Return (state_name, matched_token) for every action with unescaped ${ns.path:-...}.

1627

1628 Scans ``state.action`` only. The escaped form ``$${VAR:-value}`` is exempted by

1629 the negative lookbehind in ``_BASH_DEFAULT_RE``; the engine-native ``:default=``

1630 form does not contain ``:-`` and is therefore never matched.

1631 """

1632 findings: list[tuple[str, str]] = []

1633 for state_name, state in fsm.states.items():

1634 if not state.action:

1635 continue

1636 for match in _BASH_DEFAULT_RE.finditer(state.action):

1637 findings.append((state_name, match.group(0)))

1638 return findings

1639

1640

1641 def _validate_bash_default_interpolation(fsm: FSMLoop) -> list[ValidationError]:

1642 """Validate rule MR-7 (ENH-2348): unescaped bash :-default interpolation.

1643

1644 The FSM interpolation engine supports ``${namespace.path:default=value}`` for

1645 author-specified defaults, but does NOT support the bash parameter-expansion form

1646 ``${namespace.path:-value}``. Any loop action that uses the bash form will crash

1647 at runtime with ``Path 'ns.path:-value' not found in context``.

1648

1649 Supported alternatives:

1650 - ``${context.x:default=queue}`` — engine-native default (always preferred)

1651 - ``$${VAR:-value}`` — escaped, passed to the shell verbatim

1652

1653 Suppressed by ``bash_default_ok: true`` at the loop top-level for the rare case

1654 where an author can justify the unsupported form.

1655 """

1656 if fsm.bash_default_ok:

1657 return []

1658 errors: list[ValidationError] = []

1659 for state_name, token in _find_bash_default_tokens(fsm):

1660 errors.append(

1661 ValidationError(

1662 message=(

1663 f"[state: {state_name}] {token} uses unsupported bash ':-' default. "

1664 "The FSM interpolator will crash at runtime with 'Path not found in context'. "

1665 f"Use {token.split(':-')[0]}:default={token.split(':-')[1].rstrip('}')}}}"

1666 " (engine default) or $${{VAR:-value}} (shell, escaped). "

1667 "Set `bash_default_ok: true` to suppress. (ENH-2348)"

1668 ),

1669 path=f"states.{state_name}.action",

1670 severity=ValidationSeverity.ERROR,

1671 )

1672 )

1673 return errors

1674

1675

1676 def _validate_overescaped_shell(fsm: FSMLoop) -> list[ValidationError]:

1677 """Validate rule MR-9: over-escaped shell ``$$`` that expands to the PID.

1678

1679 The FSM interpolation engine only rewrites the brace form ``$${...}`` → ``${...}``;

1680 bare ``$(...)`` command substitution and ``$VAR`` references are passed through to

1681 ``bash -c`` untouched. Doubling them is therefore NOT an escape — ``$$(`` and

1682 ``$$VAR`` reach the shell literally, where the leading ``$$`` expands to the process

1683 ID. An ``init`` state that does ``echo "$$(pwd)/$$DIR"`` captures ``<pid>(pwd)/<pid>DIR``

1684 instead of an absolute path, silently corrupting every downstream ``${captured…}``

1685 reference (observed in interactive-/html-/svg-generator).

1686

1687 Correct forms:

1688 - ``$(pwd)`` / ``$DIR`` — command substitution / variable (shell handles them)

1689 - ``$${VAR}`` / ``$${VAR:-x}`` — ONLY brace vars collide with ``${ns.path}`` and

1690 need the ``$$`` escape; the interpolator converts ``$${`` → ``${``.

1691

1692 Scans shell actions only (``action_type`` shell/untyped, excluding slash commands);

1693 ``$$VAR`` in a prompt is inert text. Suppressed by ``shell_pid_ok: true`` at the

1694 loop top-level for the rare case where a literal PID is intended.

1695 """

1696 if fsm.shell_pid_ok:

1697 return []

1698 errors: list[ValidationError] = []

1699 for state_name, state in fsm.states.items():

1700 if not state.action:

1701 continue

1702 # Shell-only: prompt/slash-command actions never reach `bash -c`, so a `$$`

1703 # there is harmless text. Mirror the shell gate used by other shell rules.

1704 if state.action_type not in ("shell", None):

1705 continue

1706 if state.action.lstrip().startswith("/"):

1707 continue

1708 for match in _OVERESCAPED_SHELL_RE.finditer(state.action):

1709 # Show the offending token plus the next char for context (e.g. `$$(` / `$$D`).

1710 start = match.start()

1711 token = state.action[start : start + 3]

1712 errors.append(

1713 ValidationError(

1714 message=(

1715 f"[state: {state_name}] '{token}' over-escapes shell `$$`. "

1716 "The interpolator only converts the brace form $${{...}}; bare "

1717 "$(...) and $VAR pass through untouched, so $$ here expands to the "

1718 "PID at `bash -c` time (e.g. $$(pwd)/$$DIR -> <pid>(pwd)/<pid>DIR). "

1719 "Use single $ ($(pwd), $DIR); reserve $$ for the $${{VAR}} brace "

1720 "escape. Set `shell_pid_ok: true` to suppress. (MR-9)"

1721 ),

1722 path=f"states.{state_name}.action",

1723 severity=ValidationSeverity.ERROR,

1724 )

1725 )

1726 return errors

1727

1728

1729_EVIDENCE_CONTRACT_KEYWORDS: frozenset[str] = frozenset({"verbatim", "quote", "evidence"})

1730

1731

1732 def _validate_llm_evidence_contract(fsm: FSMLoop) -> list[ValidationError]:

1733 """Validate rule MR-8 (ENH-2342): LLM-judged state prompts should include evidence-contract keywords.

1734

1735 A check_semantic/llm_structured state whose ``evaluate.prompt`` does not contain any of

1736 ``{"verbatim", "quote", "evidence"}`` may produce verdicts without cited output text,

1737 defaulting to optimism (SHOR Table 1: 33–55% accuracy; Sonnet 4.6 = 33.4%).

1738

1739 States with no evaluate block (Path B: action prompt) or ``evaluate.prompt is None``

1740 inherit DEFAULT_LLM_PROMPT which includes CHECK_SEMANTIC_EVIDENCE_CONTRACT after ENH-2342

1741 and are not flagged here.

1742

1743 Suppressed by ``evidence_contract_ok: true`` at the loop top-level.

1744 """

1745 if fsm.evidence_contract_ok:

1746 return []

1747 errors: list[ValidationError] = []

1748 for state_name, state in fsm.states.items():

1749 if not _is_llm_judged(state):

1750 continue

1751 if state.evaluate is None:

1752 continue

1753 if state.evaluate.prompt is None:

1754 continue

1755 prompt_lower = state.evaluate.prompt.lower()

1756 if not any(kw in prompt_lower for kw in _EVIDENCE_CONTRACT_KEYWORDS):

1757 errors.append(

1758 ValidationError(

1759 message=(

1760 f"[state: {state_name}] check_semantic/llm_structured prompt "

1761 "does not include evidence-contract keywords (verbatim, quote, evidence). "

1762 "Verdicts without cited evidence default to optimism "

1763 "(SHOR Table 1: 33–55% accuracy). Add a requirement to quote exact "

1764 "output text, or set `evidence_contract_ok: true` to suppress. "

1765 "(ENH-2342 MR-8)"

1766 ),

1767 path=f"states.{state_name}.evaluate.prompt",

1768 severity=ValidationSeverity.WARNING,

1769 )

1770 )

1771 return errors

1772

1773

1774 def _validate_classify_route_default(fsm: FSMLoop) -> list[ValidationError]:

1775 """Validate that classify states with a route: table include a default: fallback.

1776

1777 A ``classify`` state whose ``route:`` table has no ``default:`` will dead-end

1778 whenever the action emits a token not listed in the table. This rule flags

1779 that gap as a WARNING so loop authors add a catch-all branch.

1780

1781 Suppressed by ``partial_route_ok: true`` at the loop top-level when a

1782 dead-end on an unlisted token is intentional.

1783 """

1784 if fsm.partial_route_ok:

1785 return []

1786 errors: list[ValidationError] = []

1787 for state_name, state in fsm.states.items():

1788 if state.evaluate is None or state.evaluate.type != "classify":

1789 continue

1790 if state.route is None or state.route.default is not None:

1791 continue

1792 errors.append(

1793 ValidationError(

1794 message=(

1795 f"[state: {state_name}] classify route: table has no default: — "

1796 "unknown tokens will dead-end the loop. Add a default: catch-all, "

1797 "or set `partial_route_ok: true` at the loop top-level to suppress."

1798 ),

1799 path=f"states.{state_name}",

1800 severity=ValidationSeverity.WARNING,

1801 )

1802 )

1803 return errors

1804

1805

1806 def _has_baseline_reference(fsm: FSMLoop, capture_names: set[str]) -> bool:

1807 """Return True if any evaluate block references a captured variable."""

1808 for state in fsm.states.values():

1809 ev = state.evaluate

1810 if ev is None:

1811 continue

1812 # Check string fields that may interpolate captured values

1813 candidates = [ev.previous, ev.source]

1814 if isinstance(ev.target, str):

1815 candidates.append(ev.target)

1816 for field_val in candidates:

1817 if not field_val:

1818 continue

1819 for name in capture_names:

1820 if f"captured.{name}" in field_val:

1821 return True

1822 return False

1823

1824

1825 def _validate_on_max_steps(fsm: FSMLoop, defined_states: set[str]) -> list[ValidationError]:

1826 """Validate the top-level `on_max_steps` field (BUG-2204).

1827

1828 Checks that the named state exists when `on_max_steps` is set.

1829 """

1830 errors: list[ValidationError] = []

1831 if fsm.on_max_steps is None:

1832 return errors

1833 if fsm.on_max_steps not in defined_states:

1834 errors.append(

1835 ValidationError(

1836 message=(

1837 f"on_max_steps references unknown state "

1838 f"'{fsm.on_max_steps}' (must be a declared state)"

1839 ),

1840 path="on_max_steps",

1841 )

1842 )

1843 return errors

1844

1845

1846 def _validate_on_max_iterations(fsm: FSMLoop, defined_states: set[str]) -> list[ValidationError]:

1847 """Validate the top-level `on_max_iterations` field (BUG-2204: iteration-cap summary state).

1848

1849 Checks that the named state exists when `on_max_iterations` is set.

1850 """

1851 errors: list[ValidationError] = []

1852 if fsm.on_max_iterations is None:

1853 return errors

1854 if fsm.on_max_iterations not in defined_states:

1855 errors.append(

1856 ValidationError(

1857 message=(

1858 f"on_max_iterations references unknown state "

1859 f"'{fsm.on_max_iterations}' (must be a declared state)"

1860 ),

1861 path="on_max_iterations",

1862 )

1863 )

1864 return errors

1865

1866

1867 def _validate_circuit(fsm: FSMLoop, defined_states: set[str]) -> list[ValidationError]:

1868 """Validate the top-level `circuit:` block (FEAT-1637).

1869

1870 Checks:

1871 - `circuit.repeated_failure.window` is a positive integer.

1872 - `circuit.repeated_failure.on_repeated_failure` is either the special

1873 token ``"abort"`` or the name of a declared state.

1874 """

1875 errors: list[ValidationError] = []

1876 if fsm.circuit is None or fsm.circuit.repeated_failure is None:

1877 return errors

1878

1879 rf = fsm.circuit.repeated_failure

1880 if rf.window < 1:

1881 errors.append(

1882 ValidationError(

1883 message=f"circuit.repeated_failure.window must be >= 1, got {rf.window}",

1884 path="circuit.repeated_failure.window",

1885 )

1886 )

1887

1888 if rf.recurrent_window is not None and rf.recurrent_window < 2:

1889 errors.append(

1890 ValidationError(

1891 message=(

1892 f"circuit.repeated_failure.recurrent_window must be >= 2, "

1893 f"got {rf.recurrent_window}"

1894 ),

1895 path="circuit.repeated_failure.recurrent_window",

1896 )

1897 )

1898

1899 target = rf.on_repeated_failure

1900 if target not in STALL_SPECIAL_TOKENS and target not in defined_states:

1901 errors.append(

1902 ValidationError(

1903 message=(

1904 f"circuit.repeated_failure.on_repeated_failure references "

1905 f"unknown state '{target}' (must be a declared state or "

1906 f'the literal "abort")'

1907 ),

1908 path="circuit.repeated_failure.on_repeated_failure",

1909 )

1910 )

1911

1912 return errors

1913

1914

1915# Matches common interpolation prefixes used in loop YAML paths so we can

1916# extract the portable relative component for action-string scanning.

1917_INTERPOLATION_PREFIX_RE = re.compile(r"^\$\{[^}]+\}/")

1918

1919

1920 def _strip_interpolation_prefix(path: str) -> str:

1921 """Return the path with any leading ${...}/ prefix removed."""

1922 return _INTERPOLATION_PREFIX_RE.sub("", path)

1923

1924

1925 def _validate_progress_paths_isolation(fsm: FSMLoop) -> list[ValidationError]:

1926 """Warn when a state's action writes to a file listed in progress_paths (BUG-1767).

1927

1928 When a loop's own bookkeeping files appear in both progress_paths and the

1929 state action strings, every append to those files resets the stall window,

1930 silently disabling the BUG-1674 stall guard for that loop. Authors should

1931 move such files to exclude_paths so the stall detector can still fire.

1932 """

1933 if fsm.circuit is None or fsm.circuit.repeated_failure is None:

1934 return []

1935 rf = fsm.circuit.repeated_failure

1936 if not rf.progress_paths:

1937 return []

1938

1939 # Build a set of the relative path components we need to look for.

1940 watched = {_strip_interpolation_prefix(p) for p in rf.progress_paths}

1941 # Exclude paths that are already in exclude_paths — author acknowledged.

1942 excluded = {_strip_interpolation_prefix(p) for p in rf.exclude_paths}

1943 active_watched = watched - excluded

1944 if not active_watched:

1945 return []

1946

1947 errors: list[ValidationError] = []

1948 for state_name, state in fsm.states.items():

1949 if not state.action:

1950 continue

1951 for path_fragment in active_watched:

1952 if path_fragment in state.action:

1953 errors.append(

1954 ValidationError(

1955 message=(

1956 f"State action references '{path_fragment}', which is also "

1957 "listed in circuit.repeated_failure.progress_paths. Writes "

1958 "to this file will reset the stall window every cycle, "

1959 "silently disabling stall detection. Move it to "

1960 "circuit.repeated_failure.exclude_paths to separate "

1961 "bookkeeping files from real progress signals."

1962 ),

1963 path=f"states.{state_name}.action",

1964 severity=ValidationSeverity.WARNING,

1965 )

1966 )

1967 return errors

1968

1969

1970 def _dominated_by_any(fsm: FSMLoop, dominators: set[str], dominated: str) -> bool:

1971 """Return True if the set ``dominators`` collectively dominates ``dominated``.

1972

1973 Group domination: every path from the initial state to ``dominated`` must

1974 pass through at least one state in ``dominators``. Checked by removing all

1975 dominator states from the graph and testing whether ``dominated`` is still

1976 reachable from the initial state.

1977

1978 This generalizes single-state domination — used when a capture variable is

1979 produced by more than one state on mutually-exclusive branches, where the

1980 reference is safe as long as *some* capturing state runs on every path.

1981

1982 Args:

1983 fsm: The FSM loop to analyze

1984 dominators: Names of the states that should collectively dominate

1985 dominated: Name of the state that should be dominated

1986

1987 Returns:

1988 True if the dominators collectively dominate ``dominated``

1989 """

1990 if dominated in dominators:

1991 return True

1992 if dominated not in fsm.states:

1993 return False

1994

1995 visited: set[str] = set()

1996 to_visit: deque[str] = deque([fsm.initial])

1997

1998 while to_visit:

1999 current = to_visit.popleft()

2000 if current in visited or current not in fsm.states:

2001 continue

2002 if current in dominators:

2003 continue # Block this node (simulate removal)

2004

2005 visited.add(current)

2006

2007 if current == dominated:

2008 # Reached dominated without going through any dominator

2009 return False

2010

2011 state = fsm.states[current]

2012 for ref in state.get_referenced_states():

2013 if ref != "$current" and ref not in visited:

2014 to_visit.append(ref)

2015

2016 # Dominated not reachable without the dominators → they dominate

2017 return True

2018

2019

2020 def _dominates(fsm: FSMLoop, dominator: str, dominated: str) -> bool:

2021 """Return True if dominator dominates dominated in the FSM graph.

2022

2023 A state D dominates S if every path from the initial state to S must pass

2024 through D. Thin single-state wrapper around :func:`_dominated_by_any`.

2025 """

2026 if dominator not in fsm.states:

2027 return False

2028 return _dominated_by_any(fsm, {dominator}, dominated)

2029

2030

2031 def _find_bypass_path_any(fsm: FSMLoop, dominators: set[str], dominated: str) -> list[str]:

2032 """Find an example path from initial to dominated that bypasses all dominators.

2033

2034 Uses BFS to find the shortest path that avoids every state in ``dominators``.

2035 Returns empty list if no bypass exists (should not happen when called after

2036 :func:`_dominated_by_any` returns False).

2037 """

2038 parent: dict[str, str] = {}

2039 to_visit: deque[str] = deque([fsm.initial])

2040 visited: set[str] = set()

2041

2042 while to_visit:

2043 current = to_visit.popleft()

2044 if current in visited or current not in fsm.states:

2045 continue

2046 if current in dominators:

2047 continue

2048

2049 visited.add(current)

2050

2051 if current == dominated:

2052 # Reconstruct path

2053 path = [dominated]

2054 while path[-1] in parent:

2055 path.append(parent[path[-1]])

2056 path.reverse()

2057 return path

2058

2059 state = fsm.states[current]

2060 for ref in state.get_referenced_states():

2061 if ref != "$current" and ref not in visited:

2062 if ref not in parent:

2063 parent[ref] = current

2064 to_visit.append(ref)

2065

2066 return []

2067

2068

2069 def _find_bypass_path(fsm: FSMLoop, dominator: str, dominated: str) -> list[str]:

2070 """Find an example path from initial to dominated that bypasses dominator.

2071

2072 Thin single-state wrapper around :func:`_find_bypass_path_any`.

2073 """

2074 return _find_bypass_path_any(fsm, {dominator}, dominated)

2075

2076

2077 def _has_sub_loop_state(fsm: FSMLoop) -> bool:

2078 """Return True if any state in the FSM has ``loop:`` set (delegates to a child loop).

2079

2080 Used by ENH-1961 to distinguish "capture lives in a sub-loop" from "capture is missing".

2081 """

2082 return any(state.loop is not None for state in fsm.states.values())

2083

2084

2085 def _validate_capture_reachability(fsm: FSMLoop) -> list[ValidationError]:

2086 """Validate that ``${captured.*}`` references are dominated by their capturing states.

2087

2088 ENH-1961: For each state that references ``${captured.<var>.*}`` in its action

2089 or evaluate source, checks that the capturing state dominates the referencing

2090 state (i.e., all paths from the initial state pass through the capture state).

2091

2092 Emits:

2093 - WARNING when a capture state does not dominate a referencing state

2094 (the reference may crash at runtime on paths that bypass the capture).

2095 - ERROR when a referenced capture variable has no capturing state at all

2096 in this FSM (excluding sub-loop captures which live in child namespaces).

2097 """

2098 errors: list[ValidationError] = []

2099

2100 # Step 1: Build capture map (var_name → set of capturing state names).

2101 # A variable may be captured by more than one state on mutually-exclusive

2102 # branches (e.g. fifo_pop vs select_next dispatched by schedule_mode); the

2103 # reference is safe as long as the *set* collectively dominates it.

2104 capture_map: dict[str, set[str]] = {}

2105 for state_name, state in fsm.states.items():

2106 if state.capture:

2107 capture_map.setdefault(state.capture, set()).add(state_name)

2108

2109 # Step 2: Build reference map (state_name → set of captured var names referenced)

2110 reference_map: dict[str, set[str]] = {}

2111 for state_name, state in fsm.states.items():

2112 # Skip sub-loop delegation states — their action is a loop name,

2113 # and captured vars belong to the child loop's namespace.

2114 if state.loop is not None:

2115 continue

2116

2117 # Only collect references NOT guarded by `:default=` — a guarded

2118 # reference is safe even when the capture is missing on some path.

2119 refs: set[str] = set()

2120 if state.action:

2121 refs.update(_unguarded_captured_refs(state.action))

2122 if state.evaluate is not None and state.evaluate.source:

2123 refs.update(_unguarded_captured_refs(state.evaluate.source))

2124 if refs:

2125 reference_map[state_name] = refs

2126

2127 if not reference_map:

2128 return errors

2129

2130 # Step 3: For each reference, check dominance of capturing state

2131 for ref_state_name, ref_vars in reference_map.items():

2132 for var_name in ref_vars:

2133 if var_name not in capture_map:

2134 # Referenced capture variable has no capturing state in this FSM.

2135 # ENH-1998: downgrade to WARNING (not silent skip) when sub-loops

2136 # are present — the capture may live in a child namespace, but a

2137 # typo'd name should still surface rather than go completely dark.

2138 if _has_sub_loop_state(fsm):

2139 errors.append(

2140 ValidationError(

2141 message=(

2142 f"References ${{captured.{var_name}.*}} but no state in "

2143 f"this loop captures '{var_name}'. "

2144 f"If '{var_name}' is produced by a sub-loop, this may be "

2145 f"intentional; otherwise add 'capture: {var_name}' to the "

2146 f"state that produces this value."

2147 ),

2148 path=f"states.{ref_state_name}.action",

2149 severity=ValidationSeverity.WARNING,

2150 )

2151 )

2152 continue

2153 # No sub-loops: this is genuinely missing.

2154 errors.append(

2155 ValidationError(

2156 message=(

2157 f"References ${{captured.{var_name}.*}} but no state in "

2158 f"this loop captures '{var_name}'. Add 'capture: {var_name}' "

2159 f"to the state that produces this value."

2160 ),

2161 path=f"states.{ref_state_name}.action",

2162 severity=ValidationSeverity.ERROR,

2163 )

2164 )

2165 continue

2166

2167 # Capturing states present in this FSM (shouldn't drop any normally)

2168 cap_states = {s for s in capture_map[var_name] if s in fsm.states}

2169 if not cap_states:

2170 continue

2171

2172 # Group dominance check: do the capturing states collectively

2173 # dominate ref_state_name (does at least one run on every path)?

2174 if not _dominated_by_any(fsm, cap_states, ref_state_name):

2175 bypass_path = _find_bypass_path_any(fsm, cap_states, ref_state_name)

2176 path_str = " → ".join(bypass_path) if bypass_path else "unknown path"

2177

2178 if len(cap_states) == 1:

2179 captured_by = f"state '{next(iter(cap_states))}' which may not"

2180 else:

2181 names = ", ".join(f"'{s}'" for s in sorted(cap_states))

2182 captured_by = f"states {names}, none of which"

2183

2184 errors.append(

2185 ValidationError(

2186 message=(

2187 f"References ${{captured.{var_name}.*}} but '{var_name}' "

2188 f"is captured by {captured_by} "

2189 f"execute on all paths to '{ref_state_name}'. "

2190 f"Path(s) bypassing capture: {path_str}"

2191 ),

2192 path=f"states.{ref_state_name}.action",

2193 severity=ValidationSeverity.WARNING,

2194 )

2195 )

2196

2197 return errors

2198

2199

2200 def _find_reachable_states(fsm: FSMLoop) -> set[str]:

2201 """Find all states reachable from the initial state.

2202

2203 Uses breadth-first search to find all reachable states. Seeds the BFS

2204 with the initial state plus top-level transition targets that act as

2205 alternate entry points: ``on_max_iterations`` (fires when the iteration

2206 cap is hit) and ``circuit.repeated_failure.on_repeated_failure`` (fires

2207 when the circuit breaker trips). These are real edges the runtime can

2208 take, so states reached only through them are not orphans.

2209

2210 Args:

2211 fsm: The FSM loop to analyze

2212

2213 Returns:

2214 Set of reachable state names

2215 """

2216 reachable: set[str] = set()

2217 to_visit: deque[str] = deque([fsm.initial])

2218 if fsm.on_max_steps is not None:

2219 to_visit.append(fsm.on_max_steps)

2220 if fsm.on_max_iterations is not None:

2221 to_visit.append(fsm.on_max_iterations)

2222 if fsm.circuit is not None and fsm.circuit.repeated_failure is not None:

2223 target = fsm.circuit.repeated_failure.on_repeated_failure

2224 if target not in STALL_SPECIAL_TOKENS:

2225 to_visit.append(target)

2226

2227 while to_visit:

2228 current = to_visit.popleft()

2229 if current in reachable or current not in fsm.states:

2230 continue

2231

2232 reachable.add(current)

2233 state = fsm.states[current]

2234 refs = state.get_referenced_states()

2235

2236 for ref in refs:

2237 if ref != "$current" and ref not in reachable:

2238 to_visit.append(ref)

2239

2240 return reachable

2241

2242

2243 def is_runnable_loop(path: Path) -> bool:

2244 """Cheap check for whether a YAML file is a runnable FSM loop definition.

2245

2246 Returns True iff the file parses as a YAML mapping with the required

2247 top-level keys ``name``, ``initial``, and either ``states`` or ``flow``

2248 (the shorthand resolved by :func:`resolve_flow`). This matches the

2249 required-fields gate in :func:`load_and_validate` so "counted by the

2250 verifier" stays in sync with "runnable by ll-loop validate".

2251

2252 When the raw YAML contains a ``from:`` key, inheritance is resolved first

2253 (mirroring :func:`load_and_validate`) so pure context-override stubs whose

2254 parent provides ``initial``/``states`` return True. Library fragments under

2255 ``loops/lib/`` still return False — their parent chain also lacks ``initial``.

2256 """

2257 try:

2258 data = yaml.safe_load(path.read_text())

2259 except (OSError, yaml.YAMLError):

2260 return False

2261 if not isinstance(data, dict):

2262 return False

2263 if "from" in data:

2264 try:

2265 data = resolve_inheritance(data, path.parent)

2266 except Exception:

2267 return False

2268 has_flow = "states" in data or "flow" in data

2269 return "name" in data and "initial" in data and has_flow

2270

2271

2272 def load_and_validate(

2273 path: Path,

2274 raise_on_error: bool = True,

2275) -> tuple[FSMLoop, list[ValidationError]]:

2276 """Load YAML file and validate FSM structure.

2277

2278 Args:

2279 path: Path to the YAML file to load

2280 raise_on_error: When True (default), raise ValueError on ERROR violations.

2281 When False, return all violations (errors + warnings) without raising.

2282

2283 Returns:

2284 When raise_on_error=True: (FSMLoop, list of WARNING-severity ValidationErrors)

2285 When raise_on_error=False: (FSMLoop, list of all ValidationErrors sorted errors-first)

2286

2287 Raises:

2288 FileNotFoundError: If the file doesn't exist

2289 yaml.YAMLError: If the file is not valid YAML

2290 ValueError: If raise_on_error=True and validation fails (contains error details)

2291 """

2292 if not path.exists():

2293 raise FileNotFoundError(f"FSM file not found: {path}")

2294

2295 with open(path) as f:

2296 data: dict[str, Any] = yaml.safe_load(f)

2297

2298 if not isinstance(data, dict):

2299 raise ValueError(f"FSM file must contain a YAML mapping, got {type(data)}")

2300

2301 # Resolve `from:` inheritance before any further checks, so a child loop

2302 # can omit fields its parent provides (including `initial`/`states`) and

2303 # so a parent's `import:`/`fragments:` blocks survive into the merged

2304 # result for the subsequent `resolve_fragments` pass.

2305 data = resolve_inheritance(data, path.parent)

2306

2307 # Expand flow: linear shorthand into states: before required-fields check

2308 data = resolve_flow(data)

2309

2310 # Check required fields before parsing

2311 missing = []

2312 for field in ["name", "initial"]:

2313 if field not in data:

2314 missing.append(field)

2315 if "states" not in data:

2316 missing.append("states (or flow)")

2317

2318 if missing:

2319 raise ValueError(f"FSM file missing required fields: {', '.join(missing)}")

2320

2321 # Check for unknown top-level keys before parsing

2322 unknown_key_warnings: list[ValidationError] = []

2323 unknown = set(data.keys()) - KNOWN_TOP_LEVEL_KEYS

2324 if unknown:

2325 unknown_key_warnings.append(

2326 ValidationError(

2327 path="<root>",

2328 message=f"Unknown top-level keys: {', '.join(sorted(unknown))}",

2329 severity=ValidationSeverity.WARNING,

2330 )

2331 )

2332

2333 visibility_val = data.get("visibility")

2334 if visibility_val is not None and visibility_val not in VALID_VISIBILITY:

2335 unknown_key_warnings.append(

2336 ValidationError(

2337 path="visibility",

2338 message=(

2339 f"Invalid visibility: {visibility_val!r}. "

2340 f"Must be one of: {', '.join(sorted(VALID_VISIBILITY))}. "

2341 "Loop will be treated as 'public'."

2342 ),

2343 severity=ValidationSeverity.WARNING,

2344 )

2345 )

2346

2347 # Resolve fragment libraries before parsing into dataclass

2348 data = resolve_fragments(data, path.parent)

2349

2350 # Parse into dataclass

2351 fsm = FSMLoop.from_dict(data)

2352

2353 # Validate

2354 errors = validate_fsm(fsm)

2355

2356 # Validate with: bindings against child loop parameters (requires file-system access)

2357 errors.extend(_validate_with_bindings(fsm, path.parent))

2358 errors.extend(_validate_loop_references(fsm, path.parent))

2359 errors.extend(_validate_fragment_bindings(fsm, path.parent))

2360

2361 # Filter to errors only (not warnings) for raising

2362 error_list = [e for e in errors if e.severity == ValidationSeverity.ERROR]

2363 struct_warnings = [e for e in errors if e.severity == ValidationSeverity.WARNING]

2364 all_warnings = unknown_key_warnings + struct_warnings

2365

2366 if not raise_on_error:

2367 return fsm, error_list + all_warnings

2368

2369 if error_list:

2370 error_messages = "\n ".join(str(e) for e in error_list)

2371 raise ValueError(f"FSM validation failed:\n {error_messages}")

2372

2373 for warning in all_warnings:

2374 logger.warning(str(warning))

2375

2376 return fsm, all_warnings