Coverage for little_loops / fsm / validation.py: 11%
817 statements
« prev ^ index » next coverage.py v7.12.0, created at 2026-06-29 00:54 -0500
1"""FSM loop validation logic.
3This module provides validation for FSM loop definitions, ensuring
4structural correctness and catching common configuration errors.
6Validation checks:
7- Initial state exists in states dict
8- All referenced states exist
9- At least one terminal state
10- Evaluator configs have required fields for their type
11- No conflicting routing (shorthand vs full route)
12- Numeric fields in valid ranges (max_iterations > 0, backoff >= 0, timeout > 0)
13"""
15from __future__ import annotations
17import logging
18import re
19from collections import deque
20from dataclasses import dataclass
21from enum import Enum
22from pathlib import Path
23from typing import Any
25import yaml
27from little_loops.fsm.evaluators import _NUMERIC_OPERATORS
28from little_loops.fsm.fragments import resolve_flow, resolve_fragments, resolve_inheritance
29from little_loops.fsm.schema import EvaluateConfig, FSMLoop, ParameterSpec, StateConfig
31logger = logging.getLogger(__name__)
34class ValidationSeverity(Enum):
35 """Severity level for validation issues."""
37 ERROR = "error"
38 WARNING = "warning"
41@dataclass
42class ValidationError:
43 """Structured validation error.
45 Attributes:
46 message: Human-readable error description
47 path: Path to the problematic element (e.g., "states.check.route")
48 severity: Error severity (error or warning)
49 """
51 message: str
52 path: str | None = None
53 severity: ValidationSeverity = ValidationSeverity.ERROR
55 def __str__(self) -> str:
56 """Format error for display."""
57 prefix = f"[{self.severity.value.upper()}]"
58 if self.path:
59 return f"{prefix} {self.path}: {self.message}"
60 return f"{prefix} {self.message}"
63# Evaluator type to required fields mapping
64EVALUATOR_REQUIRED_FIELDS: dict[str, list[str]] = {
65 "exit_code": [],
66 "output_numeric": ["operator", "target"],
67 "output_json": ["path", "operator", "target"],
68 "output_contains": ["pattern"],
69 "convergence": ["target"],
70 "diff_stall": [],
71 "action_stall": [],
72 "llm_structured": [],
73 "mcp_result": [],
74 "harbor_scorer": [],
75 "comparator": ["baseline_path"],
76 "contract": ["pairs"],
77 "classify": [],
78}
80# Non-LLM evaluator types: all evaluator types except llm_structured
81# Derived from EVALUATOR_REQUIRED_FIELDS so new types are automatically included
82NON_LLM_EVALUATOR_TYPES: frozenset[str] = frozenset(EVALUATOR_REQUIRED_FIELDS.keys()) - {
83 "llm_structured",
84 "comparator",
85 "contract",
86}
88# Meta-loop detector: action string patterns that indicate harness artifact writes
89_META_LOOP_ACTION_PATTERNS: tuple[re.Pattern[str], ...] = (
90 re.compile(r"loops/[\w-]+\.yaml"),
91 re.compile(r"skills/[\w-]+/SKILL\.md"),
92 re.compile(r"agents/[\w-]+\.md"),
93 re.compile(r"commands/[\w-]+\.md"),
94 re.compile(r"\.claude/(CLAUDE\.md|settings)"),
95)
97# Action string tokens that indicate meta-loop behavior
98_META_LOOP_ACTION_TOKENS: frozenset[str] = frozenset({"yaml_state_editor", "replace_action"})
100# Import paths that identify a loop as a meta-loop (harness optimization framework)
101_META_LOOP_IMPORT_TRIGGERS: frozenset[str] = frozenset({"lib/benchmark.yaml"})
103# MR-3: shared-tmp path detector. The runner injects ${context.run_dir} resolving
104# to .loops/runs/<loop>-<timestamp>/; loops that hardcode .loops/tmp/ instead
105# cause state corruption under concurrent runs (ll-parallel, retries, etc.).
106_SHARED_TMP_PATH_RE = re.compile(r"\.loops/tmp/[\w./-]+")
108# MR-7: bash-default interpolation detector. Matches ${namespace.path:-default}
109# (unescaped bash `:- ` default form) that the FSM interpolator does not support.
110# Negative lookbehind exempts the legitimate escaped form $${VAR:-value}.
111_BASH_DEFAULT_RE = re.compile(r"(?<!\$)\$\{[^}]*:-[^}]*\}")
113# MR-9: over-escaped shell `$$` detector. The FSM interpolator only rewrites the
114# brace form `$${...}` → `${...}`; bare `$(...)` and `$VAR` are passed to the shell
115# untouched (interpolation.py). Doubling them — `$$(` or `$$VAR` — is NOT an escape:
116# the runner's `bash -c` expands the leading `$$` to the PID, so `$$(pwd)/$$DIR`
117# becomes `<pid>(pwd)/<pid>DIR`. The lookahead matches `$$(` (command substitution)
118# and `$$<identifier-start>` (variable) while exempting the legitimate `$${...}`
119# brace escape (next char `{`) and a standalone PID `$$` (followed by `/`, `.`, space,
120# or end). See ENH for the interactive-/html-/svg-generator over-escape bug.
121_OVERESCAPED_SHELL_RE = re.compile(r"\$\$(?=\(|[A-Za-z_])")
123# ENH-1961: Regex for extracting captured variable names from ${captured.<var>.*} references
124_CAPTURED_REF_RE = re.compile(r"\$\{captured\.(\w+)")
126# Full-reference form, capturing the var name and the remainder up to the closing
127# brace so we can detect a `:default=` guard. A reference written as
128# `${captured.x.output:default=...}` is provably safe even on paths that bypass
129# the capturing state — the interpolation engine (interpolation.py) substitutes
130# the default when the path is missing — so it must NOT be flagged by the
131# capture-reachability check. _CAPTURED_REF_RE alone can't see the guard.
132_CAPTURED_REF_FULL_RE = re.compile(r"\$\{captured\.(\w+)([^}]*)\}")
135def _unguarded_captured_refs(text: str) -> set[str]:
136 """Return captured var names that have at least one reference WITHOUT a
137 `:default=` guard. Vars referenced only via `${captured.x...:default=...}`
138 are omitted: the default makes a missing value safe, so they should not
139 trigger missing-capture or bypass-path diagnostics.
140 """
141 refs: set[str] = set()
142 for var_name, remainder in _CAPTURED_REF_FULL_RE.findall(text):
143 if ":default=" not in remainder:
144 refs.add(var_name)
145 return refs
148# ENH-1819: Regex patterns for detecting multimodal evaluation in prompt actions
149_MULTIMODAL_EVAL_PATTERNS: tuple[re.Pattern[str], ...] = (
150 re.compile(r"Read the screenshot", re.IGNORECASE),
151 re.compile(r"view the (generated )?(website|page|image)", re.IGNORECASE),
152 re.compile(r"screenshot\.(png|jpg|jpeg|webp)"),
153 re.compile(r"\.(png|jpg|jpeg|webp)\b.*\b(read|view|evaluate|score|judge)", re.IGNORECASE),
154)
156# Valid comparison operators
157VALID_OPERATORS = {"eq", "ne", "lt", "le", "gt", "ge"}
159# Valid values for the top-level `visibility:` field (audience axis for
160# `ll-loop list` filtering). "public" is the default when the field is absent.
161VALID_VISIBILITY: frozenset[str] = frozenset({"public", "internal", "example"})
163# All top-level keys recognized by FSMLoop.from_dict()
164KNOWN_TOP_LEVEL_KEYS: frozenset[str] = frozenset(
165 {
166 "name",
167 "description",
168 "initial",
169 "states",
170 "context",
171 "parameters",
172 "scope",
173 "max_steps",
174 "on_max_steps",
175 "max_iterations",
176 "on_max_iterations",
177 "max_edge_revisits",
178 "backoff",
179 "timeout",
180 "default_timeout",
181 "maintain",
182 "llm",
183 "on_handoff",
184 "input_key",
185 "required_inputs",
186 "config",
187 "category",
188 "labels",
189 "visibility",
190 "commands",
191 "targets",
192 "circuit",
193 "meta_self_eval_ok",
194 "shared_state_ok",
195 "partial_route_ok",
196 "artifact_versioning",
197 "artifact_versioning_ok",
198 "generator_fix_ok",
199 "bash_default_ok",
200 "shell_pid_ok",
201 "import",
202 "fragments",
203 "from",
204 "flow",
205 "state_defs",
206 }
207)
209# Special tokens accepted as the `on_repeated_failure` target on the
210# stall detector — `"abort"` means terminate via _finish("stall_detected").
211STALL_SPECIAL_TOKENS: frozenset[str] = frozenset({"abort"})
213# Valid parameter types for the 'parameters:' block
214VALID_PARAMETER_TYPES: frozenset[str] = frozenset(
215 {"string", "integer", "number", "boolean", "enum", "path"}
216)
219def _validate_evaluator(state_name: str, evaluate: EvaluateConfig) -> list[ValidationError]:
220 """Validate evaluator configuration for type-specific requirements.
222 Args:
223 state_name: Name of the state containing this evaluator
224 evaluate: The evaluator configuration to validate
226 Returns:
227 List of validation errors found
228 """
229 errors: list[ValidationError] = []
230 path = f"states.{state_name}.evaluate"
232 # Check that evaluator type is recognized
233 valid_types = set(EVALUATOR_REQUIRED_FIELDS.keys())
234 if evaluate.type not in valid_types:
235 errors.append(
236 ValidationError(
237 message=f"Unknown evaluator type '{evaluate.type}'. "
238 f"Must be one of: {', '.join(sorted(valid_types))}",
239 path=path,
240 )
241 )
242 return errors # Can't check required fields for unknown type
244 # Check required fields for evaluator type
245 required = EVALUATOR_REQUIRED_FIELDS.get(evaluate.type, [])
246 for field_name in required:
247 value = getattr(evaluate, field_name, None)
248 if value is None:
249 errors.append(
250 ValidationError(
251 message=f"Evaluator type '{evaluate.type}' requires '{field_name}' field",
252 path=path,
253 )
254 )
256 # Validate operator if present
257 if evaluate.operator is not None and evaluate.operator not in VALID_OPERATORS:
258 errors.append(
259 ValidationError(
260 message=f"Invalid operator '{evaluate.operator}'. "
261 f"Must be one of: {', '.join(sorted(VALID_OPERATORS))}",
262 path=f"{path}.operator",
263 )
264 )
266 # Validate convergence-specific fields
267 if evaluate.type == "convergence":
268 if evaluate.direction not in ("minimize", "maximize"):
269 errors.append(
270 ValidationError(
271 message=f"Invalid direction '{evaluate.direction}'. "
272 "Must be 'minimize' or 'maximize'",
273 path=f"{path}.direction",
274 )
275 )
276 # Only validate tolerance if it's a numeric value (not an interpolation string)
277 if (
278 evaluate.tolerance is not None
279 and isinstance(evaluate.tolerance, (int, float))
280 and evaluate.tolerance < 0
281 ):
282 errors.append(
283 ValidationError(
284 message="Tolerance cannot be negative",
285 path=f"{path}.tolerance",
286 )
287 )
289 # Validate llm_structured-specific fields
290 if evaluate.type == "llm_structured":
291 if evaluate.min_confidence < 0 or evaluate.min_confidence > 1:
292 errors.append(
293 ValidationError(
294 message="min_confidence must be between 0 and 1",
295 path=f"{path}.min_confidence",
296 )
297 )
299 # Validate diff_stall-specific fields
300 if evaluate.type == "diff_stall":
301 if evaluate.max_stall < 1:
302 errors.append(
303 ValidationError(
304 message="max_stall must be >= 1",
305 path=f"{path}.max_stall",
306 )
307 )
309 # Validate action_stall-specific fields
310 if evaluate.type == "action_stall":
311 if evaluate.max_repeat < 1:
312 errors.append(
313 ValidationError(
314 message="max_repeat must be >= 1",
315 path=f"{path}.max_repeat",
316 )
317 )
319 return errors
322def _validate_parameters(fsm: FSMLoop) -> list[ValidationError]:
323 """Validate the loop's top-level parameters: block.
325 Args:
326 fsm: The FSM loop to validate
328 Returns:
329 List of validation errors found
330 """
331 errors: list[ValidationError] = []
333 for param_name, param_spec in fsm.parameters.items():
334 path = f"parameters.{param_name}"
336 if param_spec.type not in VALID_PARAMETER_TYPES:
337 errors.append(
338 ValidationError(
339 message=(
340 f"Unknown parameter type '{param_spec.type}'. "
341 f"Must be one of: {', '.join(sorted(VALID_PARAMETER_TYPES))}"
342 ),
343 path=path,
344 )
345 )
347 if param_spec.type == "enum" and not param_spec.values:
348 errors.append(
349 ValidationError(
350 message="Parameter type 'enum' requires a 'values' list",
351 path=path,
352 )
353 )
355 if param_spec.required and param_spec.default is not None:
356 errors.append(
357 ValidationError(
358 message="Parameter cannot be both 'required: true' and have a 'default' value",
359 path=path,
360 )
361 )
363 return errors
366def _check_param_type(value: Any, spec: ParameterSpec) -> str | None:
367 """Return an error message if value does not match spec.type, else None."""
368 if spec.type == "string" and not isinstance(value, str):
369 return f"expected string, got {type(value).__name__}"
370 if spec.type == "integer" and not isinstance(value, int):
371 return f"expected integer, got {type(value).__name__}"
372 if spec.type == "number" and not isinstance(value, (int, float)):
373 return f"expected number, got {type(value).__name__}"
374 if spec.type == "boolean" and not isinstance(value, bool):
375 return f"expected boolean, got {type(value).__name__}"
376 if spec.type == "enum" and spec.values and value not in spec.values:
377 return f"expected one of {spec.values!r}, got {value!r}"
378 return None
381def _validate_with_bindings(fsm: FSMLoop, loop_dir: Path) -> list[ValidationError]:
382 """Validate with: bindings against child loop parameter contracts.
384 Called from load_and_validate (not validate_fsm) because resolving child loops
385 requires file-system access via the loop directory path.
387 Args:
388 fsm: The parent FSM loop
389 loop_dir: Directory to resolve child loop paths from
391 Returns:
392 List of validation errors found
393 """
394 errors: list[ValidationError] = []
396 for state_name, state in fsm.states.items():
397 if state.loop is None or not state.with_:
398 continue
400 # Try to resolve and load the child loop; skip if unavailable
401 try:
402 from little_loops.cli.loop._helpers import resolve_loop_path
404 loop_path = resolve_loop_path(state.loop, loop_dir)
405 child_fsm, _ = load_and_validate(loop_path)
406 except Exception:
407 continue
409 if not child_fsm.parameters:
410 continue # Child has no declared contract — nothing to cross-validate
412 path = f"states.{state_name}"
414 # Unknown with: keys (not declared by child)
415 for key in state.with_:
416 if key not in child_fsm.parameters:
417 errors.append(
418 ValidationError(
419 message=(
420 f"'with.{key}' is not a declared parameter of loop '{state.loop}'. "
421 f"Declared: {', '.join(sorted(child_fsm.parameters))}"
422 ),
423 path=f"{path}.with.{key}",
424 )
425 )
427 # Required parameters not bound
428 for param_name, param_spec in child_fsm.parameters.items():
429 if param_spec.required and param_name not in state.with_:
430 errors.append(
431 ValidationError(
432 message=(
433 f"Required parameter '{param_name}' of loop '{state.loop}' "
434 f"is not bound in 'with'"
435 ),
436 path=f"{path}.with",
437 )
438 )
440 # Statically-detectable type mismatches (skip interpolation strings)
441 for param_name, value in state.with_.items():
442 if param_name not in child_fsm.parameters:
443 continue
444 if isinstance(value, str) and "${" in value:
445 continue
446 type_error = _check_param_type(value, child_fsm.parameters[param_name])
447 if type_error:
448 errors.append(
449 ValidationError(
450 message=f"Parameter '{param_name}': {type_error}",
451 path=f"{path}.with.{param_name}",
452 )
453 )
455 return errors
458def _validate_loop_references(fsm: FSMLoop, loop_dir: Path) -> list[ValidationError]:
459 """Validate that every state's loop: reference resolves to an actual loop file.
461 Called from load_and_validate (not validate_fsm) because resolving child loops
462 requires file-system access via the loop directory path.
463 """
464 errors: list[ValidationError] = []
465 for state_name, state in fsm.states.items():
466 if state.loop is None:
467 continue
468 # Skip dynamically interpolated loop names — they can only be checked at runtime
469 if "${" in state.loop:
470 continue
471 try:
472 from little_loops.cli.loop._helpers import resolve_loop_path
474 resolve_loop_path(state.loop, loop_dir)
475 except FileNotFoundError:
476 errors.append(
477 ValidationError(
478 message=f"Loop reference '{state.loop}' does not resolve to any file.",
479 path=f"states.{state_name}.loop",
480 severity=ValidationSeverity.WARNING,
481 )
482 )
483 return errors
486def _validate_fragment_bindings(fsm: FSMLoop, loop_dir: Path) -> list[ValidationError]:
487 """Validate fragment with: bindings against fragment parameter contracts.
489 Called from load_and_validate (not validate_fsm) because fragment parameters
490 are populated by resolve_fragments which runs before dataclass parsing.
492 Args:
493 fsm: The FSM loop to validate
494 loop_dir: Directory containing the loop file (unused; kept for API symmetry with
495 _validate_with_bindings)
497 Returns:
498 List of validation errors found
499 """
500 # Runner-injected vars available at runtime but not at static analysis time
501 RUNNER_INJECTED = {"run_dir", "loop_name", "started_at", "input_hash"}
503 errors: list[ValidationError] = []
505 for state_name, state in fsm.states.items():
506 if not state.fragment_parameters:
507 continue # No declared contract — nothing to cross-validate
509 path = f"states.{state_name}"
511 # Unknown with: keys (not declared by fragment)
512 for key in state.fragment_bindings:
513 if key not in state.fragment_parameters:
514 errors.append(
515 ValidationError(
516 message=(
517 f"'with.{key}' is not a declared parameter of fragment "
518 f"'{state.fragment_name}'. "
519 f"Declared: {', '.join(sorted(state.fragment_parameters))}"
520 ),
521 path=f"{path}.with.{key}",
522 )
523 )
525 # Required parameters not bound (whitelist runner-injected vars)
526 for param_name, param_spec in state.fragment_parameters.items():
527 if param_spec.required and param_name not in state.fragment_bindings:
528 if param_name in RUNNER_INJECTED:
529 continue # Available at runtime; not a static error
530 errors.append(
531 ValidationError(
532 message=(
533 f"Required parameter '{param_name}' of fragment "
534 f"'{state.fragment_name}' is not bound in 'with'"
535 ),
536 path=f"{path}.with",
537 )
538 )
540 # Statically-detectable type mismatches (skip interpolation strings)
541 for param_name, value in state.fragment_bindings.items():
542 if param_name not in state.fragment_parameters:
543 continue
544 if isinstance(value, str) and "${" in value:
545 continue
546 type_error = _check_param_type(value, state.fragment_parameters[param_name])
547 if type_error:
548 errors.append(
549 ValidationError(
550 message=f"Parameter '{param_name}': {type_error}",
551 path=f"{path}.with.{param_name}",
552 )
553 )
555 return errors
558def _validate_state_action(state_name: str, state: StateConfig) -> list[ValidationError]:
559 """Validate state action configuration.
561 Args:
562 state_name: Name of the state to validate
563 state: The state configuration to validate
565 Returns:
566 List of validation errors found
567 """
568 errors: list[ValidationError] = []
569 path = f"states.{state_name}"
571 # append_to_messages must contain at least one ${...} interpolation expression
572 if state.append_to_messages is not None:
573 if "${" not in state.append_to_messages:
574 errors.append(
575 ValidationError(
576 message=(
577 "'append_to_messages' must contain a ${...} interpolation expression "
578 f"(e.g. '${{captured.{state_name}.output}}')"
579 ),
580 path=f"{path}.append_to_messages",
581 )
582 )
584 # model: override is silently ignored for non-prompt states (host CLI is not invoked)
585 if state.model is not None and state.action_type not in ("prompt", "slash_command", None):
586 errors.append(
587 ValidationError(
588 message="model: override is ignored for shell/mcp_tool/contract states",
589 path=f"{path}.model",
590 severity=ValidationSeverity.WARNING,
591 )
592 )
594 # params field is only valid for mcp_tool states
595 if state.params and state.action_type != "mcp_tool":
596 errors.append(
597 ValidationError(
598 message="'params' field is only valid when action_type is 'mcp_tool'",
599 path=f"{path}.params",
600 )
601 )
603 # loop and action are mutually exclusive
604 if state.loop is not None and state.action is not None:
605 errors.append(
606 ValidationError(
607 message="'loop' and 'action' are mutually exclusive — "
608 "a sub-loop state cannot also have an action",
609 path=f"{path}",
610 )
611 )
613 # with: requires loop: to be set
614 if state.with_ and state.loop is None:
615 errors.append(
616 ValidationError(
617 message="'with' is only valid when 'loop' is set",
618 path=f"{path}.with",
619 )
620 )
622 # FEAT-1283: type=learning requires a populated LearningConfig
623 if state.type == "learning" and state.learning is not None:
624 if not state.learning.targets and not state.learning.targets_csv:
625 errors.append(
626 ValidationError(
627 message="type=learning requires non-empty 'learning.targets' or 'learning.targets_csv'",
628 path=f"{path}.learning.targets",
629 )
630 )
631 if state.learning.max_retries < 0:
632 errors.append(
633 ValidationError(
634 message=(
635 f"learning.max_retries must be >= 0, got {state.learning.max_retries}"
636 ),
637 path=f"{path}.learning.max_retries",
638 )
639 )
640 if state.on_yes is None:
641 errors.append(
642 ValidationError(
643 message="type=learning requires 'on_yes' (target for all-proven)",
644 path=f"{path}.on_yes",
645 )
646 )
647 if state.on_blocked is None and state.on_no is None:
648 errors.append(
649 ValidationError(
650 message=(
651 "type=learning requires 'on_blocked' or 'on_no' "
652 "(target for refuted / retries_exhausted)"
653 ),
654 path=f"{path}",
655 )
656 )
658 # with: and context_passthrough are mutually exclusive
659 if state.with_ and state.context_passthrough:
660 errors.append(
661 ValidationError(
662 message=(
663 "'with' and 'context_passthrough' are mutually exclusive — "
664 "use 'with' for explicit parameter bindings or 'context_passthrough' "
665 "for legacy bulk passthrough, not both"
666 ),
667 path=f"{path}",
668 )
669 )
671 return errors
674def _validate_state_routing(state_name: str, state: StateConfig) -> list[ValidationError]:
675 """Validate state routing configuration.
677 Checks for conflicting routing definitions (shorthand vs full route).
679 Args:
680 state_name: Name of the state to validate
681 state: The state configuration to validate
683 Returns:
684 List of validation errors/warnings found
685 """
686 errors: list[ValidationError] = []
687 path = f"states.{state_name}"
689 has_shorthand = (
690 state.on_yes is not None
691 or state.on_no is not None
692 or state.on_error is not None
693 or state.on_partial is not None
694 or state.on_blocked is not None
695 or bool(state.extra_routes)
696 )
697 has_route = state.route is not None
699 # Warn about conflicting definitions
700 if has_shorthand and has_route:
701 errors.append(
702 ValidationError(
703 message="Both shorthand routing (on_yes/on_no/on_error) "
704 "and full route table defined. Route table will take precedence.",
705 path=path,
706 severity=ValidationSeverity.WARNING,
707 )
708 )
710 # Check for no valid transition definition
711 has_next = state.next is not None
712 has_terminal = state.terminal
713 has_loop = state.loop is not None
715 if not has_shorthand and not has_route and not has_next and not has_terminal and not has_loop:
716 errors.append(
717 ValidationError(
718 message="State has no transition defined. Add routing, 'next', "
719 "or mark as 'terminal: true'",
720 path=path,
721 )
722 )
724 # Validate retry field pairing: max_retries requires on_retry_exhausted and vice versa
725 if state.max_retries is not None and state.on_retry_exhausted is None:
726 errors.append(
727 ValidationError(
728 message="'max_retries' requires 'on_retry_exhausted' to also be set",
729 path=path,
730 )
731 )
732 if state.on_retry_exhausted is not None and state.max_retries is None:
733 errors.append(
734 ValidationError(
735 message="'on_retry_exhausted' requires 'max_retries' to also be set",
736 path=path,
737 )
738 )
739 if state.max_retries is not None and state.max_retries < 1:
740 errors.append(
741 ValidationError(
742 message=f"'max_retries' must be >= 1, got {state.max_retries}",
743 path=path,
744 )
745 )
747 # Validate retryable_exit_codes: requires on_error; all codes must be positive ints
748 if state.retryable_exit_codes is not None:
749 if state.on_error is None:
750 errors.append(
751 ValidationError(
752 message="'retryable_exit_codes' requires 'on_error' to also be set",
753 path=path,
754 )
755 )
756 for code in state.retryable_exit_codes:
757 if not isinstance(code, int) or code < 1:
758 errors.append(
759 ValidationError(
760 message=(
761 f"'retryable_exit_codes' entries must be positive "
762 f"integers, got {code!r}"
763 ),
764 path=f"{path}.retryable_exit_codes",
765 )
766 )
767 break
769 # Validate rate-limit retry field pairing (mirrors max_retries/on_retry_exhausted)
770 if state.max_rate_limit_retries is not None and state.on_rate_limit_exhausted is None:
771 errors.append(
772 ValidationError(
773 message="'max_rate_limit_retries' requires 'on_rate_limit_exhausted' to also be set",
774 path=path,
775 )
776 )
777 if state.on_rate_limit_exhausted is not None and state.max_rate_limit_retries is None:
778 errors.append(
779 ValidationError(
780 message="'on_rate_limit_exhausted' requires 'max_rate_limit_retries' to also be set",
781 path=path,
782 )
783 )
784 if state.max_rate_limit_retries is not None and state.max_rate_limit_retries < 1:
785 errors.append(
786 ValidationError(
787 message=f"'max_rate_limit_retries' must be >= 1, got {state.max_rate_limit_retries}",
788 path=path,
789 )
790 )
791 if (
792 state.rate_limit_backoff_base_seconds is not None
793 and state.rate_limit_backoff_base_seconds < 1
794 ):
795 errors.append(
796 ValidationError(
797 message=(
798 f"'rate_limit_backoff_base_seconds' must be >= 1, "
799 f"got {state.rate_limit_backoff_base_seconds}"
800 ),
801 path=path,
802 )
803 )
804 if state.rate_limit_max_wait_seconds is not None and state.rate_limit_max_wait_seconds < 1:
805 errors.append(
806 ValidationError(
807 message=(
808 f"'rate_limit_max_wait_seconds' must be >= 1, "
809 f"got {state.rate_limit_max_wait_seconds}"
810 ),
811 path=path,
812 )
813 )
814 if state.rate_limit_long_wait_ladder is not None:
815 if len(state.rate_limit_long_wait_ladder) == 0:
816 errors.append(
817 ValidationError(
818 message="'rate_limit_long_wait_ladder' must be non-empty if specified",
819 path=path,
820 )
821 )
822 else:
823 for idx, value in enumerate(state.rate_limit_long_wait_ladder):
824 if not isinstance(value, int) or value < 1:
825 errors.append(
826 ValidationError(
827 message=(
828 f"'rate_limit_long_wait_ladder[{idx}]' must be a "
829 f"positive integer, got {value!r}"
830 ),
831 path=path,
832 )
833 )
835 # Validate throttle config when present
836 if state.throttle is not None:
837 t = state.throttle
838 fields = {
839 "normal_max": t.normal_max,
840 "warn_max": t.warn_max,
841 "hard_max": t.hard_max,
842 }
843 for field_name, val in fields.items():
844 if val is not None and (not isinstance(val, int) or val < 1):
845 errors.append(
846 ValidationError(
847 message=f"'throttle.{field_name}' must be a positive integer, got {val!r}",
848 path=path,
849 )
850 )
851 # Enforce ordering when all three are set
852 if t.normal_max is not None and t.warn_max is not None and t.normal_max >= t.warn_max:
853 errors.append(
854 ValidationError(
855 message=(
856 f"'throttle.normal_max' ({t.normal_max}) must be less than "
857 f"'throttle.warn_max' ({t.warn_max})"
858 ),
859 path=path,
860 )
861 )
862 if t.warn_max is not None and t.hard_max is not None and t.warn_max >= t.hard_max:
863 errors.append(
864 ValidationError(
865 message=(
866 f"'throttle.warn_max' ({t.warn_max}) must be less than "
867 f"'throttle.hard_max' ({t.hard_max})"
868 ),
869 path=path,
870 )
871 )
873 return errors
876def _validate_targets(fsm: FSMLoop) -> list[ValidationError]:
877 """Validate top-level targets[] entries (ENH-1552).
879 Rejects any targets[].states[] entry whose sibling file: value does not
880 end with a .yaml extension.
881 """
882 errors: list[ValidationError] = []
883 for i, target in enumerate(fsm.targets):
884 if target.file is not None and not target.file.endswith(".yaml"):
885 errors.append(
886 ValidationError(
887 message=(f"targets[{i}].file must be a .yaml file, got '{target.file}'"),
888 path=f"targets[{i}].file",
889 )
890 )
891 return errors
894def _validate_failure_terminal_action(fsm: FSMLoop) -> list[ValidationError]:
895 """Warn when a failure-named terminal state has no diagnostic predecessor.
897 Failure terminals (failed, error, aborted) should have at least one
898 predecessor state with an action or sub-loop that provides diagnostic
899 output before termination. Otherwise the failure is silent — the
900 executor calls _finish("terminal") before any action on the terminal
901 itself can execute.
903 Severity is WARNING (not ERROR) so that existing loops with bare
904 failure terminals continue to load, and test_terminal_only_state_valid
905 (which filters by ERROR) passes without modification.
906 """
907 FAILURE_TERMINAL_NAMES: frozenset[str] = frozenset({"failed", "error", "aborted"})
908 errors: list[ValidationError] = []
910 terminal_states = fsm.get_terminal_states()
911 failure_terminals = terminal_states & FAILURE_TERMINAL_NAMES
913 for ft_name in failure_terminals:
914 has_diagnostic_predecessor = False
915 for state_name, state in fsm.states.items():
916 if state_name == ft_name:
917 continue
918 if ft_name in state.get_referenced_states():
919 if state.action is not None or state.loop is not None:
920 has_diagnostic_predecessor = True
921 break
923 if not has_diagnostic_predecessor:
924 errors.append(
925 ValidationError(
926 message=(
927 f"Failure-named terminal state '{ft_name}' has no predecessor "
928 "state with a diagnostic action. Add a non-terminal diagnostic "
929 "state (e.g. 'diagnose') with an action or sub-loop that routes "
930 f"to '{ft_name}'."
931 ),
932 path=f"states.{ft_name}",
933 severity=ValidationSeverity.WARNING,
934 )
935 )
937 return errors
940def validate_fsm(fsm: FSMLoop) -> list[ValidationError]:
941 """Validate FSM structure and return list of errors.
943 Performs comprehensive validation:
944 - Initial state exists
945 - All referenced states exist
946 - At least one terminal state
947 - Evaluator configurations are valid
948 - Routing configurations are valid
949 - Numeric fields are in valid ranges (max_iterations > 0, backoff >= 0, timeout > 0)
951 Args:
952 fsm: The FSM loop to validate
954 Returns:
955 List of validation errors (empty if valid)
956 """
957 errors: list[ValidationError] = []
958 defined_states = fsm.get_all_state_names()
960 # Warn when no top-level description: field is set. The field is optional
961 # for FSM execution but required for goal-alignment skills (debug-loop-run,
962 # audit-loop-run) and for ll-loop show --json to surface intent text.
963 if not fsm.description:
964 errors.append(
965 ValidationError(
966 path="<root>",
967 message=("No 'description' field defined. Add a top-level description: key."),
968 severity=ValidationSeverity.WARNING,
969 )
970 )
972 # Validate parameters block
973 errors.extend(_validate_parameters(fsm))
975 # Validate targets block (ENH-1552)
976 errors.extend(_validate_targets(fsm))
978 # Check initial state exists
979 if fsm.initial not in defined_states:
980 errors.append(
981 ValidationError(
982 message=f"Initial state '{fsm.initial}' not found in states",
983 path="initial",
984 )
985 )
987 # Check at least one terminal state
988 terminal_states = fsm.get_terminal_states()
989 if not terminal_states:
990 errors.append(
991 ValidationError(
992 message="No terminal state defined. At least one state must have 'terminal: true'",
993 path="states",
994 )
995 )
997 # Validate each state
998 for state_name, state in fsm.states.items():
999 # Check all referenced states exist
1000 refs = state.get_referenced_states()
1001 for ref in refs:
1002 # $current is a special token for retry
1003 if ref != "$current" and ref not in defined_states:
1004 errors.append(
1005 ValidationError(
1006 message=f"References unknown state '{ref}'",
1007 path=f"states.{state_name}",
1008 )
1009 )
1011 # Validate action configuration
1012 errors.extend(_validate_state_action(state_name, state))
1014 # Validate evaluator if present
1015 if state.evaluate is not None:
1016 errors.extend(_validate_evaluator(state_name, state.evaluate))
1018 # Validate routing configuration
1019 errors.extend(_validate_state_routing(state_name, state))
1021 # Check numeric field ranges
1022 if fsm.max_steps <= 0:
1023 errors.append(
1024 ValidationError(
1025 message=f"max_steps must be > 0, got {fsm.max_steps}",
1026 path="max_steps",
1027 )
1028 )
1029 if fsm.max_iterations is not None and fsm.max_iterations <= 0:
1030 errors.append(
1031 ValidationError(
1032 message=f"max_iterations must be > 0, got {fsm.max_iterations}",
1033 path="max_iterations",
1034 )
1035 )
1036 if fsm.max_edge_revisits <= 0:
1037 errors.append(
1038 ValidationError(
1039 message=f"max_edge_revisits must be > 0, got {fsm.max_edge_revisits}",
1040 path="max_edge_revisits",
1041 )
1042 )
1043 if fsm.backoff is not None and fsm.backoff < 0:
1044 errors.append(
1045 ValidationError(
1046 message=f"backoff must be >= 0, got {fsm.backoff}",
1047 path="backoff",
1048 )
1049 )
1050 if fsm.timeout is not None and fsm.timeout <= 0:
1051 errors.append(
1052 ValidationError(
1053 message=f"timeout must be > 0, got {fsm.timeout}",
1054 path="timeout",
1055 )
1056 )
1057 if fsm.llm.max_tokens <= 0:
1058 errors.append(
1059 ValidationError(
1060 message=f"llm.max_tokens must be > 0, got {fsm.llm.max_tokens}",
1061 path="llm.max_tokens",
1062 )
1063 )
1064 if fsm.llm.timeout <= 0:
1065 errors.append(
1066 ValidationError(
1067 message=f"llm.timeout must be > 0, got {fsm.llm.timeout}",
1068 path="llm.timeout",
1069 )
1070 )
1072 # Check for unreachable states (warning only)
1073 reachable = _find_reachable_states(fsm)
1074 unreachable = defined_states - reachable
1075 for state_name in unreachable:
1076 errors.append(
1077 ValidationError(
1078 message="State is not reachable from initial state",
1079 path=f"states.{state_name}",
1080 severity=ValidationSeverity.WARNING,
1081 )
1082 )
1084 errors.extend(_validate_failure_terminal_action(fsm))
1086 errors.extend(_validate_meta_loop_evaluation(fsm))
1088 errors.extend(_validate_input_key_without_guard(fsm))
1090 errors.extend(_validate_artifact_isolation(fsm))
1092 errors.extend(_validate_harness_multimodal_evaluator_blind_spot(fsm))
1094 errors.extend(_validate_partial_route_dead_end(fsm))
1096 errors.extend(_validate_artifact_overwrite(fsm))
1098 errors.extend(_validate_generator_fix_discipline(fsm))
1100 errors.extend(_validate_bash_default_interpolation(fsm))
1102 errors.extend(_validate_overescaped_shell(fsm))
1104 errors.extend(_validate_classify_route_default(fsm))
1106 errors.extend(_validate_zero_retry_counter(fsm))
1108 errors.extend(_validate_on_max_steps(fsm, defined_states))
1109 errors.extend(_validate_on_max_iterations(fsm, defined_states))
1111 errors.extend(_validate_circuit(fsm, defined_states))
1113 errors.extend(_validate_progress_paths_isolation(fsm))
1115 errors.extend(_validate_capture_reachability(fsm))
1117 errors.extend(_validate_llm_evidence_contract(fsm))
1119 return errors
1122def _is_meta_loop(fsm: FSMLoop) -> bool:
1123 """Return True if fsm is classified as a meta-loop.
1125 A loop is meta if ANY of the following match:
1126 1. Any state action string matches a harness-artifact path regex
1127 (writes another loop YAML, skill, agent, command, or project config)
1128 2. The loop's import list contains lib/benchmark.yaml
1129 3. Any state action references yaml_state_editor or replace_action
1130 """
1131 # Condition 2: imports lib/benchmark.yaml
1132 if any(imp in _META_LOOP_IMPORT_TRIGGERS for imp in fsm.imports):
1133 return True
1134 # Conditions 1 and 3: scan action strings
1135 for state in fsm.states.values():
1136 if state.action is None:
1137 continue
1138 for pattern in _META_LOOP_ACTION_PATTERNS:
1139 if pattern.search(state.action):
1140 return True
1141 for token in _META_LOOP_ACTION_TOKENS:
1142 if token in state.action:
1143 return True
1144 return False
1147def _validate_meta_loop_evaluation(fsm: FSMLoop) -> list[ValidationError]:
1148 """Validate meta-loop evaluation rules MR-1 and MR-2.
1150 MR-1 (ERROR): meta-loop must have at least one non-LLM evaluator.
1151 MR-2 (WARNING): meta-loop should reference a captured baseline in an evaluator.
1153 Both rules are suppressed by ``meta_self_eval_ok: true`` at the loop top-level.
1154 """
1155 errors: list[ValidationError] = []
1156 if fsm.meta_self_eval_ok or not _is_meta_loop(fsm):
1157 return errors
1159 # Collect all evaluator types used across all states
1160 evaluator_types: set[str] = set()
1161 for state in fsm.states.values():
1162 if state.evaluate is not None:
1163 evaluator_types.add(state.evaluate.type)
1165 # MR-1: must have at least one non-LLM evaluator
1166 if not evaluator_types & NON_LLM_EVALUATOR_TYPES:
1167 errors.append(
1168 ValidationError(
1169 message=(
1170 "Loop modifies harness artifacts but has no non-LLM evaluator. "
1171 "LLM self-grades on harness updates are unreliable (SHOR Table 1: "
1172 "33-55% accuracy). Pair every check_semantic state with at least one "
1173 "of: exit_code, output_numeric, convergence, diff_stall, action_stall, mcp_result. "
1174 "Note: llm_structured and comparator both use the LLM and do not satisfy MR-1. "
1175 "To suppress with justification, set `meta_self_eval_ok: true` at the "
1176 "loop top-level."
1177 ),
1178 path="<root>",
1179 severity=ValidationSeverity.ERROR,
1180 )
1181 )
1183 # MR-2: should reference a captured baseline in a later evaluator
1184 capture_names: set[str] = {state.capture for state in fsm.states.values() if state.capture}
1185 if capture_names and not _has_baseline_reference(fsm, capture_names):
1186 errors.append(
1187 ValidationError(
1188 message=(
1189 "Meta-loop appears to lack a measure→propose→apply→re-measure "
1190 "spine: no captured baseline value is referenced by a later evaluator. "
1191 "Meta-loops should compare a post-change score against a pre-change "
1192 "baseline (see loops/harness-optimize.yaml as reference template). "
1193 "To suppress, set `meta_self_eval_ok: true`."
1194 ),
1195 path="<root>",
1196 severity=ValidationSeverity.WARNING,
1197 )
1198 )
1200 return errors
1203# Regex patterns for detecting counter-increment actions.
1204# Must contain a printf/echo writing to a file AND an arithmetic increment.
1205_COUNTER_FILE_WRITE_RE = re.compile(r"(?:printf|echo)\s+.*>")
1206_COUNTER_INCREMENT_RE = re.compile(
1207 r"\$\(\(.*\+\s*1\s*\)\)" # $((N + 1)) or $((N+1))
1208 r"|\+\+" # C-style increment
1209 r"|\+=1" # compound assignment
1210 r"|awk\s+.*\+\+" # awk with increment
1211)
1214def _validate_zero_retry_counter(fsm: FSMLoop) -> list[ValidationError]:
1215 """Detect counter + output_numeric combos that yield zero effective retries.
1217 A common loop-authoring footgun: a state increments a counter file and then
1218 evaluates ``output_numeric`` with ``operator: lt, target: 1`` against it.
1219 After the first increment the counter is 1, ``1 < 1 == false``, so the
1220 retry budget is 0 by construction. Author almost always intended target=2.
1221 """
1222 errors: list[ValidationError] = []
1224 for state_name, state in fsm.states.items():
1225 if not state.action or not state.evaluate:
1226 continue
1228 ev = state.evaluate
1229 if ev.type != "output_numeric":
1230 continue
1231 if ev.operator is None or ev.target is None:
1232 continue
1234 # Must be a number-like target for numeric comparison
1235 try:
1236 target = float(ev.target)
1237 except (ValueError, TypeError):
1238 continue
1240 if not _is_counter_action(state.action):
1241 continue
1243 # Check: after first increment (0→1), does operator(1, target) already fail?
1244 op_fn = _NUMERIC_OPERATORS.get(ev.operator)
1245 if op_fn is None:
1246 continue
1248 if not op_fn(1.0, target):
1249 suggested_target = _suggested_target(ev.operator, target)
1250 errors.append(
1251 ValidationError(
1252 message=(
1253 f"Zero retry budget: operator={ev.operator} target={target} "
1254 f"means the first post-increment value (1) already fails "
1255 f"({ev.operator}(1, {target}) == False). "
1256 f"Did you mean target={suggested_target}?"
1257 ),
1258 path=f"states.{state_name}.evaluate",
1259 severity=ValidationSeverity.WARNING,
1260 )
1261 )
1263 return errors
1266def _is_counter_action(action: str) -> bool:
1267 """Return True if the action string contains a counter-increment pattern."""
1268 return bool(_COUNTER_FILE_WRITE_RE.search(action) and _COUNTER_INCREMENT_RE.search(action))
1271def _suggested_target(operator: str, target: float) -> str:
1272 """Suggest a target value that allows at least one retry."""
1273 # For lt/le with a too-low target, suggest target+1 so first post-increment passes
1274 if operator in ("lt", "le"):
1275 return str(int(target) + 1)
1276 # For eq with target=0, suggest 1 so the counter can eventually match
1277 if operator == "eq" and target == 0:
1278 return "1"
1279 # For other cases, suggest target+1 as a default nudge
1280 return str(int(target) + 1)
1283def _validate_harness_multimodal_evaluator_blind_spot(fsm: FSMLoop) -> list[ValidationError]:
1284 """Warn when harness loops use LLM multimodal eval as sole gate to terminal.
1286 LLMs can silently fall back to text-only analysis when reading images,
1287 producing verdicts based on incomplete information. The output_contains
1288 evaluator can verify the LLM wrote the pass string but not that it
1289 actually processed the image. This is the same class of failure as MR-1
1290 (LLM self-evaluation bias) applied to artifact evaluation rather than
1291 harness modification.
1293 Suppressed by ``meta_self_eval_ok: true`` at the loop top-level.
1294 """
1295 errors: list[ValidationError] = []
1296 if fsm.meta_self_eval_ok or fsm.category != "harness":
1297 return errors
1299 terminal_states = fsm.get_terminal_states()
1301 for state_name, state in fsm.states.items():
1302 if state.action_type != "prompt" or not state.action:
1303 continue
1304 if state.evaluate is None or state.evaluate.type != "output_contains":
1305 continue
1306 if not any(p.search(state.action) for p in _MULTIMODAL_EVAL_PATTERNS):
1307 continue
1308 if state.on_yes not in terminal_states:
1309 continue
1310 errors.append(
1311 ValidationError(
1312 message=(
1313 f"State '{state_name}' evaluates a screenshot/image via LLM "
1314 "prompt and routes directly to a terminal on success. The "
1315 "output_contains evaluator can verify the LLM wrote the pass "
1316 "string but not that the LLM actually processed the image. "
1317 "Consider adding a shell-action verification state (e.g., "
1318 "functional smoke test) between scoring and the terminal."
1319 ),
1320 path=f"states.{state_name}",
1321 severity=ValidationSeverity.WARNING,
1322 )
1323 )
1325 return errors
1328def _find_shared_tmp_writes(fsm: FSMLoop) -> list[tuple[str, str]]:
1329 """Return (state_name, matched_path) for every action referencing shared .loops/tmp/.
1331 Scans `state.action` only. Prompts and sub-loop bindings can also reference
1332 paths, but those are out of static-scan reach: action strings are the
1333 place where loop YAMLs directly encode artifact paths.
1334 """
1335 findings: list[tuple[str, str]] = []
1336 for state_name, state in fsm.states.items():
1337 if not state.action:
1338 continue
1339 for match in _SHARED_TMP_PATH_RE.finditer(state.action):
1340 findings.append((state_name, match.group(0)))
1341 return findings
1344def _validate_input_key_without_guard(fsm: FSMLoop) -> list[ValidationError]:
1345 """Warn when a loop sets a custom input_key but omits required_inputs.
1347 A loop that accepts a runtime input via a named key (e.g. input_key: description)
1348 but doesn't declare required_inputs will silently proceed with an empty value if the
1349 user forgets to pass one. Declaring required_inputs shifts that failure to start-time.
1350 """
1351 if fsm.input_key == "input":
1352 return []
1353 if fsm.required_inputs:
1354 return []
1355 return [
1356 ValidationError(
1357 message=(
1358 f"Loop sets input_key: '{fsm.input_key}' but does not declare "
1359 f"required_inputs. If this input is mandatory, add "
1360 f"'required_inputs: [\"{fsm.input_key}\"]' to make the runner "
1361 f"abort when no input is provided."
1362 ),
1363 path="required_inputs",
1364 severity=ValidationSeverity.WARNING,
1365 )
1366 ]
1369def _validate_artifact_isolation(fsm: FSMLoop) -> list[ValidationError]:
1370 """Validate rule MR-3: loops must isolate artifacts to ${context.run_dir}.
1372 The runner injects ${context.run_dir} pointing at .loops/runs/<loop>-<ts>/
1373 and creates the folder before execution. Loops that write intermediate
1374 state (queues, checkpoints, generated files) to shared .loops/tmp/ paths
1375 will corrupt each other under concurrent runs (ll-parallel workers, retries,
1376 repeated invocations).
1378 Suppressed by `shared_state_ok: true` at the loop top-level for loops that
1379 intentionally share state across runs.
1380 """
1381 if fsm.shared_state_ok:
1382 return []
1383 errors: list[ValidationError] = []
1384 for state_name, path in _find_shared_tmp_writes(fsm):
1385 errors.append(
1386 ValidationError(
1387 message=(
1388 f"State writes to shared '{path}' instead of "
1389 "'${context.run_dir}/...'. Concurrent runs of this loop "
1390 "(e.g., under ll-parallel) will corrupt each other's state. "
1391 "Use the runner-injected `${context.run_dir}` for per-run "
1392 "artifact paths, or set `shared_state_ok: true` at the loop "
1393 "top-level if cross-run sharing is intentional."
1394 ),
1395 path=f"states.{state_name}.action",
1396 severity=ValidationSeverity.WARNING,
1397 )
1398 )
1399 return errors
1402def _is_llm_judged(state: StateConfig) -> bool:
1403 """Return True if this state will be graded by the default LLM judge.
1405 Mirrors the action-mode detection in executor._action_mode() (not imported
1406 here because that is runtime code). A state is LLM-judged when:
1407 - it has no explicit evaluate block AND its action is a prompt or slash_command, OR
1408 - it has an explicit evaluate block of type llm_structured or check_semantic.
1409 """
1410 if state.evaluate is None:
1411 # Heuristic: explicit action_type wins; fall back to leading "/" on action string.
1412 action_type = state.action_type
1413 if action_type in ("prompt", "slash_command"):
1414 return True
1415 if action_type is None and state.action and state.action.lstrip().startswith("/"):
1416 return True
1417 return False
1418 return state.evaluate.type in ("llm_structured", "check_semantic")
1421def _validate_partial_route_dead_end(fsm: FSMLoop) -> list[ValidationError]:
1422 """Validate rule MR-4: LLM-judged states with only on_yes have a partial/no dead-end.
1424 A state gated by the default LLM judge can receive yes/no/partial verdicts.
1425 If only on_yes is mapped (and no on_no, on_partial, next, or route table with
1426 a default exist), a partial or no verdict returns None from _route and silently
1427 terminates the loop — the parent treats this as failed.
1429 Suppressed by `partial_route_ok: true` at the loop top-level for the rare
1430 case where dead-ending on a non-yes verdict is intentional.
1431 """
1432 if fsm.partial_route_ok:
1433 return []
1434 errors: list[ValidationError] = []
1435 for state_name, state in fsm.states.items():
1436 if not _is_llm_judged(state):
1437 continue
1438 # States with an unconditional next: or a full route: table are safe.
1439 if state.next is not None or state.route is not None:
1440 continue
1441 # Only flag when on_yes is set but at least one of on_no/on_partial is missing.
1442 if state.on_yes is None:
1443 continue
1444 missing = [v for v in ("no", "partial") if getattr(state, f"on_{v}") is None]
1445 if not missing:
1446 continue
1447 unrouted = " or ".join(f"`{v}`" for v in missing)
1448 errors.append(
1449 ValidationError(
1450 message=(
1451 f"[state: {state_name}] LLM-judged prompt routes only on_yes; "
1452 f"a {unrouted} verdict has no route and will dead-end the loop "
1453 "(parent reads this as failed). Add on_no/on_partial, use `next:` "
1454 "for an unconditional handoff, or a `route:` table with a default. "
1455 "Set `partial_route_ok: true` at the loop top-level to suppress "
1456 "if intentional. (ENH-1917)"
1457 ),
1458 path=f"states.{state_name}",
1459 severity=ValidationSeverity.WARNING,
1460 )
1461 )
1462 return errors
1465def _validate_artifact_overwrite(fsm: FSMLoop) -> list[ValidationError]:
1466 """Validate rule MR-5 (ENH-1957): harness loops should version artifacts per iteration.
1468 A harness-category loop that iteratively generates and overwrites a flat artifact
1469 path (e.g. ``${context.run_dir}/image.svg``) loses all intermediate versions.
1470 Only the final iteration survives. This rule flags iterative generate→evaluate→generate
1471 cycles that write to artifact paths without declaring versioning intent.
1473 Suppressed by ``artifact_versioning: true`` (loop snapshots per-iteration artifacts)
1474 or ``artifact_versioning_ok: true`` (intentional overwrite, e.g. artifact varies
1475 by task).
1476 """
1477 if fsm.artifact_versioning or fsm.artifact_versioning_ok:
1478 return []
1479 if fsm.category not in ("harness",):
1480 return []
1482 errors: list[ValidationError] = []
1484 # Find states that write to artifact paths (shell actions with file output)
1485 writers: dict[str, set[str]] = {} # state_name -> set of artifact paths
1486 for state_name, state in fsm.states.items():
1487 if not state.action or state.action_type not in ("shell", None):
1488 continue
1489 # Skip sub-loop delegation states
1490 if state.action_type == "loop":
1491 continue
1492 action = state.action
1493 # Find run_dir-based artifact writes: ${context.run_dir}/<path> or $RUNDIR/<path>
1494 import re
1496 # Match patterns like: ${context.run_dir}/output.svg, $RUNDIR/image.png, > path
1497 # We look for output redirections or cp/mv commands writing to run_dir
1498 artifact_refs = set()
1499 for pattern in (
1500 r'\$\{context\.run_dir\}/([^\s"\';&]+)',
1501 r'\$\{captured\.[^}]+\}/([^\s"\';&]+)',
1502 ):
1503 for m in re.finditer(pattern, action):
1504 artifact_refs.add(m.group(1))
1505 # Also detect explicit cp/mv writing to run_dir paths
1506 for m in re.finditer(r'(?:cp|mv)\s+.*\s+\$\{context\.run_dir\}/([^\s"\';&]+)', action):
1507 artifact_refs.add(m.group(1))
1508 if artifact_refs:
1509 writers[state_name] = artifact_refs
1511 if not writers:
1512 return []
1514 # Detect iterative cycles: a writer state that is reachable from itself
1515 # via a non-trivial path through other states (generate → evaluate → generate)
1516 for state_name in writers:
1517 refs = fsm.states[state_name].get_referenced_states()
1518 # Check if this writer or its downstream states loop back to this writer
1519 visited: set[str] = set()
1520 to_visit = list(refs - {"$current"})
1521 while to_visit:
1522 target = to_visit.pop()
1523 if target in visited or target not in fsm.states:
1524 continue
1525 visited.add(target)
1526 if target == state_name:
1527 # Found a cycle: this writer state is reachable from itself
1528 artifact_list = ", ".join(sorted(writers[state_name]))
1529 errors.append(
1530 ValidationError(
1531 message=(
1532 f"[state: {state_name}] Harness loop writes artifact(s) "
1533 f"({artifact_list}) to a flat path in an iterative cycle "
1534 f"({state_name} → ... → {state_name}). Per-iteration versions "
1535 "are lost; only the final output survives. Add per-iteration "
1536 "snapshots (see oracle generator-evaluator for pattern) and "
1537 "declare `artifact_versioning: true`, or set "
1538 "`artifact_versioning_ok: true` if intentional. (ENH-1957)"
1539 ),
1540 path=f"states.{state_name}",
1541 severity=ValidationSeverity.WARNING,
1542 )
1543 )
1544 break
1545 target_state = fsm.states.get(target)
1546 if target_state is not None:
1547 target_refs = target_state.get_referenced_states()
1548 for r in target_refs:
1549 if r != "$current" and r not in visited:
1550 to_visit.append(r)
1552 return errors
1555def _validate_generator_fix_discipline(fsm: FSMLoop) -> list[ValidationError]:
1556 """Validate rule MR-6 (ENH-2079): meta-loops should not hand-patch generator artifacts.
1558 Detects the hand-patching anti-pattern: a ``shell``-type state that writes to the
1559 same file path as a non-shell (LLM-type) generator state in the same loop.
1560 Hand-patching creates fragile output that diverges from the generator on the next
1561 run; the stable fix is to update the generator action instead.
1563 Suppressed by ``generator_fix_ok: true`` at the loop top-level for intentional
1564 post-processing cases.
1565 """
1566 if fsm.generator_fix_ok or not _is_meta_loop(fsm):
1567 return []
1569 # Markers that indicate a prompt/slash_command state is generating file artifacts
1570 _GENERATOR_MARKERS = ("yaml_state_editor", "replace_action", "to_file:")
1572 _PATH_PATTERNS = (
1573 re.compile(r'\$\{context\.run_dir\}/([^\s"\';&|]+)'),
1574 re.compile(r'\$\{captured\.[^}]+\}/([^\s"\';&|]+)'),
1575 )
1577 def _extract_paths(action: str) -> set[str]:
1578 paths: set[str] = set()
1579 for pat in _PATH_PATTERNS:
1580 for m in pat.finditer(action):
1581 paths.add(m.group(1).rstrip("/"))
1582 return paths
1584 shell_targets: dict[str, set[str]] = {} # state_name -> set of file paths
1585 generator_targets: dict[str, set[str]] = {} # state_name -> set of file paths
1587 for state_name, state in fsm.states.items():
1588 if not state.action:
1589 continue
1590 action = state.action
1591 paths = _extract_paths(action)
1592 if not paths:
1593 continue
1594 action_type = state.action_type
1595 if action_type in ("shell", None):
1596 shell_targets[state_name] = paths
1597 elif action_type in ("prompt", "slash_command"):
1598 if any(marker in action for marker in _GENERATOR_MARKERS):
1599 generator_targets[state_name] = paths
1601 errors: list[ValidationError] = []
1602 for gen_name, gen_paths in generator_targets.items():
1603 for shell_name, shell_paths in shell_targets.items():
1604 overlap = gen_paths & shell_paths
1605 if overlap:
1606 artifact_list = ", ".join(sorted(overlap))
1607 errors.append(
1608 ValidationError(
1609 message=(
1610 f"[states: {gen_name}, {shell_name}] Hand-patching anti-pattern: "
1611 f"LLM-generator state '{gen_name}' and shell state '{shell_name}' "
1612 f"both write to ({artifact_list}). Move the fix into the generator "
1613 "action so every run produces correct output automatically. "
1614 "Set `generator_fix_ok: true` to suppress for intentional "
1615 "post-processing. (ENH-2079)"
1616 ),
1617 path=f"states.{shell_name}",
1618 severity=ValidationSeverity.WARNING,
1619 )
1620 )
1622 return errors
1625def _find_bash_default_tokens(fsm: FSMLoop) -> list[tuple[str, str]]:
1626 """Return (state_name, matched_token) for every action with unescaped ${ns.path:-...}.
1628 Scans ``state.action`` only. The escaped form ``$${VAR:-value}`` is exempted by
1629 the negative lookbehind in ``_BASH_DEFAULT_RE``; the engine-native ``:default=``
1630 form does not contain ``:-`` and is therefore never matched.
1631 """
1632 findings: list[tuple[str, str]] = []
1633 for state_name, state in fsm.states.items():
1634 if not state.action:
1635 continue
1636 for match in _BASH_DEFAULT_RE.finditer(state.action):
1637 findings.append((state_name, match.group(0)))
1638 return findings
1641def _validate_bash_default_interpolation(fsm: FSMLoop) -> list[ValidationError]:
1642 """Validate rule MR-7 (ENH-2348): unescaped bash :-default interpolation.
1644 The FSM interpolation engine supports ``${namespace.path:default=value}`` for
1645 author-specified defaults, but does NOT support the bash parameter-expansion form
1646 ``${namespace.path:-value}``. Any loop action that uses the bash form will crash
1647 at runtime with ``Path 'ns.path:-value' not found in context``.
1649 Supported alternatives:
1650 - ``${context.x:default=queue}`` — engine-native default (always preferred)
1651 - ``$${VAR:-value}`` — escaped, passed to the shell verbatim
1653 Suppressed by ``bash_default_ok: true`` at the loop top-level for the rare case
1654 where an author can justify the unsupported form.
1655 """
1656 if fsm.bash_default_ok:
1657 return []
1658 errors: list[ValidationError] = []
1659 for state_name, token in _find_bash_default_tokens(fsm):
1660 errors.append(
1661 ValidationError(
1662 message=(
1663 f"[state: {state_name}] {token} uses unsupported bash ':-' default. "
1664 "The FSM interpolator will crash at runtime with 'Path not found in context'. "
1665 f"Use {token.split(':-')[0]}:default={token.split(':-')[1].rstrip('}')}}}"
1666 " (engine default) or $${{VAR:-value}} (shell, escaped). "
1667 "Set `bash_default_ok: true` to suppress. (ENH-2348)"
1668 ),
1669 path=f"states.{state_name}.action",
1670 severity=ValidationSeverity.ERROR,
1671 )
1672 )
1673 return errors
1676def _validate_overescaped_shell(fsm: FSMLoop) -> list[ValidationError]:
1677 """Validate rule MR-9: over-escaped shell ``$$`` that expands to the PID.
1679 The FSM interpolation engine only rewrites the brace form ``$${...}`` → ``${...}``;
1680 bare ``$(...)`` command substitution and ``$VAR`` references are passed through to
1681 ``bash -c`` untouched. Doubling them is therefore NOT an escape — ``$$(`` and
1682 ``$$VAR`` reach the shell literally, where the leading ``$$`` expands to the process
1683 ID. An ``init`` state that does ``echo "$$(pwd)/$$DIR"`` captures ``<pid>(pwd)/<pid>DIR``
1684 instead of an absolute path, silently corrupting every downstream ``${captured…}``
1685 reference (observed in interactive-/html-/svg-generator).
1687 Correct forms:
1688 - ``$(pwd)`` / ``$DIR`` — command substitution / variable (shell handles them)
1689 - ``$${VAR}`` / ``$${VAR:-x}`` — ONLY brace vars collide with ``${ns.path}`` and
1690 need the ``$$`` escape; the interpolator converts ``$${`` → ``${``.
1692 Scans shell actions only (``action_type`` shell/untyped, excluding slash commands);
1693 ``$$VAR`` in a prompt is inert text. Suppressed by ``shell_pid_ok: true`` at the
1694 loop top-level for the rare case where a literal PID is intended.
1695 """
1696 if fsm.shell_pid_ok:
1697 return []
1698 errors: list[ValidationError] = []
1699 for state_name, state in fsm.states.items():
1700 if not state.action:
1701 continue
1702 # Shell-only: prompt/slash-command actions never reach `bash -c`, so a `$$`
1703 # there is harmless text. Mirror the shell gate used by other shell rules.
1704 if state.action_type not in ("shell", None):
1705 continue
1706 if state.action.lstrip().startswith("/"):
1707 continue
1708 for match in _OVERESCAPED_SHELL_RE.finditer(state.action):
1709 # Show the offending token plus the next char for context (e.g. `$$(` / `$$D`).
1710 start = match.start()
1711 token = state.action[start : start + 3]
1712 errors.append(
1713 ValidationError(
1714 message=(
1715 f"[state: {state_name}] '{token}' over-escapes shell `$$`. "
1716 "The interpolator only converts the brace form $${{...}}; bare "
1717 "$(...) and $VAR pass through untouched, so $$ here expands to the "
1718 "PID at `bash -c` time (e.g. $$(pwd)/$$DIR -> <pid>(pwd)/<pid>DIR). "
1719 "Use single $ ($(pwd), $DIR); reserve $$ for the $${{VAR}} brace "
1720 "escape. Set `shell_pid_ok: true` to suppress. (MR-9)"
1721 ),
1722 path=f"states.{state_name}.action",
1723 severity=ValidationSeverity.ERROR,
1724 )
1725 )
1726 return errors
1729_EVIDENCE_CONTRACT_KEYWORDS: frozenset[str] = frozenset({"verbatim", "quote", "evidence"})
1732def _validate_llm_evidence_contract(fsm: FSMLoop) -> list[ValidationError]:
1733 """Validate rule MR-8 (ENH-2342): LLM-judged state prompts should include evidence-contract keywords.
1735 A check_semantic/llm_structured state whose ``evaluate.prompt`` does not contain any of
1736 ``{"verbatim", "quote", "evidence"}`` may produce verdicts without cited output text,
1737 defaulting to optimism (SHOR Table 1: 33–55% accuracy; Sonnet 4.6 = 33.4%).
1739 States with no evaluate block (Path B: action prompt) or ``evaluate.prompt is None``
1740 inherit DEFAULT_LLM_PROMPT which includes CHECK_SEMANTIC_EVIDENCE_CONTRACT after ENH-2342
1741 and are not flagged here.
1743 Suppressed by ``evidence_contract_ok: true`` at the loop top-level.
1744 """
1745 if fsm.evidence_contract_ok:
1746 return []
1747 errors: list[ValidationError] = []
1748 for state_name, state in fsm.states.items():
1749 if not _is_llm_judged(state):
1750 continue
1751 if state.evaluate is None:
1752 continue
1753 if state.evaluate.prompt is None:
1754 continue
1755 prompt_lower = state.evaluate.prompt.lower()
1756 if not any(kw in prompt_lower for kw in _EVIDENCE_CONTRACT_KEYWORDS):
1757 errors.append(
1758 ValidationError(
1759 message=(
1760 f"[state: {state_name}] check_semantic/llm_structured prompt "
1761 "does not include evidence-contract keywords (verbatim, quote, evidence). "
1762 "Verdicts without cited evidence default to optimism "
1763 "(SHOR Table 1: 33–55% accuracy). Add a requirement to quote exact "
1764 "output text, or set `evidence_contract_ok: true` to suppress. "
1765 "(ENH-2342 MR-8)"
1766 ),
1767 path=f"states.{state_name}.evaluate.prompt",
1768 severity=ValidationSeverity.WARNING,
1769 )
1770 )
1771 return errors
1774def _validate_classify_route_default(fsm: FSMLoop) -> list[ValidationError]:
1775 """Validate that classify states with a route: table include a default: fallback.
1777 A ``classify`` state whose ``route:`` table has no ``default:`` will dead-end
1778 whenever the action emits a token not listed in the table. This rule flags
1779 that gap as a WARNING so loop authors add a catch-all branch.
1781 Suppressed by ``partial_route_ok: true`` at the loop top-level when a
1782 dead-end on an unlisted token is intentional.
1783 """
1784 if fsm.partial_route_ok:
1785 return []
1786 errors: list[ValidationError] = []
1787 for state_name, state in fsm.states.items():
1788 if state.evaluate is None or state.evaluate.type != "classify":
1789 continue
1790 if state.route is None or state.route.default is not None:
1791 continue
1792 errors.append(
1793 ValidationError(
1794 message=(
1795 f"[state: {state_name}] classify route: table has no default: — "
1796 "unknown tokens will dead-end the loop. Add a default: catch-all, "
1797 "or set `partial_route_ok: true` at the loop top-level to suppress."
1798 ),
1799 path=f"states.{state_name}",
1800 severity=ValidationSeverity.WARNING,
1801 )
1802 )
1803 return errors
1806def _has_baseline_reference(fsm: FSMLoop, capture_names: set[str]) -> bool:
1807 """Return True if any evaluate block references a captured variable."""
1808 for state in fsm.states.values():
1809 ev = state.evaluate
1810 if ev is None:
1811 continue
1812 # Check string fields that may interpolate captured values
1813 candidates = [ev.previous, ev.source]
1814 if isinstance(ev.target, str):
1815 candidates.append(ev.target)
1816 for field_val in candidates:
1817 if not field_val:
1818 continue
1819 for name in capture_names:
1820 if f"captured.{name}" in field_val:
1821 return True
1822 return False
1825def _validate_on_max_steps(fsm: FSMLoop, defined_states: set[str]) -> list[ValidationError]:
1826 """Validate the top-level `on_max_steps` field (BUG-2204).
1828 Checks that the named state exists when `on_max_steps` is set.
1829 """
1830 errors: list[ValidationError] = []
1831 if fsm.on_max_steps is None:
1832 return errors
1833 if fsm.on_max_steps not in defined_states:
1834 errors.append(
1835 ValidationError(
1836 message=(
1837 f"on_max_steps references unknown state "
1838 f"'{fsm.on_max_steps}' (must be a declared state)"
1839 ),
1840 path="on_max_steps",
1841 )
1842 )
1843 return errors
1846def _validate_on_max_iterations(fsm: FSMLoop, defined_states: set[str]) -> list[ValidationError]:
1847 """Validate the top-level `on_max_iterations` field (BUG-2204: iteration-cap summary state).
1849 Checks that the named state exists when `on_max_iterations` is set.
1850 """
1851 errors: list[ValidationError] = []
1852 if fsm.on_max_iterations is None:
1853 return errors
1854 if fsm.on_max_iterations not in defined_states:
1855 errors.append(
1856 ValidationError(
1857 message=(
1858 f"on_max_iterations references unknown state "
1859 f"'{fsm.on_max_iterations}' (must be a declared state)"
1860 ),
1861 path="on_max_iterations",
1862 )
1863 )
1864 return errors
1867def _validate_circuit(fsm: FSMLoop, defined_states: set[str]) -> list[ValidationError]:
1868 """Validate the top-level `circuit:` block (FEAT-1637).
1870 Checks:
1871 - `circuit.repeated_failure.window` is a positive integer.
1872 - `circuit.repeated_failure.on_repeated_failure` is either the special
1873 token ``"abort"`` or the name of a declared state.
1874 """
1875 errors: list[ValidationError] = []
1876 if fsm.circuit is None or fsm.circuit.repeated_failure is None:
1877 return errors
1879 rf = fsm.circuit.repeated_failure
1880 if rf.window < 1:
1881 errors.append(
1882 ValidationError(
1883 message=f"circuit.repeated_failure.window must be >= 1, got {rf.window}",
1884 path="circuit.repeated_failure.window",
1885 )
1886 )
1888 if rf.recurrent_window is not None and rf.recurrent_window < 2:
1889 errors.append(
1890 ValidationError(
1891 message=(
1892 f"circuit.repeated_failure.recurrent_window must be >= 2, "
1893 f"got {rf.recurrent_window}"
1894 ),
1895 path="circuit.repeated_failure.recurrent_window",
1896 )
1897 )
1899 target = rf.on_repeated_failure
1900 if target not in STALL_SPECIAL_TOKENS and target not in defined_states:
1901 errors.append(
1902 ValidationError(
1903 message=(
1904 f"circuit.repeated_failure.on_repeated_failure references "
1905 f"unknown state '{target}' (must be a declared state or "
1906 f'the literal "abort")'
1907 ),
1908 path="circuit.repeated_failure.on_repeated_failure",
1909 )
1910 )
1912 return errors
1915# Matches common interpolation prefixes used in loop YAML paths so we can
1916# extract the portable relative component for action-string scanning.
1917_INTERPOLATION_PREFIX_RE = re.compile(r"^\$\{[^}]+\}/")
1920def _strip_interpolation_prefix(path: str) -> str:
1921 """Return the path with any leading ${...}/ prefix removed."""
1922 return _INTERPOLATION_PREFIX_RE.sub("", path)
1925def _validate_progress_paths_isolation(fsm: FSMLoop) -> list[ValidationError]:
1926 """Warn when a state's action writes to a file listed in progress_paths (BUG-1767).
1928 When a loop's own bookkeeping files appear in both progress_paths and the
1929 state action strings, every append to those files resets the stall window,
1930 silently disabling the BUG-1674 stall guard for that loop. Authors should
1931 move such files to exclude_paths so the stall detector can still fire.
1932 """
1933 if fsm.circuit is None or fsm.circuit.repeated_failure is None:
1934 return []
1935 rf = fsm.circuit.repeated_failure
1936 if not rf.progress_paths:
1937 return []
1939 # Build a set of the relative path components we need to look for.
1940 watched = {_strip_interpolation_prefix(p) for p in rf.progress_paths}
1941 # Exclude paths that are already in exclude_paths — author acknowledged.
1942 excluded = {_strip_interpolation_prefix(p) for p in rf.exclude_paths}
1943 active_watched = watched - excluded
1944 if not active_watched:
1945 return []
1947 errors: list[ValidationError] = []
1948 for state_name, state in fsm.states.items():
1949 if not state.action:
1950 continue
1951 for path_fragment in active_watched:
1952 if path_fragment in state.action:
1953 errors.append(
1954 ValidationError(
1955 message=(
1956 f"State action references '{path_fragment}', which is also "
1957 "listed in circuit.repeated_failure.progress_paths. Writes "
1958 "to this file will reset the stall window every cycle, "
1959 "silently disabling stall detection. Move it to "
1960 "circuit.repeated_failure.exclude_paths to separate "
1961 "bookkeeping files from real progress signals."
1962 ),
1963 path=f"states.{state_name}.action",
1964 severity=ValidationSeverity.WARNING,
1965 )
1966 )
1967 return errors
1970def _dominated_by_any(fsm: FSMLoop, dominators: set[str], dominated: str) -> bool:
1971 """Return True if the set ``dominators`` collectively dominates ``dominated``.
1973 Group domination: every path from the initial state to ``dominated`` must
1974 pass through at least one state in ``dominators``. Checked by removing all
1975 dominator states from the graph and testing whether ``dominated`` is still
1976 reachable from the initial state.
1978 This generalizes single-state domination — used when a capture variable is
1979 produced by more than one state on mutually-exclusive branches, where the
1980 reference is safe as long as *some* capturing state runs on every path.
1982 Args:
1983 fsm: The FSM loop to analyze
1984 dominators: Names of the states that should collectively dominate
1985 dominated: Name of the state that should be dominated
1987 Returns:
1988 True if the dominators collectively dominate ``dominated``
1989 """
1990 if dominated in dominators:
1991 return True
1992 if dominated not in fsm.states:
1993 return False
1995 visited: set[str] = set()
1996 to_visit: deque[str] = deque([fsm.initial])
1998 while to_visit:
1999 current = to_visit.popleft()
2000 if current in visited or current not in fsm.states:
2001 continue
2002 if current in dominators:
2003 continue # Block this node (simulate removal)
2005 visited.add(current)
2007 if current == dominated:
2008 # Reached dominated without going through any dominator
2009 return False
2011 state = fsm.states[current]
2012 for ref in state.get_referenced_states():
2013 if ref != "$current" and ref not in visited:
2014 to_visit.append(ref)
2016 # Dominated not reachable without the dominators → they dominate
2017 return True
2020def _dominates(fsm: FSMLoop, dominator: str, dominated: str) -> bool:
2021 """Return True if dominator dominates dominated in the FSM graph.
2023 A state D dominates S if every path from the initial state to S must pass
2024 through D. Thin single-state wrapper around :func:`_dominated_by_any`.
2025 """
2026 if dominator not in fsm.states:
2027 return False
2028 return _dominated_by_any(fsm, {dominator}, dominated)
2031def _find_bypass_path_any(fsm: FSMLoop, dominators: set[str], dominated: str) -> list[str]:
2032 """Find an example path from initial to dominated that bypasses all dominators.
2034 Uses BFS to find the shortest path that avoids every state in ``dominators``.
2035 Returns empty list if no bypass exists (should not happen when called after
2036 :func:`_dominated_by_any` returns False).
2037 """
2038 parent: dict[str, str] = {}
2039 to_visit: deque[str] = deque([fsm.initial])
2040 visited: set[str] = set()
2042 while to_visit:
2043 current = to_visit.popleft()
2044 if current in visited or current not in fsm.states:
2045 continue
2046 if current in dominators:
2047 continue
2049 visited.add(current)
2051 if current == dominated:
2052 # Reconstruct path
2053 path = [dominated]
2054 while path[-1] in parent:
2055 path.append(parent[path[-1]])
2056 path.reverse()
2057 return path
2059 state = fsm.states[current]
2060 for ref in state.get_referenced_states():
2061 if ref != "$current" and ref not in visited:
2062 if ref not in parent:
2063 parent[ref] = current
2064 to_visit.append(ref)
2066 return []
2069def _find_bypass_path(fsm: FSMLoop, dominator: str, dominated: str) -> list[str]:
2070 """Find an example path from initial to dominated that bypasses dominator.
2072 Thin single-state wrapper around :func:`_find_bypass_path_any`.
2073 """
2074 return _find_bypass_path_any(fsm, {dominator}, dominated)
2077def _has_sub_loop_state(fsm: FSMLoop) -> bool:
2078 """Return True if any state in the FSM has ``loop:`` set (delegates to a child loop).
2080 Used by ENH-1961 to distinguish "capture lives in a sub-loop" from "capture is missing".
2081 """
2082 return any(state.loop is not None for state in fsm.states.values())
2085def _validate_capture_reachability(fsm: FSMLoop) -> list[ValidationError]:
2086 """Validate that ``${captured.*}`` references are dominated by their capturing states.
2088 ENH-1961: For each state that references ``${captured.<var>.*}`` in its action
2089 or evaluate source, checks that the capturing state dominates the referencing
2090 state (i.e., all paths from the initial state pass through the capture state).
2092 Emits:
2093 - WARNING when a capture state does not dominate a referencing state
2094 (the reference may crash at runtime on paths that bypass the capture).
2095 - ERROR when a referenced capture variable has no capturing state at all
2096 in this FSM (excluding sub-loop captures which live in child namespaces).
2097 """
2098 errors: list[ValidationError] = []
2100 # Step 1: Build capture map (var_name → set of capturing state names).
2101 # A variable may be captured by more than one state on mutually-exclusive
2102 # branches (e.g. fifo_pop vs select_next dispatched by schedule_mode); the
2103 # reference is safe as long as the *set* collectively dominates it.
2104 capture_map: dict[str, set[str]] = {}
2105 for state_name, state in fsm.states.items():
2106 if state.capture:
2107 capture_map.setdefault(state.capture, set()).add(state_name)
2109 # Step 2: Build reference map (state_name → set of captured var names referenced)
2110 reference_map: dict[str, set[str]] = {}
2111 for state_name, state in fsm.states.items():
2112 # Skip sub-loop delegation states — their action is a loop name,
2113 # and captured vars belong to the child loop's namespace.
2114 if state.loop is not None:
2115 continue
2117 # Only collect references NOT guarded by `:default=` — a guarded
2118 # reference is safe even when the capture is missing on some path.
2119 refs: set[str] = set()
2120 if state.action:
2121 refs.update(_unguarded_captured_refs(state.action))
2122 if state.evaluate is not None and state.evaluate.source:
2123 refs.update(_unguarded_captured_refs(state.evaluate.source))
2124 if refs:
2125 reference_map[state_name] = refs
2127 if not reference_map:
2128 return errors
2130 # Step 3: For each reference, check dominance of capturing state
2131 for ref_state_name, ref_vars in reference_map.items():
2132 for var_name in ref_vars:
2133 if var_name not in capture_map:
2134 # Referenced capture variable has no capturing state in this FSM.
2135 # ENH-1998: downgrade to WARNING (not silent skip) when sub-loops
2136 # are present — the capture may live in a child namespace, but a
2137 # typo'd name should still surface rather than go completely dark.
2138 if _has_sub_loop_state(fsm):
2139 errors.append(
2140 ValidationError(
2141 message=(
2142 f"References ${{captured.{var_name}.*}} but no state in "
2143 f"this loop captures '{var_name}'. "
2144 f"If '{var_name}' is produced by a sub-loop, this may be "
2145 f"intentional; otherwise add 'capture: {var_name}' to the "
2146 f"state that produces this value."
2147 ),
2148 path=f"states.{ref_state_name}.action",
2149 severity=ValidationSeverity.WARNING,
2150 )
2151 )
2152 continue
2153 # No sub-loops: this is genuinely missing.
2154 errors.append(
2155 ValidationError(
2156 message=(
2157 f"References ${{captured.{var_name}.*}} but no state in "
2158 f"this loop captures '{var_name}'. Add 'capture: {var_name}' "
2159 f"to the state that produces this value."
2160 ),
2161 path=f"states.{ref_state_name}.action",
2162 severity=ValidationSeverity.ERROR,
2163 )
2164 )
2165 continue
2167 # Capturing states present in this FSM (shouldn't drop any normally)
2168 cap_states = {s for s in capture_map[var_name] if s in fsm.states}
2169 if not cap_states:
2170 continue
2172 # Group dominance check: do the capturing states collectively
2173 # dominate ref_state_name (does at least one run on every path)?
2174 if not _dominated_by_any(fsm, cap_states, ref_state_name):
2175 bypass_path = _find_bypass_path_any(fsm, cap_states, ref_state_name)
2176 path_str = " → ".join(bypass_path) if bypass_path else "unknown path"
2178 if len(cap_states) == 1:
2179 captured_by = f"state '{next(iter(cap_states))}' which may not"
2180 else:
2181 names = ", ".join(f"'{s}'" for s in sorted(cap_states))
2182 captured_by = f"states {names}, none of which"
2184 errors.append(
2185 ValidationError(
2186 message=(
2187 f"References ${{captured.{var_name}.*}} but '{var_name}' "
2188 f"is captured by {captured_by} "
2189 f"execute on all paths to '{ref_state_name}'. "
2190 f"Path(s) bypassing capture: {path_str}"
2191 ),
2192 path=f"states.{ref_state_name}.action",
2193 severity=ValidationSeverity.WARNING,
2194 )
2195 )
2197 return errors
2200def _find_reachable_states(fsm: FSMLoop) -> set[str]:
2201 """Find all states reachable from the initial state.
2203 Uses breadth-first search to find all reachable states. Seeds the BFS
2204 with the initial state plus top-level transition targets that act as
2205 alternate entry points: ``on_max_iterations`` (fires when the iteration
2206 cap is hit) and ``circuit.repeated_failure.on_repeated_failure`` (fires
2207 when the circuit breaker trips). These are real edges the runtime can
2208 take, so states reached only through them are not orphans.
2210 Args:
2211 fsm: The FSM loop to analyze
2213 Returns:
2214 Set of reachable state names
2215 """
2216 reachable: set[str] = set()
2217 to_visit: deque[str] = deque([fsm.initial])
2218 if fsm.on_max_steps is not None:
2219 to_visit.append(fsm.on_max_steps)
2220 if fsm.on_max_iterations is not None:
2221 to_visit.append(fsm.on_max_iterations)
2222 if fsm.circuit is not None and fsm.circuit.repeated_failure is not None:
2223 target = fsm.circuit.repeated_failure.on_repeated_failure
2224 if target not in STALL_SPECIAL_TOKENS:
2225 to_visit.append(target)
2227 while to_visit:
2228 current = to_visit.popleft()
2229 if current in reachable or current not in fsm.states:
2230 continue
2232 reachable.add(current)
2233 state = fsm.states[current]
2234 refs = state.get_referenced_states()
2236 for ref in refs:
2237 if ref != "$current" and ref not in reachable:
2238 to_visit.append(ref)
2240 return reachable
2243def is_runnable_loop(path: Path) -> bool:
2244 """Cheap check for whether a YAML file is a runnable FSM loop definition.
2246 Returns True iff the file parses as a YAML mapping with the required
2247 top-level keys ``name``, ``initial``, and either ``states`` or ``flow``
2248 (the shorthand resolved by :func:`resolve_flow`). This matches the
2249 required-fields gate in :func:`load_and_validate` so "counted by the
2250 verifier" stays in sync with "runnable by ll-loop validate".
2252 When the raw YAML contains a ``from:`` key, inheritance is resolved first
2253 (mirroring :func:`load_and_validate`) so pure context-override stubs whose
2254 parent provides ``initial``/``states`` return True. Library fragments under
2255 ``loops/lib/`` still return False — their parent chain also lacks ``initial``.
2256 """
2257 try:
2258 data = yaml.safe_load(path.read_text())
2259 except (OSError, yaml.YAMLError):
2260 return False
2261 if not isinstance(data, dict):
2262 return False
2263 if "from" in data:
2264 try:
2265 data = resolve_inheritance(data, path.parent)
2266 except Exception:
2267 return False
2268 has_flow = "states" in data or "flow" in data
2269 return "name" in data and "initial" in data and has_flow
2272def load_and_validate(
2273 path: Path,
2274 raise_on_error: bool = True,
2275) -> tuple[FSMLoop, list[ValidationError]]:
2276 """Load YAML file and validate FSM structure.
2278 Args:
2279 path: Path to the YAML file to load
2280 raise_on_error: When True (default), raise ValueError on ERROR violations.
2281 When False, return all violations (errors + warnings) without raising.
2283 Returns:
2284 When raise_on_error=True: (FSMLoop, list of WARNING-severity ValidationErrors)
2285 When raise_on_error=False: (FSMLoop, list of all ValidationErrors sorted errors-first)
2287 Raises:
2288 FileNotFoundError: If the file doesn't exist
2289 yaml.YAMLError: If the file is not valid YAML
2290 ValueError: If raise_on_error=True and validation fails (contains error details)
2291 """
2292 if not path.exists():
2293 raise FileNotFoundError(f"FSM file not found: {path}")
2295 with open(path) as f:
2296 data: dict[str, Any] = yaml.safe_load(f)
2298 if not isinstance(data, dict):
2299 raise ValueError(f"FSM file must contain a YAML mapping, got {type(data)}")
2301 # Resolve `from:` inheritance before any further checks, so a child loop
2302 # can omit fields its parent provides (including `initial`/`states`) and
2303 # so a parent's `import:`/`fragments:` blocks survive into the merged
2304 # result for the subsequent `resolve_fragments` pass.
2305 data = resolve_inheritance(data, path.parent)
2307 # Expand flow: linear shorthand into states: before required-fields check
2308 data = resolve_flow(data)
2310 # Check required fields before parsing
2311 missing = []
2312 for field in ["name", "initial"]:
2313 if field not in data:
2314 missing.append(field)
2315 if "states" not in data:
2316 missing.append("states (or flow)")
2318 if missing:
2319 raise ValueError(f"FSM file missing required fields: {', '.join(missing)}")
2321 # Check for unknown top-level keys before parsing
2322 unknown_key_warnings: list[ValidationError] = []
2323 unknown = set(data.keys()) - KNOWN_TOP_LEVEL_KEYS
2324 if unknown:
2325 unknown_key_warnings.append(
2326 ValidationError(
2327 path="<root>",
2328 message=f"Unknown top-level keys: {', '.join(sorted(unknown))}",
2329 severity=ValidationSeverity.WARNING,
2330 )
2331 )
2333 visibility_val = data.get("visibility")
2334 if visibility_val is not None and visibility_val not in VALID_VISIBILITY:
2335 unknown_key_warnings.append(
2336 ValidationError(
2337 path="visibility",
2338 message=(
2339 f"Invalid visibility: {visibility_val!r}. "
2340 f"Must be one of: {', '.join(sorted(VALID_VISIBILITY))}. "
2341 "Loop will be treated as 'public'."
2342 ),
2343 severity=ValidationSeverity.WARNING,
2344 )
2345 )
2347 # Resolve fragment libraries before parsing into dataclass
2348 data = resolve_fragments(data, path.parent)
2350 # Parse into dataclass
2351 fsm = FSMLoop.from_dict(data)
2353 # Validate
2354 errors = validate_fsm(fsm)
2356 # Validate with: bindings against child loop parameters (requires file-system access)
2357 errors.extend(_validate_with_bindings(fsm, path.parent))
2358 errors.extend(_validate_loop_references(fsm, path.parent))
2359 errors.extend(_validate_fragment_bindings(fsm, path.parent))
2361 # Filter to errors only (not warnings) for raising
2362 error_list = [e for e in errors if e.severity == ValidationSeverity.ERROR]
2363 struct_warnings = [e for e in errors if e.severity == ValidationSeverity.WARNING]
2364 all_warnings = unknown_key_warnings + struct_warnings
2366 if not raise_on_error:
2367 return fsm, error_list + all_warnings
2369 if error_list:
2370 error_messages = "\n ".join(str(e) for e in error_list)
2371 raise ValueError(f"FSM validation failed:\n {error_messages}")
2373 for warning in all_warnings:
2374 logger.warning(str(warning))
2376 return fsm, all_warnings