Coverage for little_loops / fsm / validation.py: 10%
765 statements
« prev ^ index » next coverage.py v7.12.0, created at 2026-06-26 17:38 -0500
1"""FSM loop validation logic.
3This module provides validation for FSM loop definitions, ensuring
4structural correctness and catching common configuration errors.
6Validation checks:
7- Initial state exists in states dict
8- All referenced states exist
9- At least one terminal state
10- Evaluator configs have required fields for their type
11- No conflicting routing (shorthand vs full route)
12- Numeric fields in valid ranges (max_iterations > 0, backoff >= 0, timeout > 0)
13"""
15from __future__ import annotations
17import logging
18import re
19from collections import deque
20from dataclasses import dataclass
21from enum import Enum
22from pathlib import Path
23from typing import Any
25import yaml
27from little_loops.fsm.evaluators import _NUMERIC_OPERATORS
28from little_loops.fsm.fragments import resolve_flow, resolve_fragments, resolve_inheritance
29from little_loops.fsm.schema import EvaluateConfig, FSMLoop, ParameterSpec, StateConfig
31logger = logging.getLogger(__name__)
34class ValidationSeverity(Enum):
35 """Severity level for validation issues."""
37 ERROR = "error"
38 WARNING = "warning"
41@dataclass
42class ValidationError:
43 """Structured validation error.
45 Attributes:
46 message: Human-readable error description
47 path: Path to the problematic element (e.g., "states.check.route")
48 severity: Error severity (error or warning)
49 """
51 message: str
52 path: str | None = None
53 severity: ValidationSeverity = ValidationSeverity.ERROR
55 def __str__(self) -> str:
56 """Format error for display."""
57 prefix = f"[{self.severity.value.upper()}]"
58 if self.path:
59 return f"{prefix} {self.path}: {self.message}"
60 return f"{prefix} {self.message}"
63# Evaluator type to required fields mapping
64EVALUATOR_REQUIRED_FIELDS: dict[str, list[str]] = {
65 "exit_code": [],
66 "output_numeric": ["operator", "target"],
67 "output_json": ["path", "operator", "target"],
68 "output_contains": ["pattern"],
69 "convergence": ["target"],
70 "diff_stall": [],
71 "action_stall": [],
72 "llm_structured": [],
73 "mcp_result": [],
74 "harbor_scorer": [],
75 "comparator": ["baseline_path"],
76 "contract": ["pairs"],
77 "classify": [],
78}
80# Non-LLM evaluator types: all evaluator types except llm_structured
81# Derived from EVALUATOR_REQUIRED_FIELDS so new types are automatically included
82NON_LLM_EVALUATOR_TYPES: frozenset[str] = frozenset(EVALUATOR_REQUIRED_FIELDS.keys()) - {
83 "llm_structured",
84 "comparator",
85 "contract",
86}
88# Meta-loop detector: action string patterns that indicate harness artifact writes
89_META_LOOP_ACTION_PATTERNS: tuple[re.Pattern[str], ...] = (
90 re.compile(r"loops/[\w-]+\.yaml"),
91 re.compile(r"skills/[\w-]+/SKILL\.md"),
92 re.compile(r"agents/[\w-]+\.md"),
93 re.compile(r"commands/[\w-]+\.md"),
94 re.compile(r"\.claude/(CLAUDE\.md|settings)"),
95)
97# Action string tokens that indicate meta-loop behavior
98_META_LOOP_ACTION_TOKENS: frozenset[str] = frozenset({"yaml_state_editor", "replace_action"})
100# Import paths that identify a loop as a meta-loop (harness optimization framework)
101_META_LOOP_IMPORT_TRIGGERS: frozenset[str] = frozenset({"lib/benchmark.yaml"})
103# MR-3: shared-tmp path detector. The runner injects ${context.run_dir} resolving
104# to .loops/runs/<loop>-<timestamp>/; loops that hardcode .loops/tmp/ instead
105# cause state corruption under concurrent runs (ll-parallel, retries, etc.).
106_SHARED_TMP_PATH_RE = re.compile(r"\.loops/tmp/[\w./-]+")
108# ENH-1961: Regex for extracting captured variable names from ${captured.<var>.*} references
109_CAPTURED_REF_RE = re.compile(r"\$\{captured\.(\w+)")
111# Full-reference form, capturing the var name and the remainder up to the closing
112# brace so we can detect a `:default=` guard. A reference written as
113# `${captured.x.output:default=...}` is provably safe even on paths that bypass
114# the capturing state — the interpolation engine (interpolation.py) substitutes
115# the default when the path is missing — so it must NOT be flagged by the
116# capture-reachability check. _CAPTURED_REF_RE alone can't see the guard.
117_CAPTURED_REF_FULL_RE = re.compile(r"\$\{captured\.(\w+)([^}]*)\}")
120def _unguarded_captured_refs(text: str) -> set[str]:
121 """Return captured var names that have at least one reference WITHOUT a
122 `:default=` guard. Vars referenced only via `${captured.x...:default=...}`
123 are omitted: the default makes a missing value safe, so they should not
124 trigger missing-capture or bypass-path diagnostics.
125 """
126 refs: set[str] = set()
127 for var_name, remainder in _CAPTURED_REF_FULL_RE.findall(text):
128 if ":default=" not in remainder:
129 refs.add(var_name)
130 return refs
133# ENH-1819: Regex patterns for detecting multimodal evaluation in prompt actions
134_MULTIMODAL_EVAL_PATTERNS: tuple[re.Pattern[str], ...] = (
135 re.compile(r"Read the screenshot", re.IGNORECASE),
136 re.compile(r"view the (generated )?(website|page|image)", re.IGNORECASE),
137 re.compile(r"screenshot\.(png|jpg|jpeg|webp)"),
138 re.compile(r"\.(png|jpg|jpeg|webp)\b.*\b(read|view|evaluate|score|judge)", re.IGNORECASE),
139)
141# Valid comparison operators
142VALID_OPERATORS = {"eq", "ne", "lt", "le", "gt", "ge"}
144# Valid values for the top-level `visibility:` field (audience axis for
145# `ll-loop list` filtering). "public" is the default when the field is absent.
146VALID_VISIBILITY: frozenset[str] = frozenset({"public", "internal", "example"})
148# All top-level keys recognized by FSMLoop.from_dict()
149KNOWN_TOP_LEVEL_KEYS: frozenset[str] = frozenset(
150 {
151 "name",
152 "description",
153 "initial",
154 "states",
155 "context",
156 "parameters",
157 "scope",
158 "max_steps",
159 "on_max_steps",
160 "max_iterations",
161 "on_max_iterations",
162 "max_edge_revisits",
163 "backoff",
164 "timeout",
165 "default_timeout",
166 "maintain",
167 "llm",
168 "on_handoff",
169 "input_key",
170 "required_inputs",
171 "config",
172 "category",
173 "labels",
174 "visibility",
175 "commands",
176 "targets",
177 "circuit",
178 "meta_self_eval_ok",
179 "shared_state_ok",
180 "partial_route_ok",
181 "artifact_versioning",
182 "artifact_versioning_ok",
183 "generator_fix_ok",
184 "import",
185 "fragments",
186 "from",
187 "flow",
188 "state_defs",
189 }
190)
192# Special tokens accepted as the `on_repeated_failure` target on the
193# stall detector — `"abort"` means terminate via _finish("stall_detected").
194STALL_SPECIAL_TOKENS: frozenset[str] = frozenset({"abort"})
196# Valid parameter types for the 'parameters:' block
197VALID_PARAMETER_TYPES: frozenset[str] = frozenset(
198 {"string", "integer", "number", "boolean", "enum", "path"}
199)
202def _validate_evaluator(state_name: str, evaluate: EvaluateConfig) -> list[ValidationError]:
203 """Validate evaluator configuration for type-specific requirements.
205 Args:
206 state_name: Name of the state containing this evaluator
207 evaluate: The evaluator configuration to validate
209 Returns:
210 List of validation errors found
211 """
212 errors: list[ValidationError] = []
213 path = f"states.{state_name}.evaluate"
215 # Check that evaluator type is recognized
216 valid_types = set(EVALUATOR_REQUIRED_FIELDS.keys())
217 if evaluate.type not in valid_types:
218 errors.append(
219 ValidationError(
220 message=f"Unknown evaluator type '{evaluate.type}'. "
221 f"Must be one of: {', '.join(sorted(valid_types))}",
222 path=path,
223 )
224 )
225 return errors # Can't check required fields for unknown type
227 # Check required fields for evaluator type
228 required = EVALUATOR_REQUIRED_FIELDS.get(evaluate.type, [])
229 for field_name in required:
230 value = getattr(evaluate, field_name, None)
231 if value is None:
232 errors.append(
233 ValidationError(
234 message=f"Evaluator type '{evaluate.type}' requires '{field_name}' field",
235 path=path,
236 )
237 )
239 # Validate operator if present
240 if evaluate.operator is not None and evaluate.operator not in VALID_OPERATORS:
241 errors.append(
242 ValidationError(
243 message=f"Invalid operator '{evaluate.operator}'. "
244 f"Must be one of: {', '.join(sorted(VALID_OPERATORS))}",
245 path=f"{path}.operator",
246 )
247 )
249 # Validate convergence-specific fields
250 if evaluate.type == "convergence":
251 if evaluate.direction not in ("minimize", "maximize"):
252 errors.append(
253 ValidationError(
254 message=f"Invalid direction '{evaluate.direction}'. "
255 "Must be 'minimize' or 'maximize'",
256 path=f"{path}.direction",
257 )
258 )
259 # Only validate tolerance if it's a numeric value (not an interpolation string)
260 if (
261 evaluate.tolerance is not None
262 and isinstance(evaluate.tolerance, (int, float))
263 and evaluate.tolerance < 0
264 ):
265 errors.append(
266 ValidationError(
267 message="Tolerance cannot be negative",
268 path=f"{path}.tolerance",
269 )
270 )
272 # Validate llm_structured-specific fields
273 if evaluate.type == "llm_structured":
274 if evaluate.min_confidence < 0 or evaluate.min_confidence > 1:
275 errors.append(
276 ValidationError(
277 message="min_confidence must be between 0 and 1",
278 path=f"{path}.min_confidence",
279 )
280 )
282 # Validate diff_stall-specific fields
283 if evaluate.type == "diff_stall":
284 if evaluate.max_stall < 1:
285 errors.append(
286 ValidationError(
287 message="max_stall must be >= 1",
288 path=f"{path}.max_stall",
289 )
290 )
292 # Validate action_stall-specific fields
293 if evaluate.type == "action_stall":
294 if evaluate.max_repeat < 1:
295 errors.append(
296 ValidationError(
297 message="max_repeat must be >= 1",
298 path=f"{path}.max_repeat",
299 )
300 )
302 return errors
305def _validate_parameters(fsm: FSMLoop) -> list[ValidationError]:
306 """Validate the loop's top-level parameters: block.
308 Args:
309 fsm: The FSM loop to validate
311 Returns:
312 List of validation errors found
313 """
314 errors: list[ValidationError] = []
316 for param_name, param_spec in fsm.parameters.items():
317 path = f"parameters.{param_name}"
319 if param_spec.type not in VALID_PARAMETER_TYPES:
320 errors.append(
321 ValidationError(
322 message=(
323 f"Unknown parameter type '{param_spec.type}'. "
324 f"Must be one of: {', '.join(sorted(VALID_PARAMETER_TYPES))}"
325 ),
326 path=path,
327 )
328 )
330 if param_spec.type == "enum" and not param_spec.values:
331 errors.append(
332 ValidationError(
333 message="Parameter type 'enum' requires a 'values' list",
334 path=path,
335 )
336 )
338 if param_spec.required and param_spec.default is not None:
339 errors.append(
340 ValidationError(
341 message="Parameter cannot be both 'required: true' and have a 'default' value",
342 path=path,
343 )
344 )
346 return errors
349def _check_param_type(value: Any, spec: ParameterSpec) -> str | None:
350 """Return an error message if value does not match spec.type, else None."""
351 if spec.type == "string" and not isinstance(value, str):
352 return f"expected string, got {type(value).__name__}"
353 if spec.type == "integer" and not isinstance(value, int):
354 return f"expected integer, got {type(value).__name__}"
355 if spec.type == "number" and not isinstance(value, (int, float)):
356 return f"expected number, got {type(value).__name__}"
357 if spec.type == "boolean" and not isinstance(value, bool):
358 return f"expected boolean, got {type(value).__name__}"
359 if spec.type == "enum" and spec.values and value not in spec.values:
360 return f"expected one of {spec.values!r}, got {value!r}"
361 return None
364def _validate_with_bindings(fsm: FSMLoop, loop_dir: Path) -> list[ValidationError]:
365 """Validate with: bindings against child loop parameter contracts.
367 Called from load_and_validate (not validate_fsm) because resolving child loops
368 requires file-system access via the loop directory path.
370 Args:
371 fsm: The parent FSM loop
372 loop_dir: Directory to resolve child loop paths from
374 Returns:
375 List of validation errors found
376 """
377 errors: list[ValidationError] = []
379 for state_name, state in fsm.states.items():
380 if state.loop is None or not state.with_:
381 continue
383 # Try to resolve and load the child loop; skip if unavailable
384 try:
385 from little_loops.cli.loop._helpers import resolve_loop_path
387 loop_path = resolve_loop_path(state.loop, loop_dir)
388 child_fsm, _ = load_and_validate(loop_path)
389 except Exception:
390 continue
392 if not child_fsm.parameters:
393 continue # Child has no declared contract — nothing to cross-validate
395 path = f"states.{state_name}"
397 # Unknown with: keys (not declared by child)
398 for key in state.with_:
399 if key not in child_fsm.parameters:
400 errors.append(
401 ValidationError(
402 message=(
403 f"'with.{key}' is not a declared parameter of loop '{state.loop}'. "
404 f"Declared: {', '.join(sorted(child_fsm.parameters))}"
405 ),
406 path=f"{path}.with.{key}",
407 )
408 )
410 # Required parameters not bound
411 for param_name, param_spec in child_fsm.parameters.items():
412 if param_spec.required and param_name not in state.with_:
413 errors.append(
414 ValidationError(
415 message=(
416 f"Required parameter '{param_name}' of loop '{state.loop}' "
417 f"is not bound in 'with'"
418 ),
419 path=f"{path}.with",
420 )
421 )
423 # Statically-detectable type mismatches (skip interpolation strings)
424 for param_name, value in state.with_.items():
425 if param_name not in child_fsm.parameters:
426 continue
427 if isinstance(value, str) and "${" in value:
428 continue
429 type_error = _check_param_type(value, child_fsm.parameters[param_name])
430 if type_error:
431 errors.append(
432 ValidationError(
433 message=f"Parameter '{param_name}': {type_error}",
434 path=f"{path}.with.{param_name}",
435 )
436 )
438 return errors
441def _validate_loop_references(fsm: FSMLoop, loop_dir: Path) -> list[ValidationError]:
442 """Validate that every state's loop: reference resolves to an actual loop file.
444 Called from load_and_validate (not validate_fsm) because resolving child loops
445 requires file-system access via the loop directory path.
446 """
447 errors: list[ValidationError] = []
448 for state_name, state in fsm.states.items():
449 if state.loop is None:
450 continue
451 # Skip dynamically interpolated loop names — they can only be checked at runtime
452 if "${" in state.loop:
453 continue
454 try:
455 from little_loops.cli.loop._helpers import resolve_loop_path
457 resolve_loop_path(state.loop, loop_dir)
458 except FileNotFoundError:
459 errors.append(
460 ValidationError(
461 message=f"Loop reference '{state.loop}' does not resolve to any file.",
462 path=f"states.{state_name}.loop",
463 severity=ValidationSeverity.WARNING,
464 )
465 )
466 return errors
469def _validate_fragment_bindings(fsm: FSMLoop, loop_dir: Path) -> list[ValidationError]:
470 """Validate fragment with: bindings against fragment parameter contracts.
472 Called from load_and_validate (not validate_fsm) because fragment parameters
473 are populated by resolve_fragments which runs before dataclass parsing.
475 Args:
476 fsm: The FSM loop to validate
477 loop_dir: Directory containing the loop file (unused; kept for API symmetry with
478 _validate_with_bindings)
480 Returns:
481 List of validation errors found
482 """
483 # Runner-injected vars available at runtime but not at static analysis time
484 RUNNER_INJECTED = {"run_dir", "loop_name", "started_at", "input_hash"}
486 errors: list[ValidationError] = []
488 for state_name, state in fsm.states.items():
489 if not state.fragment_parameters:
490 continue # No declared contract — nothing to cross-validate
492 path = f"states.{state_name}"
494 # Unknown with: keys (not declared by fragment)
495 for key in state.fragment_bindings:
496 if key not in state.fragment_parameters:
497 errors.append(
498 ValidationError(
499 message=(
500 f"'with.{key}' is not a declared parameter of fragment "
501 f"'{state.fragment_name}'. "
502 f"Declared: {', '.join(sorted(state.fragment_parameters))}"
503 ),
504 path=f"{path}.with.{key}",
505 )
506 )
508 # Required parameters not bound (whitelist runner-injected vars)
509 for param_name, param_spec in state.fragment_parameters.items():
510 if param_spec.required and param_name not in state.fragment_bindings:
511 if param_name in RUNNER_INJECTED:
512 continue # Available at runtime; not a static error
513 errors.append(
514 ValidationError(
515 message=(
516 f"Required parameter '{param_name}' of fragment "
517 f"'{state.fragment_name}' is not bound in 'with'"
518 ),
519 path=f"{path}.with",
520 )
521 )
523 # Statically-detectable type mismatches (skip interpolation strings)
524 for param_name, value in state.fragment_bindings.items():
525 if param_name not in state.fragment_parameters:
526 continue
527 if isinstance(value, str) and "${" in value:
528 continue
529 type_error = _check_param_type(value, state.fragment_parameters[param_name])
530 if type_error:
531 errors.append(
532 ValidationError(
533 message=f"Parameter '{param_name}': {type_error}",
534 path=f"{path}.with.{param_name}",
535 )
536 )
538 return errors
541def _validate_state_action(state_name: str, state: StateConfig) -> list[ValidationError]:
542 """Validate state action configuration.
544 Args:
545 state_name: Name of the state to validate
546 state: The state configuration to validate
548 Returns:
549 List of validation errors found
550 """
551 errors: list[ValidationError] = []
552 path = f"states.{state_name}"
554 # append_to_messages must contain at least one ${...} interpolation expression
555 if state.append_to_messages is not None:
556 if "${" not in state.append_to_messages:
557 errors.append(
558 ValidationError(
559 message=(
560 "'append_to_messages' must contain a ${...} interpolation expression "
561 f"(e.g. '${{captured.{state_name}.output}}')"
562 ),
563 path=f"{path}.append_to_messages",
564 )
565 )
567 # model: override is silently ignored for non-prompt states (host CLI is not invoked)
568 if state.model is not None and state.action_type not in ("prompt", "slash_command", None):
569 errors.append(
570 ValidationError(
571 message="model: override is ignored for shell/mcp_tool/contract states",
572 path=f"{path}.model",
573 severity=ValidationSeverity.WARNING,
574 )
575 )
577 # params field is only valid for mcp_tool states
578 if state.params and state.action_type != "mcp_tool":
579 errors.append(
580 ValidationError(
581 message="'params' field is only valid when action_type is 'mcp_tool'",
582 path=f"{path}.params",
583 )
584 )
586 # loop and action are mutually exclusive
587 if state.loop is not None and state.action is not None:
588 errors.append(
589 ValidationError(
590 message="'loop' and 'action' are mutually exclusive — "
591 "a sub-loop state cannot also have an action",
592 path=f"{path}",
593 )
594 )
596 # with: requires loop: to be set
597 if state.with_ and state.loop is None:
598 errors.append(
599 ValidationError(
600 message="'with' is only valid when 'loop' is set",
601 path=f"{path}.with",
602 )
603 )
605 # FEAT-1283: type=learning requires a populated LearningConfig
606 if state.type == "learning" and state.learning is not None:
607 if not state.learning.targets and not state.learning.targets_csv:
608 errors.append(
609 ValidationError(
610 message="type=learning requires non-empty 'learning.targets' or 'learning.targets_csv'",
611 path=f"{path}.learning.targets",
612 )
613 )
614 if state.learning.max_retries < 0:
615 errors.append(
616 ValidationError(
617 message=(
618 f"learning.max_retries must be >= 0, got {state.learning.max_retries}"
619 ),
620 path=f"{path}.learning.max_retries",
621 )
622 )
623 if state.on_yes is None:
624 errors.append(
625 ValidationError(
626 message="type=learning requires 'on_yes' (target for all-proven)",
627 path=f"{path}.on_yes",
628 )
629 )
630 if state.on_blocked is None and state.on_no is None:
631 errors.append(
632 ValidationError(
633 message=(
634 "type=learning requires 'on_blocked' or 'on_no' "
635 "(target for refuted / retries_exhausted)"
636 ),
637 path=f"{path}",
638 )
639 )
641 # with: and context_passthrough are mutually exclusive
642 if state.with_ and state.context_passthrough:
643 errors.append(
644 ValidationError(
645 message=(
646 "'with' and 'context_passthrough' are mutually exclusive — "
647 "use 'with' for explicit parameter bindings or 'context_passthrough' "
648 "for legacy bulk passthrough, not both"
649 ),
650 path=f"{path}",
651 )
652 )
654 return errors
657def _validate_state_routing(state_name: str, state: StateConfig) -> list[ValidationError]:
658 """Validate state routing configuration.
660 Checks for conflicting routing definitions (shorthand vs full route).
662 Args:
663 state_name: Name of the state to validate
664 state: The state configuration to validate
666 Returns:
667 List of validation errors/warnings found
668 """
669 errors: list[ValidationError] = []
670 path = f"states.{state_name}"
672 has_shorthand = (
673 state.on_yes is not None
674 or state.on_no is not None
675 or state.on_error is not None
676 or state.on_partial is not None
677 or state.on_blocked is not None
678 or bool(state.extra_routes)
679 )
680 has_route = state.route is not None
682 # Warn about conflicting definitions
683 if has_shorthand and has_route:
684 errors.append(
685 ValidationError(
686 message="Both shorthand routing (on_yes/on_no/on_error) "
687 "and full route table defined. Route table will take precedence.",
688 path=path,
689 severity=ValidationSeverity.WARNING,
690 )
691 )
693 # Check for no valid transition definition
694 has_next = state.next is not None
695 has_terminal = state.terminal
696 has_loop = state.loop is not None
698 if not has_shorthand and not has_route and not has_next and not has_terminal and not has_loop:
699 errors.append(
700 ValidationError(
701 message="State has no transition defined. Add routing, 'next', "
702 "or mark as 'terminal: true'",
703 path=path,
704 )
705 )
707 # Validate retry field pairing: max_retries requires on_retry_exhausted and vice versa
708 if state.max_retries is not None and state.on_retry_exhausted is None:
709 errors.append(
710 ValidationError(
711 message="'max_retries' requires 'on_retry_exhausted' to also be set",
712 path=path,
713 )
714 )
715 if state.on_retry_exhausted is not None and state.max_retries is None:
716 errors.append(
717 ValidationError(
718 message="'on_retry_exhausted' requires 'max_retries' to also be set",
719 path=path,
720 )
721 )
722 if state.max_retries is not None and state.max_retries < 1:
723 errors.append(
724 ValidationError(
725 message=f"'max_retries' must be >= 1, got {state.max_retries}",
726 path=path,
727 )
728 )
730 # Validate retryable_exit_codes: requires on_error; all codes must be positive ints
731 if state.retryable_exit_codes is not None:
732 if state.on_error is None:
733 errors.append(
734 ValidationError(
735 message="'retryable_exit_codes' requires 'on_error' to also be set",
736 path=path,
737 )
738 )
739 for code in state.retryable_exit_codes:
740 if not isinstance(code, int) or code < 1:
741 errors.append(
742 ValidationError(
743 message=(
744 f"'retryable_exit_codes' entries must be positive "
745 f"integers, got {code!r}"
746 ),
747 path=f"{path}.retryable_exit_codes",
748 )
749 )
750 break
752 # Validate rate-limit retry field pairing (mirrors max_retries/on_retry_exhausted)
753 if state.max_rate_limit_retries is not None and state.on_rate_limit_exhausted is None:
754 errors.append(
755 ValidationError(
756 message="'max_rate_limit_retries' requires 'on_rate_limit_exhausted' to also be set",
757 path=path,
758 )
759 )
760 if state.on_rate_limit_exhausted is not None and state.max_rate_limit_retries is None:
761 errors.append(
762 ValidationError(
763 message="'on_rate_limit_exhausted' requires 'max_rate_limit_retries' to also be set",
764 path=path,
765 )
766 )
767 if state.max_rate_limit_retries is not None and state.max_rate_limit_retries < 1:
768 errors.append(
769 ValidationError(
770 message=f"'max_rate_limit_retries' must be >= 1, got {state.max_rate_limit_retries}",
771 path=path,
772 )
773 )
774 if (
775 state.rate_limit_backoff_base_seconds is not None
776 and state.rate_limit_backoff_base_seconds < 1
777 ):
778 errors.append(
779 ValidationError(
780 message=(
781 f"'rate_limit_backoff_base_seconds' must be >= 1, "
782 f"got {state.rate_limit_backoff_base_seconds}"
783 ),
784 path=path,
785 )
786 )
787 if state.rate_limit_max_wait_seconds is not None and state.rate_limit_max_wait_seconds < 1:
788 errors.append(
789 ValidationError(
790 message=(
791 f"'rate_limit_max_wait_seconds' must be >= 1, "
792 f"got {state.rate_limit_max_wait_seconds}"
793 ),
794 path=path,
795 )
796 )
797 if state.rate_limit_long_wait_ladder is not None:
798 if len(state.rate_limit_long_wait_ladder) == 0:
799 errors.append(
800 ValidationError(
801 message="'rate_limit_long_wait_ladder' must be non-empty if specified",
802 path=path,
803 )
804 )
805 else:
806 for idx, value in enumerate(state.rate_limit_long_wait_ladder):
807 if not isinstance(value, int) or value < 1:
808 errors.append(
809 ValidationError(
810 message=(
811 f"'rate_limit_long_wait_ladder[{idx}]' must be a "
812 f"positive integer, got {value!r}"
813 ),
814 path=path,
815 )
816 )
818 # Validate throttle config when present
819 if state.throttle is not None:
820 t = state.throttle
821 fields = {
822 "normal_max": t.normal_max,
823 "warn_max": t.warn_max,
824 "hard_max": t.hard_max,
825 }
826 for field_name, val in fields.items():
827 if val is not None and (not isinstance(val, int) or val < 1):
828 errors.append(
829 ValidationError(
830 message=f"'throttle.{field_name}' must be a positive integer, got {val!r}",
831 path=path,
832 )
833 )
834 # Enforce ordering when all three are set
835 if t.normal_max is not None and t.warn_max is not None and t.normal_max >= t.warn_max:
836 errors.append(
837 ValidationError(
838 message=(
839 f"'throttle.normal_max' ({t.normal_max}) must be less than "
840 f"'throttle.warn_max' ({t.warn_max})"
841 ),
842 path=path,
843 )
844 )
845 if t.warn_max is not None and t.hard_max is not None and t.warn_max >= t.hard_max:
846 errors.append(
847 ValidationError(
848 message=(
849 f"'throttle.warn_max' ({t.warn_max}) must be less than "
850 f"'throttle.hard_max' ({t.hard_max})"
851 ),
852 path=path,
853 )
854 )
856 return errors
859def _validate_targets(fsm: FSMLoop) -> list[ValidationError]:
860 """Validate top-level targets[] entries (ENH-1552).
862 Rejects any targets[].states[] entry whose sibling file: value does not
863 end with a .yaml extension.
864 """
865 errors: list[ValidationError] = []
866 for i, target in enumerate(fsm.targets):
867 if target.file is not None and not target.file.endswith(".yaml"):
868 errors.append(
869 ValidationError(
870 message=(f"targets[{i}].file must be a .yaml file, got '{target.file}'"),
871 path=f"targets[{i}].file",
872 )
873 )
874 return errors
877def _validate_failure_terminal_action(fsm: FSMLoop) -> list[ValidationError]:
878 """Warn when a failure-named terminal state has no diagnostic predecessor.
880 Failure terminals (failed, error, aborted) should have at least one
881 predecessor state with an action or sub-loop that provides diagnostic
882 output before termination. Otherwise the failure is silent — the
883 executor calls _finish("terminal") before any action on the terminal
884 itself can execute.
886 Severity is WARNING (not ERROR) so that existing loops with bare
887 failure terminals continue to load, and test_terminal_only_state_valid
888 (which filters by ERROR) passes without modification.
889 """
890 FAILURE_TERMINAL_NAMES: frozenset[str] = frozenset({"failed", "error", "aborted"})
891 errors: list[ValidationError] = []
893 terminal_states = fsm.get_terminal_states()
894 failure_terminals = terminal_states & FAILURE_TERMINAL_NAMES
896 for ft_name in failure_terminals:
897 has_diagnostic_predecessor = False
898 for state_name, state in fsm.states.items():
899 if state_name == ft_name:
900 continue
901 if ft_name in state.get_referenced_states():
902 if state.action is not None or state.loop is not None:
903 has_diagnostic_predecessor = True
904 break
906 if not has_diagnostic_predecessor:
907 errors.append(
908 ValidationError(
909 message=(
910 f"Failure-named terminal state '{ft_name}' has no predecessor "
911 "state with a diagnostic action. Add a non-terminal diagnostic "
912 "state (e.g. 'diagnose') with an action or sub-loop that routes "
913 f"to '{ft_name}'."
914 ),
915 path=f"states.{ft_name}",
916 severity=ValidationSeverity.WARNING,
917 )
918 )
920 return errors
923def validate_fsm(fsm: FSMLoop) -> list[ValidationError]:
924 """Validate FSM structure and return list of errors.
926 Performs comprehensive validation:
927 - Initial state exists
928 - All referenced states exist
929 - At least one terminal state
930 - Evaluator configurations are valid
931 - Routing configurations are valid
932 - Numeric fields are in valid ranges (max_iterations > 0, backoff >= 0, timeout > 0)
934 Args:
935 fsm: The FSM loop to validate
937 Returns:
938 List of validation errors (empty if valid)
939 """
940 errors: list[ValidationError] = []
941 defined_states = fsm.get_all_state_names()
943 # Warn when no top-level description: field is set. The field is optional
944 # for FSM execution but required for goal-alignment skills (debug-loop-run,
945 # audit-loop-run) and for ll-loop show --json to surface intent text.
946 if not fsm.description:
947 errors.append(
948 ValidationError(
949 path="<root>",
950 message=("No 'description' field defined. Add a top-level description: key."),
951 severity=ValidationSeverity.WARNING,
952 )
953 )
955 # Validate parameters block
956 errors.extend(_validate_parameters(fsm))
958 # Validate targets block (ENH-1552)
959 errors.extend(_validate_targets(fsm))
961 # Check initial state exists
962 if fsm.initial not in defined_states:
963 errors.append(
964 ValidationError(
965 message=f"Initial state '{fsm.initial}' not found in states",
966 path="initial",
967 )
968 )
970 # Check at least one terminal state
971 terminal_states = fsm.get_terminal_states()
972 if not terminal_states:
973 errors.append(
974 ValidationError(
975 message="No terminal state defined. At least one state must have 'terminal: true'",
976 path="states",
977 )
978 )
980 # Validate each state
981 for state_name, state in fsm.states.items():
982 # Check all referenced states exist
983 refs = state.get_referenced_states()
984 for ref in refs:
985 # $current is a special token for retry
986 if ref != "$current" and ref not in defined_states:
987 errors.append(
988 ValidationError(
989 message=f"References unknown state '{ref}'",
990 path=f"states.{state_name}",
991 )
992 )
994 # Validate action configuration
995 errors.extend(_validate_state_action(state_name, state))
997 # Validate evaluator if present
998 if state.evaluate is not None:
999 errors.extend(_validate_evaluator(state_name, state.evaluate))
1001 # Validate routing configuration
1002 errors.extend(_validate_state_routing(state_name, state))
1004 # Check numeric field ranges
1005 if fsm.max_steps <= 0:
1006 errors.append(
1007 ValidationError(
1008 message=f"max_steps must be > 0, got {fsm.max_steps}",
1009 path="max_steps",
1010 )
1011 )
1012 if fsm.max_iterations is not None and fsm.max_iterations <= 0:
1013 errors.append(
1014 ValidationError(
1015 message=f"max_iterations must be > 0, got {fsm.max_iterations}",
1016 path="max_iterations",
1017 )
1018 )
1019 if fsm.max_edge_revisits <= 0:
1020 errors.append(
1021 ValidationError(
1022 message=f"max_edge_revisits must be > 0, got {fsm.max_edge_revisits}",
1023 path="max_edge_revisits",
1024 )
1025 )
1026 if fsm.backoff is not None and fsm.backoff < 0:
1027 errors.append(
1028 ValidationError(
1029 message=f"backoff must be >= 0, got {fsm.backoff}",
1030 path="backoff",
1031 )
1032 )
1033 if fsm.timeout is not None and fsm.timeout <= 0:
1034 errors.append(
1035 ValidationError(
1036 message=f"timeout must be > 0, got {fsm.timeout}",
1037 path="timeout",
1038 )
1039 )
1040 if fsm.llm.max_tokens <= 0:
1041 errors.append(
1042 ValidationError(
1043 message=f"llm.max_tokens must be > 0, got {fsm.llm.max_tokens}",
1044 path="llm.max_tokens",
1045 )
1046 )
1047 if fsm.llm.timeout <= 0:
1048 errors.append(
1049 ValidationError(
1050 message=f"llm.timeout must be > 0, got {fsm.llm.timeout}",
1051 path="llm.timeout",
1052 )
1053 )
1055 # Check for unreachable states (warning only)
1056 reachable = _find_reachable_states(fsm)
1057 unreachable = defined_states - reachable
1058 for state_name in unreachable:
1059 errors.append(
1060 ValidationError(
1061 message="State is not reachable from initial state",
1062 path=f"states.{state_name}",
1063 severity=ValidationSeverity.WARNING,
1064 )
1065 )
1067 errors.extend(_validate_failure_terminal_action(fsm))
1069 errors.extend(_validate_meta_loop_evaluation(fsm))
1071 errors.extend(_validate_input_key_without_guard(fsm))
1073 errors.extend(_validate_artifact_isolation(fsm))
1075 errors.extend(_validate_harness_multimodal_evaluator_blind_spot(fsm))
1077 errors.extend(_validate_partial_route_dead_end(fsm))
1079 errors.extend(_validate_artifact_overwrite(fsm))
1081 errors.extend(_validate_generator_fix_discipline(fsm))
1083 errors.extend(_validate_classify_route_default(fsm))
1085 errors.extend(_validate_zero_retry_counter(fsm))
1087 errors.extend(_validate_on_max_steps(fsm, defined_states))
1088 errors.extend(_validate_on_max_iterations(fsm, defined_states))
1090 errors.extend(_validate_circuit(fsm, defined_states))
1092 errors.extend(_validate_progress_paths_isolation(fsm))
1094 errors.extend(_validate_capture_reachability(fsm))
1096 return errors
1099def _is_meta_loop(fsm: FSMLoop) -> bool:
1100 """Return True if fsm is classified as a meta-loop.
1102 A loop is meta if ANY of the following match:
1103 1. Any state action string matches a harness-artifact path regex
1104 (writes another loop YAML, skill, agent, command, or project config)
1105 2. The loop's import list contains lib/benchmark.yaml
1106 3. Any state action references yaml_state_editor or replace_action
1107 """
1108 # Condition 2: imports lib/benchmark.yaml
1109 if any(imp in _META_LOOP_IMPORT_TRIGGERS for imp in fsm.imports):
1110 return True
1111 # Conditions 1 and 3: scan action strings
1112 for state in fsm.states.values():
1113 if state.action is None:
1114 continue
1115 for pattern in _META_LOOP_ACTION_PATTERNS:
1116 if pattern.search(state.action):
1117 return True
1118 for token in _META_LOOP_ACTION_TOKENS:
1119 if token in state.action:
1120 return True
1121 return False
1124def _validate_meta_loop_evaluation(fsm: FSMLoop) -> list[ValidationError]:
1125 """Validate meta-loop evaluation rules MR-1 and MR-2.
1127 MR-1 (ERROR): meta-loop must have at least one non-LLM evaluator.
1128 MR-2 (WARNING): meta-loop should reference a captured baseline in an evaluator.
1130 Both rules are suppressed by ``meta_self_eval_ok: true`` at the loop top-level.
1131 """
1132 errors: list[ValidationError] = []
1133 if fsm.meta_self_eval_ok or not _is_meta_loop(fsm):
1134 return errors
1136 # Collect all evaluator types used across all states
1137 evaluator_types: set[str] = set()
1138 for state in fsm.states.values():
1139 if state.evaluate is not None:
1140 evaluator_types.add(state.evaluate.type)
1142 # MR-1: must have at least one non-LLM evaluator
1143 if not evaluator_types & NON_LLM_EVALUATOR_TYPES:
1144 errors.append(
1145 ValidationError(
1146 message=(
1147 "Loop modifies harness artifacts but has no non-LLM evaluator. "
1148 "LLM self-grades on harness updates are unreliable (SHOR Table 1: "
1149 "33-55% accuracy). Pair every check_semantic state with at least one "
1150 "of: exit_code, output_numeric, convergence, diff_stall, action_stall, mcp_result. "
1151 "Note: llm_structured and comparator both use the LLM and do not satisfy MR-1. "
1152 "To suppress with justification, set `meta_self_eval_ok: true` at the "
1153 "loop top-level."
1154 ),
1155 path="<root>",
1156 severity=ValidationSeverity.ERROR,
1157 )
1158 )
1160 # MR-2: should reference a captured baseline in a later evaluator
1161 capture_names: set[str] = {state.capture for state in fsm.states.values() if state.capture}
1162 if capture_names and not _has_baseline_reference(fsm, capture_names):
1163 errors.append(
1164 ValidationError(
1165 message=(
1166 "Meta-loop appears to lack a measure→propose→apply→re-measure "
1167 "spine: no captured baseline value is referenced by a later evaluator. "
1168 "Meta-loops should compare a post-change score against a pre-change "
1169 "baseline (see loops/harness-optimize.yaml as reference template). "
1170 "To suppress, set `meta_self_eval_ok: true`."
1171 ),
1172 path="<root>",
1173 severity=ValidationSeverity.WARNING,
1174 )
1175 )
1177 return errors
1180# Regex patterns for detecting counter-increment actions.
1181# Must contain a printf/echo writing to a file AND an arithmetic increment.
1182_COUNTER_FILE_WRITE_RE = re.compile(r"(?:printf|echo)\s+.*>")
1183_COUNTER_INCREMENT_RE = re.compile(
1184 r"\$\(\(.*\+\s*1\s*\)\)" # $((N + 1)) or $((N+1))
1185 r"|\+\+" # C-style increment
1186 r"|\+=1" # compound assignment
1187 r"|awk\s+.*\+\+" # awk with increment
1188)
1191def _validate_zero_retry_counter(fsm: FSMLoop) -> list[ValidationError]:
1192 """Detect counter + output_numeric combos that yield zero effective retries.
1194 A common loop-authoring footgun: a state increments a counter file and then
1195 evaluates ``output_numeric`` with ``operator: lt, target: 1`` against it.
1196 After the first increment the counter is 1, ``1 < 1 == false``, so the
1197 retry budget is 0 by construction. Author almost always intended target=2.
1198 """
1199 errors: list[ValidationError] = []
1201 for state_name, state in fsm.states.items():
1202 if not state.action or not state.evaluate:
1203 continue
1205 ev = state.evaluate
1206 if ev.type != "output_numeric":
1207 continue
1208 if ev.operator is None or ev.target is None:
1209 continue
1211 # Must be a number-like target for numeric comparison
1212 try:
1213 target = float(ev.target)
1214 except (ValueError, TypeError):
1215 continue
1217 if not _is_counter_action(state.action):
1218 continue
1220 # Check: after first increment (0→1), does operator(1, target) already fail?
1221 op_fn = _NUMERIC_OPERATORS.get(ev.operator)
1222 if op_fn is None:
1223 continue
1225 if not op_fn(1.0, target):
1226 suggested_target = _suggested_target(ev.operator, target)
1227 errors.append(
1228 ValidationError(
1229 message=(
1230 f"Zero retry budget: operator={ev.operator} target={target} "
1231 f"means the first post-increment value (1) already fails "
1232 f"({ev.operator}(1, {target}) == False). "
1233 f"Did you mean target={suggested_target}?"
1234 ),
1235 path=f"states.{state_name}.evaluate",
1236 severity=ValidationSeverity.WARNING,
1237 )
1238 )
1240 return errors
1243def _is_counter_action(action: str) -> bool:
1244 """Return True if the action string contains a counter-increment pattern."""
1245 return bool(_COUNTER_FILE_WRITE_RE.search(action) and _COUNTER_INCREMENT_RE.search(action))
1248def _suggested_target(operator: str, target: float) -> str:
1249 """Suggest a target value that allows at least one retry."""
1250 # For lt/le with a too-low target, suggest target+1 so first post-increment passes
1251 if operator in ("lt", "le"):
1252 return str(int(target) + 1)
1253 # For eq with target=0, suggest 1 so the counter can eventually match
1254 if operator == "eq" and target == 0:
1255 return "1"
1256 # For other cases, suggest target+1 as a default nudge
1257 return str(int(target) + 1)
1260def _validate_harness_multimodal_evaluator_blind_spot(fsm: FSMLoop) -> list[ValidationError]:
1261 """Warn when harness loops use LLM multimodal eval as sole gate to terminal.
1263 LLMs can silently fall back to text-only analysis when reading images,
1264 producing verdicts based on incomplete information. The output_contains
1265 evaluator can verify the LLM wrote the pass string but not that it
1266 actually processed the image. This is the same class of failure as MR-1
1267 (LLM self-evaluation bias) applied to artifact evaluation rather than
1268 harness modification.
1270 Suppressed by ``meta_self_eval_ok: true`` at the loop top-level.
1271 """
1272 errors: list[ValidationError] = []
1273 if fsm.meta_self_eval_ok or fsm.category != "harness":
1274 return errors
1276 terminal_states = fsm.get_terminal_states()
1278 for state_name, state in fsm.states.items():
1279 if state.action_type != "prompt" or not state.action:
1280 continue
1281 if state.evaluate is None or state.evaluate.type != "output_contains":
1282 continue
1283 if not any(p.search(state.action) for p in _MULTIMODAL_EVAL_PATTERNS):
1284 continue
1285 if state.on_yes not in terminal_states:
1286 continue
1287 errors.append(
1288 ValidationError(
1289 message=(
1290 f"State '{state_name}' evaluates a screenshot/image via LLM "
1291 "prompt and routes directly to a terminal on success. The "
1292 "output_contains evaluator can verify the LLM wrote the pass "
1293 "string but not that the LLM actually processed the image. "
1294 "Consider adding a shell-action verification state (e.g., "
1295 "functional smoke test) between scoring and the terminal."
1296 ),
1297 path=f"states.{state_name}",
1298 severity=ValidationSeverity.WARNING,
1299 )
1300 )
1302 return errors
1305def _find_shared_tmp_writes(fsm: FSMLoop) -> list[tuple[str, str]]:
1306 """Return (state_name, matched_path) for every action referencing shared .loops/tmp/.
1308 Scans `state.action` only. Prompts and sub-loop bindings can also reference
1309 paths, but those are out of static-scan reach: action strings are the
1310 place where loop YAMLs directly encode artifact paths.
1311 """
1312 findings: list[tuple[str, str]] = []
1313 for state_name, state in fsm.states.items():
1314 if not state.action:
1315 continue
1316 for match in _SHARED_TMP_PATH_RE.finditer(state.action):
1317 findings.append((state_name, match.group(0)))
1318 return findings
1321def _validate_input_key_without_guard(fsm: FSMLoop) -> list[ValidationError]:
1322 """Warn when a loop sets a custom input_key but omits required_inputs.
1324 A loop that accepts a runtime input via a named key (e.g. input_key: description)
1325 but doesn't declare required_inputs will silently proceed with an empty value if the
1326 user forgets to pass one. Declaring required_inputs shifts that failure to start-time.
1327 """
1328 if fsm.input_key == "input":
1329 return []
1330 if fsm.required_inputs:
1331 return []
1332 return [
1333 ValidationError(
1334 message=(
1335 f"Loop sets input_key: '{fsm.input_key}' but does not declare "
1336 f"required_inputs. If this input is mandatory, add "
1337 f"'required_inputs: [\"{fsm.input_key}\"]' to make the runner "
1338 f"abort when no input is provided."
1339 ),
1340 path="required_inputs",
1341 severity=ValidationSeverity.WARNING,
1342 )
1343 ]
1346def _validate_artifact_isolation(fsm: FSMLoop) -> list[ValidationError]:
1347 """Validate rule MR-3: loops must isolate artifacts to ${context.run_dir}.
1349 The runner injects ${context.run_dir} pointing at .loops/runs/<loop>-<ts>/
1350 and creates the folder before execution. Loops that write intermediate
1351 state (queues, checkpoints, generated files) to shared .loops/tmp/ paths
1352 will corrupt each other under concurrent runs (ll-parallel workers, retries,
1353 repeated invocations).
1355 Suppressed by `shared_state_ok: true` at the loop top-level for loops that
1356 intentionally share state across runs.
1357 """
1358 if fsm.shared_state_ok:
1359 return []
1360 errors: list[ValidationError] = []
1361 for state_name, path in _find_shared_tmp_writes(fsm):
1362 errors.append(
1363 ValidationError(
1364 message=(
1365 f"State writes to shared '{path}' instead of "
1366 "'${context.run_dir}/...'. Concurrent runs of this loop "
1367 "(e.g., under ll-parallel) will corrupt each other's state. "
1368 "Use the runner-injected `${context.run_dir}` for per-run "
1369 "artifact paths, or set `shared_state_ok: true` at the loop "
1370 "top-level if cross-run sharing is intentional."
1371 ),
1372 path=f"states.{state_name}.action",
1373 severity=ValidationSeverity.WARNING,
1374 )
1375 )
1376 return errors
1379def _is_llm_judged(state: StateConfig) -> bool:
1380 """Return True if this state will be graded by the default LLM judge.
1382 Mirrors the action-mode detection in executor._action_mode() (not imported
1383 here because that is runtime code). A state is LLM-judged when:
1384 - it has no explicit evaluate block AND its action is a prompt or slash_command, OR
1385 - it has an explicit evaluate block of type llm_structured or check_semantic.
1386 """
1387 if state.evaluate is None:
1388 # Heuristic: explicit action_type wins; fall back to leading "/" on action string.
1389 action_type = state.action_type
1390 if action_type in ("prompt", "slash_command"):
1391 return True
1392 if action_type is None and state.action and state.action.lstrip().startswith("/"):
1393 return True
1394 return False
1395 return state.evaluate.type in ("llm_structured", "check_semantic")
1398def _validate_partial_route_dead_end(fsm: FSMLoop) -> list[ValidationError]:
1399 """Validate rule MR-4: LLM-judged states with only on_yes have a partial/no dead-end.
1401 A state gated by the default LLM judge can receive yes/no/partial verdicts.
1402 If only on_yes is mapped (and no on_no, on_partial, next, or route table with
1403 a default exist), a partial or no verdict returns None from _route and silently
1404 terminates the loop — the parent treats this as failed.
1406 Suppressed by `partial_route_ok: true` at the loop top-level for the rare
1407 case where dead-ending on a non-yes verdict is intentional.
1408 """
1409 if fsm.partial_route_ok:
1410 return []
1411 errors: list[ValidationError] = []
1412 for state_name, state in fsm.states.items():
1413 if not _is_llm_judged(state):
1414 continue
1415 # States with an unconditional next: or a full route: table are safe.
1416 if state.next is not None or state.route is not None:
1417 continue
1418 # Only flag when on_yes is set but at least one of on_no/on_partial is missing.
1419 if state.on_yes is None:
1420 continue
1421 missing = [v for v in ("no", "partial") if getattr(state, f"on_{v}") is None]
1422 if not missing:
1423 continue
1424 unrouted = " or ".join(f"`{v}`" for v in missing)
1425 errors.append(
1426 ValidationError(
1427 message=(
1428 f"[state: {state_name}] LLM-judged prompt routes only on_yes; "
1429 f"a {unrouted} verdict has no route and will dead-end the loop "
1430 "(parent reads this as failed). Add on_no/on_partial, use `next:` "
1431 "for an unconditional handoff, or a `route:` table with a default. "
1432 "Set `partial_route_ok: true` at the loop top-level to suppress "
1433 "if intentional. (ENH-1917)"
1434 ),
1435 path=f"states.{state_name}",
1436 severity=ValidationSeverity.WARNING,
1437 )
1438 )
1439 return errors
1442def _validate_artifact_overwrite(fsm: FSMLoop) -> list[ValidationError]:
1443 """Validate rule MR-5 (ENH-1957): harness loops should version artifacts per iteration.
1445 A harness-category loop that iteratively generates and overwrites a flat artifact
1446 path (e.g. ``${context.run_dir}/image.svg``) loses all intermediate versions.
1447 Only the final iteration survives. This rule flags iterative generate→evaluate→generate
1448 cycles that write to artifact paths without declaring versioning intent.
1450 Suppressed by ``artifact_versioning: true`` (loop snapshots per-iteration artifacts)
1451 or ``artifact_versioning_ok: true`` (intentional overwrite, e.g. artifact varies
1452 by task).
1453 """
1454 if fsm.artifact_versioning or fsm.artifact_versioning_ok:
1455 return []
1456 if fsm.category not in ("harness",):
1457 return []
1459 errors: list[ValidationError] = []
1461 # Find states that write to artifact paths (shell actions with file output)
1462 writers: dict[str, set[str]] = {} # state_name -> set of artifact paths
1463 for state_name, state in fsm.states.items():
1464 if not state.action or state.action_type not in ("shell", None):
1465 continue
1466 # Skip sub-loop delegation states
1467 if state.action_type == "loop":
1468 continue
1469 action = state.action
1470 # Find run_dir-based artifact writes: ${context.run_dir}/<path> or $RUNDIR/<path>
1471 import re
1473 # Match patterns like: ${context.run_dir}/output.svg, $RUNDIR/image.png, > path
1474 # We look for output redirections or cp/mv commands writing to run_dir
1475 artifact_refs = set()
1476 for pattern in (
1477 r'\$\{context\.run_dir\}/([^\s"\';&]+)',
1478 r'\$\{captured\.[^}]+\}/([^\s"\';&]+)',
1479 ):
1480 for m in re.finditer(pattern, action):
1481 artifact_refs.add(m.group(1))
1482 # Also detect explicit cp/mv writing to run_dir paths
1483 for m in re.finditer(r'(?:cp|mv)\s+.*\s+\$\{context\.run_dir\}/([^\s"\';&]+)', action):
1484 artifact_refs.add(m.group(1))
1485 if artifact_refs:
1486 writers[state_name] = artifact_refs
1488 if not writers:
1489 return []
1491 # Detect iterative cycles: a writer state that is reachable from itself
1492 # via a non-trivial path through other states (generate → evaluate → generate)
1493 for state_name in writers:
1494 refs = fsm.states[state_name].get_referenced_states()
1495 # Check if this writer or its downstream states loop back to this writer
1496 visited: set[str] = set()
1497 to_visit = list(refs - {"$current"})
1498 while to_visit:
1499 target = to_visit.pop()
1500 if target in visited or target not in fsm.states:
1501 continue
1502 visited.add(target)
1503 if target == state_name:
1504 # Found a cycle: this writer state is reachable from itself
1505 artifact_list = ", ".join(sorted(writers[state_name]))
1506 errors.append(
1507 ValidationError(
1508 message=(
1509 f"[state: {state_name}] Harness loop writes artifact(s) "
1510 f"({artifact_list}) to a flat path in an iterative cycle "
1511 f"({state_name} → ... → {state_name}). Per-iteration versions "
1512 "are lost; only the final output survives. Add per-iteration "
1513 "snapshots (see oracle generator-evaluator for pattern) and "
1514 "declare `artifact_versioning: true`, or set "
1515 "`artifact_versioning_ok: true` if intentional. (ENH-1957)"
1516 ),
1517 path=f"states.{state_name}",
1518 severity=ValidationSeverity.WARNING,
1519 )
1520 )
1521 break
1522 target_state = fsm.states.get(target)
1523 if target_state is not None:
1524 target_refs = target_state.get_referenced_states()
1525 for r in target_refs:
1526 if r != "$current" and r not in visited:
1527 to_visit.append(r)
1529 return errors
1532def _validate_generator_fix_discipline(fsm: FSMLoop) -> list[ValidationError]:
1533 """Validate rule MR-6 (ENH-2079): meta-loops should not hand-patch generator artifacts.
1535 Detects the hand-patching anti-pattern: a ``shell``-type state that writes to the
1536 same file path as a non-shell (LLM-type) generator state in the same loop.
1537 Hand-patching creates fragile output that diverges from the generator on the next
1538 run; the stable fix is to update the generator action instead.
1540 Suppressed by ``generator_fix_ok: true`` at the loop top-level for intentional
1541 post-processing cases.
1542 """
1543 if fsm.generator_fix_ok or not _is_meta_loop(fsm):
1544 return []
1546 # Markers that indicate a prompt/slash_command state is generating file artifacts
1547 _GENERATOR_MARKERS = ("yaml_state_editor", "replace_action", "to_file:")
1549 _PATH_PATTERNS = (
1550 re.compile(r'\$\{context\.run_dir\}/([^\s"\';&|]+)'),
1551 re.compile(r'\$\{captured\.[^}]+\}/([^\s"\';&|]+)'),
1552 )
1554 def _extract_paths(action: str) -> set[str]:
1555 paths: set[str] = set()
1556 for pat in _PATH_PATTERNS:
1557 for m in pat.finditer(action):
1558 paths.add(m.group(1).rstrip("/"))
1559 return paths
1561 shell_targets: dict[str, set[str]] = {} # state_name -> set of file paths
1562 generator_targets: dict[str, set[str]] = {} # state_name -> set of file paths
1564 for state_name, state in fsm.states.items():
1565 if not state.action:
1566 continue
1567 action = state.action
1568 paths = _extract_paths(action)
1569 if not paths:
1570 continue
1571 action_type = state.action_type
1572 if action_type in ("shell", None):
1573 shell_targets[state_name] = paths
1574 elif action_type in ("prompt", "slash_command"):
1575 if any(marker in action for marker in _GENERATOR_MARKERS):
1576 generator_targets[state_name] = paths
1578 errors: list[ValidationError] = []
1579 for gen_name, gen_paths in generator_targets.items():
1580 for shell_name, shell_paths in shell_targets.items():
1581 overlap = gen_paths & shell_paths
1582 if overlap:
1583 artifact_list = ", ".join(sorted(overlap))
1584 errors.append(
1585 ValidationError(
1586 message=(
1587 f"[states: {gen_name}, {shell_name}] Hand-patching anti-pattern: "
1588 f"LLM-generator state '{gen_name}' and shell state '{shell_name}' "
1589 f"both write to ({artifact_list}). Move the fix into the generator "
1590 "action so every run produces correct output automatically. "
1591 "Set `generator_fix_ok: true` to suppress for intentional "
1592 "post-processing. (ENH-2079)"
1593 ),
1594 path=f"states.{shell_name}",
1595 severity=ValidationSeverity.WARNING,
1596 )
1597 )
1599 return errors
1602def _validate_classify_route_default(fsm: FSMLoop) -> list[ValidationError]:
1603 """Validate that classify states with a route: table include a default: fallback.
1605 A ``classify`` state whose ``route:`` table has no ``default:`` will dead-end
1606 whenever the action emits a token not listed in the table. This rule flags
1607 that gap as a WARNING so loop authors add a catch-all branch.
1609 Suppressed by ``partial_route_ok: true`` at the loop top-level when a
1610 dead-end on an unlisted token is intentional.
1611 """
1612 if fsm.partial_route_ok:
1613 return []
1614 errors: list[ValidationError] = []
1615 for state_name, state in fsm.states.items():
1616 if state.evaluate is None or state.evaluate.type != "classify":
1617 continue
1618 if state.route is None or state.route.default is not None:
1619 continue
1620 errors.append(
1621 ValidationError(
1622 message=(
1623 f"[state: {state_name}] classify route: table has no default: — "
1624 "unknown tokens will dead-end the loop. Add a default: catch-all, "
1625 "or set `partial_route_ok: true` at the loop top-level to suppress."
1626 ),
1627 path=f"states.{state_name}",
1628 severity=ValidationSeverity.WARNING,
1629 )
1630 )
1631 return errors
1634def _has_baseline_reference(fsm: FSMLoop, capture_names: set[str]) -> bool:
1635 """Return True if any evaluate block references a captured variable."""
1636 for state in fsm.states.values():
1637 ev = state.evaluate
1638 if ev is None:
1639 continue
1640 # Check string fields that may interpolate captured values
1641 candidates = [ev.previous, ev.source]
1642 if isinstance(ev.target, str):
1643 candidates.append(ev.target)
1644 for field_val in candidates:
1645 if not field_val:
1646 continue
1647 for name in capture_names:
1648 if f"captured.{name}" in field_val:
1649 return True
1650 return False
1653def _validate_on_max_steps(fsm: FSMLoop, defined_states: set[str]) -> list[ValidationError]:
1654 """Validate the top-level `on_max_steps` field (BUG-2204).
1656 Checks that the named state exists when `on_max_steps` is set.
1657 """
1658 errors: list[ValidationError] = []
1659 if fsm.on_max_steps is None:
1660 return errors
1661 if fsm.on_max_steps not in defined_states:
1662 errors.append(
1663 ValidationError(
1664 message=(
1665 f"on_max_steps references unknown state "
1666 f"'{fsm.on_max_steps}' (must be a declared state)"
1667 ),
1668 path="on_max_steps",
1669 )
1670 )
1671 return errors
1674def _validate_on_max_iterations(fsm: FSMLoop, defined_states: set[str]) -> list[ValidationError]:
1675 """Validate the top-level `on_max_iterations` field (BUG-2204: iteration-cap summary state).
1677 Checks that the named state exists when `on_max_iterations` is set.
1678 """
1679 errors: list[ValidationError] = []
1680 if fsm.on_max_iterations is None:
1681 return errors
1682 if fsm.on_max_iterations not in defined_states:
1683 errors.append(
1684 ValidationError(
1685 message=(
1686 f"on_max_iterations references unknown state "
1687 f"'{fsm.on_max_iterations}' (must be a declared state)"
1688 ),
1689 path="on_max_iterations",
1690 )
1691 )
1692 return errors
1695def _validate_circuit(fsm: FSMLoop, defined_states: set[str]) -> list[ValidationError]:
1696 """Validate the top-level `circuit:` block (FEAT-1637).
1698 Checks:
1699 - `circuit.repeated_failure.window` is a positive integer.
1700 - `circuit.repeated_failure.on_repeated_failure` is either the special
1701 token ``"abort"`` or the name of a declared state.
1702 """
1703 errors: list[ValidationError] = []
1704 if fsm.circuit is None or fsm.circuit.repeated_failure is None:
1705 return errors
1707 rf = fsm.circuit.repeated_failure
1708 if rf.window < 1:
1709 errors.append(
1710 ValidationError(
1711 message=f"circuit.repeated_failure.window must be >= 1, got {rf.window}",
1712 path="circuit.repeated_failure.window",
1713 )
1714 )
1716 if rf.recurrent_window is not None and rf.recurrent_window < 2:
1717 errors.append(
1718 ValidationError(
1719 message=(
1720 f"circuit.repeated_failure.recurrent_window must be >= 2, "
1721 f"got {rf.recurrent_window}"
1722 ),
1723 path="circuit.repeated_failure.recurrent_window",
1724 )
1725 )
1727 target = rf.on_repeated_failure
1728 if target not in STALL_SPECIAL_TOKENS and target not in defined_states:
1729 errors.append(
1730 ValidationError(
1731 message=(
1732 f"circuit.repeated_failure.on_repeated_failure references "
1733 f"unknown state '{target}' (must be a declared state or "
1734 f'the literal "abort")'
1735 ),
1736 path="circuit.repeated_failure.on_repeated_failure",
1737 )
1738 )
1740 return errors
1743# Matches common interpolation prefixes used in loop YAML paths so we can
1744# extract the portable relative component for action-string scanning.
1745_INTERPOLATION_PREFIX_RE = re.compile(r"^\$\{[^}]+\}/")
1748def _strip_interpolation_prefix(path: str) -> str:
1749 """Return the path with any leading ${...}/ prefix removed."""
1750 return _INTERPOLATION_PREFIX_RE.sub("", path)
1753def _validate_progress_paths_isolation(fsm: FSMLoop) -> list[ValidationError]:
1754 """Warn when a state's action writes to a file listed in progress_paths (BUG-1767).
1756 When a loop's own bookkeeping files appear in both progress_paths and the
1757 state action strings, every append to those files resets the stall window,
1758 silently disabling the BUG-1674 stall guard for that loop. Authors should
1759 move such files to exclude_paths so the stall detector can still fire.
1760 """
1761 if fsm.circuit is None or fsm.circuit.repeated_failure is None:
1762 return []
1763 rf = fsm.circuit.repeated_failure
1764 if not rf.progress_paths:
1765 return []
1767 # Build a set of the relative path components we need to look for.
1768 watched = {_strip_interpolation_prefix(p) for p in rf.progress_paths}
1769 # Exclude paths that are already in exclude_paths — author acknowledged.
1770 excluded = {_strip_interpolation_prefix(p) for p in rf.exclude_paths}
1771 active_watched = watched - excluded
1772 if not active_watched:
1773 return []
1775 errors: list[ValidationError] = []
1776 for state_name, state in fsm.states.items():
1777 if not state.action:
1778 continue
1779 for path_fragment in active_watched:
1780 if path_fragment in state.action:
1781 errors.append(
1782 ValidationError(
1783 message=(
1784 f"State action references '{path_fragment}', which is also "
1785 "listed in circuit.repeated_failure.progress_paths. Writes "
1786 "to this file will reset the stall window every cycle, "
1787 "silently disabling stall detection. Move it to "
1788 "circuit.repeated_failure.exclude_paths to separate "
1789 "bookkeeping files from real progress signals."
1790 ),
1791 path=f"states.{state_name}.action",
1792 severity=ValidationSeverity.WARNING,
1793 )
1794 )
1795 return errors
1798def _dominated_by_any(fsm: FSMLoop, dominators: set[str], dominated: str) -> bool:
1799 """Return True if the set ``dominators`` collectively dominates ``dominated``.
1801 Group domination: every path from the initial state to ``dominated`` must
1802 pass through at least one state in ``dominators``. Checked by removing all
1803 dominator states from the graph and testing whether ``dominated`` is still
1804 reachable from the initial state.
1806 This generalizes single-state domination — used when a capture variable is
1807 produced by more than one state on mutually-exclusive branches, where the
1808 reference is safe as long as *some* capturing state runs on every path.
1810 Args:
1811 fsm: The FSM loop to analyze
1812 dominators: Names of the states that should collectively dominate
1813 dominated: Name of the state that should be dominated
1815 Returns:
1816 True if the dominators collectively dominate ``dominated``
1817 """
1818 if dominated in dominators:
1819 return True
1820 if dominated not in fsm.states:
1821 return False
1823 visited: set[str] = set()
1824 to_visit: deque[str] = deque([fsm.initial])
1826 while to_visit:
1827 current = to_visit.popleft()
1828 if current in visited or current not in fsm.states:
1829 continue
1830 if current in dominators:
1831 continue # Block this node (simulate removal)
1833 visited.add(current)
1835 if current == dominated:
1836 # Reached dominated without going through any dominator
1837 return False
1839 state = fsm.states[current]
1840 for ref in state.get_referenced_states():
1841 if ref != "$current" and ref not in visited:
1842 to_visit.append(ref)
1844 # Dominated not reachable without the dominators → they dominate
1845 return True
1848def _dominates(fsm: FSMLoop, dominator: str, dominated: str) -> bool:
1849 """Return True if dominator dominates dominated in the FSM graph.
1851 A state D dominates S if every path from the initial state to S must pass
1852 through D. Thin single-state wrapper around :func:`_dominated_by_any`.
1853 """
1854 if dominator not in fsm.states:
1855 return False
1856 return _dominated_by_any(fsm, {dominator}, dominated)
1859def _find_bypass_path_any(fsm: FSMLoop, dominators: set[str], dominated: str) -> list[str]:
1860 """Find an example path from initial to dominated that bypasses all dominators.
1862 Uses BFS to find the shortest path that avoids every state in ``dominators``.
1863 Returns empty list if no bypass exists (should not happen when called after
1864 :func:`_dominated_by_any` returns False).
1865 """
1866 parent: dict[str, str] = {}
1867 to_visit: deque[str] = deque([fsm.initial])
1868 visited: set[str] = set()
1870 while to_visit:
1871 current = to_visit.popleft()
1872 if current in visited or current not in fsm.states:
1873 continue
1874 if current in dominators:
1875 continue
1877 visited.add(current)
1879 if current == dominated:
1880 # Reconstruct path
1881 path = [dominated]
1882 while path[-1] in parent:
1883 path.append(parent[path[-1]])
1884 path.reverse()
1885 return path
1887 state = fsm.states[current]
1888 for ref in state.get_referenced_states():
1889 if ref != "$current" and ref not in visited:
1890 if ref not in parent:
1891 parent[ref] = current
1892 to_visit.append(ref)
1894 return []
1897def _find_bypass_path(fsm: FSMLoop, dominator: str, dominated: str) -> list[str]:
1898 """Find an example path from initial to dominated that bypasses dominator.
1900 Thin single-state wrapper around :func:`_find_bypass_path_any`.
1901 """
1902 return _find_bypass_path_any(fsm, {dominator}, dominated)
1905def _has_sub_loop_state(fsm: FSMLoop) -> bool:
1906 """Return True if any state in the FSM has ``loop:`` set (delegates to a child loop).
1908 Used by ENH-1961 to distinguish "capture lives in a sub-loop" from "capture is missing".
1909 """
1910 return any(state.loop is not None for state in fsm.states.values())
1913def _validate_capture_reachability(fsm: FSMLoop) -> list[ValidationError]:
1914 """Validate that ``${captured.*}`` references are dominated by their capturing states.
1916 ENH-1961: For each state that references ``${captured.<var>.*}`` in its action
1917 or evaluate source, checks that the capturing state dominates the referencing
1918 state (i.e., all paths from the initial state pass through the capture state).
1920 Emits:
1921 - WARNING when a capture state does not dominate a referencing state
1922 (the reference may crash at runtime on paths that bypass the capture).
1923 - ERROR when a referenced capture variable has no capturing state at all
1924 in this FSM (excluding sub-loop captures which live in child namespaces).
1925 """
1926 errors: list[ValidationError] = []
1928 # Step 1: Build capture map (var_name → set of capturing state names).
1929 # A variable may be captured by more than one state on mutually-exclusive
1930 # branches (e.g. fifo_pop vs select_next dispatched by schedule_mode); the
1931 # reference is safe as long as the *set* collectively dominates it.
1932 capture_map: dict[str, set[str]] = {}
1933 for state_name, state in fsm.states.items():
1934 if state.capture:
1935 capture_map.setdefault(state.capture, set()).add(state_name)
1937 # Step 2: Build reference map (state_name → set of captured var names referenced)
1938 reference_map: dict[str, set[str]] = {}
1939 for state_name, state in fsm.states.items():
1940 # Skip sub-loop delegation states — their action is a loop name,
1941 # and captured vars belong to the child loop's namespace.
1942 if state.loop is not None:
1943 continue
1945 # Only collect references NOT guarded by `:default=` — a guarded
1946 # reference is safe even when the capture is missing on some path.
1947 refs: set[str] = set()
1948 if state.action:
1949 refs.update(_unguarded_captured_refs(state.action))
1950 if state.evaluate is not None and state.evaluate.source:
1951 refs.update(_unguarded_captured_refs(state.evaluate.source))
1952 if refs:
1953 reference_map[state_name] = refs
1955 if not reference_map:
1956 return errors
1958 # Step 3: For each reference, check dominance of capturing state
1959 for ref_state_name, ref_vars in reference_map.items():
1960 for var_name in ref_vars:
1961 if var_name not in capture_map:
1962 # Referenced capture variable has no capturing state in this FSM.
1963 # ENH-1998: downgrade to WARNING (not silent skip) when sub-loops
1964 # are present — the capture may live in a child namespace, but a
1965 # typo'd name should still surface rather than go completely dark.
1966 if _has_sub_loop_state(fsm):
1967 errors.append(
1968 ValidationError(
1969 message=(
1970 f"References ${{captured.{var_name}.*}} but no state in "
1971 f"this loop captures '{var_name}'. "
1972 f"If '{var_name}' is produced by a sub-loop, this may be "
1973 f"intentional; otherwise add 'capture: {var_name}' to the "
1974 f"state that produces this value."
1975 ),
1976 path=f"states.{ref_state_name}.action",
1977 severity=ValidationSeverity.WARNING,
1978 )
1979 )
1980 continue
1981 # No sub-loops: this is genuinely missing.
1982 errors.append(
1983 ValidationError(
1984 message=(
1985 f"References ${{captured.{var_name}.*}} but no state in "
1986 f"this loop captures '{var_name}'. Add 'capture: {var_name}' "
1987 f"to the state that produces this value."
1988 ),
1989 path=f"states.{ref_state_name}.action",
1990 severity=ValidationSeverity.ERROR,
1991 )
1992 )
1993 continue
1995 # Capturing states present in this FSM (shouldn't drop any normally)
1996 cap_states = {s for s in capture_map[var_name] if s in fsm.states}
1997 if not cap_states:
1998 continue
2000 # Group dominance check: do the capturing states collectively
2001 # dominate ref_state_name (does at least one run on every path)?
2002 if not _dominated_by_any(fsm, cap_states, ref_state_name):
2003 bypass_path = _find_bypass_path_any(fsm, cap_states, ref_state_name)
2004 path_str = " → ".join(bypass_path) if bypass_path else "unknown path"
2006 if len(cap_states) == 1:
2007 captured_by = f"state '{next(iter(cap_states))}' which may not"
2008 else:
2009 names = ", ".join(f"'{s}'" for s in sorted(cap_states))
2010 captured_by = f"states {names}, none of which"
2012 errors.append(
2013 ValidationError(
2014 message=(
2015 f"References ${{captured.{var_name}.*}} but '{var_name}' "
2016 f"is captured by {captured_by} "
2017 f"execute on all paths to '{ref_state_name}'. "
2018 f"Path(s) bypassing capture: {path_str}"
2019 ),
2020 path=f"states.{ref_state_name}.action",
2021 severity=ValidationSeverity.WARNING,
2022 )
2023 )
2025 return errors
2028def _find_reachable_states(fsm: FSMLoop) -> set[str]:
2029 """Find all states reachable from the initial state.
2031 Uses breadth-first search to find all reachable states. Seeds the BFS
2032 with the initial state plus top-level transition targets that act as
2033 alternate entry points: ``on_max_iterations`` (fires when the iteration
2034 cap is hit) and ``circuit.repeated_failure.on_repeated_failure`` (fires
2035 when the circuit breaker trips). These are real edges the runtime can
2036 take, so states reached only through them are not orphans.
2038 Args:
2039 fsm: The FSM loop to analyze
2041 Returns:
2042 Set of reachable state names
2043 """
2044 reachable: set[str] = set()
2045 to_visit: deque[str] = deque([fsm.initial])
2046 if fsm.on_max_steps is not None:
2047 to_visit.append(fsm.on_max_steps)
2048 if fsm.on_max_iterations is not None:
2049 to_visit.append(fsm.on_max_iterations)
2050 if fsm.circuit is not None and fsm.circuit.repeated_failure is not None:
2051 target = fsm.circuit.repeated_failure.on_repeated_failure
2052 if target not in STALL_SPECIAL_TOKENS:
2053 to_visit.append(target)
2055 while to_visit:
2056 current = to_visit.popleft()
2057 if current in reachable or current not in fsm.states:
2058 continue
2060 reachable.add(current)
2061 state = fsm.states[current]
2062 refs = state.get_referenced_states()
2064 for ref in refs:
2065 if ref != "$current" and ref not in reachable:
2066 to_visit.append(ref)
2068 return reachable
2071def is_runnable_loop(path: Path) -> bool:
2072 """Cheap check for whether a YAML file is a runnable FSM loop definition.
2074 Returns True iff the file parses as a YAML mapping with the required
2075 top-level keys ``name``, ``initial``, and either ``states`` or ``flow``
2076 (the shorthand resolved by :func:`resolve_flow`). This matches the
2077 required-fields gate in :func:`load_and_validate` so "counted by the
2078 verifier" stays in sync with "runnable by ll-loop validate".
2080 When the raw YAML contains a ``from:`` key, inheritance is resolved first
2081 (mirroring :func:`load_and_validate`) so pure context-override stubs whose
2082 parent provides ``initial``/``states`` return True. Library fragments under
2083 ``loops/lib/`` still return False — their parent chain also lacks ``initial``.
2084 """
2085 try:
2086 data = yaml.safe_load(path.read_text())
2087 except (OSError, yaml.YAMLError):
2088 return False
2089 if not isinstance(data, dict):
2090 return False
2091 if "from" in data:
2092 try:
2093 data = resolve_inheritance(data, path.parent)
2094 except Exception:
2095 return False
2096 has_flow = "states" in data or "flow" in data
2097 return "name" in data and "initial" in data and has_flow
2100def load_and_validate(
2101 path: Path,
2102 raise_on_error: bool = True,
2103) -> tuple[FSMLoop, list[ValidationError]]:
2104 """Load YAML file and validate FSM structure.
2106 Args:
2107 path: Path to the YAML file to load
2108 raise_on_error: When True (default), raise ValueError on ERROR violations.
2109 When False, return all violations (errors + warnings) without raising.
2111 Returns:
2112 When raise_on_error=True: (FSMLoop, list of WARNING-severity ValidationErrors)
2113 When raise_on_error=False: (FSMLoop, list of all ValidationErrors sorted errors-first)
2115 Raises:
2116 FileNotFoundError: If the file doesn't exist
2117 yaml.YAMLError: If the file is not valid YAML
2118 ValueError: If raise_on_error=True and validation fails (contains error details)
2119 """
2120 if not path.exists():
2121 raise FileNotFoundError(f"FSM file not found: {path}")
2123 with open(path) as f:
2124 data: dict[str, Any] = yaml.safe_load(f)
2126 if not isinstance(data, dict):
2127 raise ValueError(f"FSM file must contain a YAML mapping, got {type(data)}")
2129 # Resolve `from:` inheritance before any further checks, so a child loop
2130 # can omit fields its parent provides (including `initial`/`states`) and
2131 # so a parent's `import:`/`fragments:` blocks survive into the merged
2132 # result for the subsequent `resolve_fragments` pass.
2133 data = resolve_inheritance(data, path.parent)
2135 # Expand flow: linear shorthand into states: before required-fields check
2136 data = resolve_flow(data)
2138 # Check required fields before parsing
2139 missing = []
2140 for field in ["name", "initial"]:
2141 if field not in data:
2142 missing.append(field)
2143 if "states" not in data:
2144 missing.append("states (or flow)")
2146 if missing:
2147 raise ValueError(f"FSM file missing required fields: {', '.join(missing)}")
2149 # Check for unknown top-level keys before parsing
2150 unknown_key_warnings: list[ValidationError] = []
2151 unknown = set(data.keys()) - KNOWN_TOP_LEVEL_KEYS
2152 if unknown:
2153 unknown_key_warnings.append(
2154 ValidationError(
2155 path="<root>",
2156 message=f"Unknown top-level keys: {', '.join(sorted(unknown))}",
2157 severity=ValidationSeverity.WARNING,
2158 )
2159 )
2161 visibility_val = data.get("visibility")
2162 if visibility_val is not None and visibility_val not in VALID_VISIBILITY:
2163 unknown_key_warnings.append(
2164 ValidationError(
2165 path="visibility",
2166 message=(
2167 f"Invalid visibility: {visibility_val!r}. "
2168 f"Must be one of: {', '.join(sorted(VALID_VISIBILITY))}. "
2169 "Loop will be treated as 'public'."
2170 ),
2171 severity=ValidationSeverity.WARNING,
2172 )
2173 )
2175 # Resolve fragment libraries before parsing into dataclass
2176 data = resolve_fragments(data, path.parent)
2178 # Parse into dataclass
2179 fsm = FSMLoop.from_dict(data)
2181 # Validate
2182 errors = validate_fsm(fsm)
2184 # Validate with: bindings against child loop parameters (requires file-system access)
2185 errors.extend(_validate_with_bindings(fsm, path.parent))
2186 errors.extend(_validate_loop_references(fsm, path.parent))
2187 errors.extend(_validate_fragment_bindings(fsm, path.parent))
2189 # Filter to errors only (not warnings) for raising
2190 error_list = [e for e in errors if e.severity == ValidationSeverity.ERROR]
2191 struct_warnings = [e for e in errors if e.severity == ValidationSeverity.WARNING]
2192 all_warnings = unknown_key_warnings + struct_warnings
2194 if not raise_on_error:
2195 return fsm, error_list + all_warnings
2197 if error_list:
2198 error_messages = "\n ".join(str(e) for e in error_list)
2199 raise ValueError(f"FSM validation failed:\n {error_messages}")
2201 for warning in all_warnings:
2202 logger.warning(str(warning))
2204 return fsm, all_warnings