Coverage for little_loops / fsm / validation.py: 58%
531 statements
« prev ^ index » next coverage.py v7.12.0, created at 2026-06-04 12:21 -0500
1"""FSM loop validation logic.
3This module provides validation for FSM loop definitions, ensuring
4structural correctness and catching common configuration errors.
6Validation checks:
7- Initial state exists in states dict
8- All referenced states exist
9- At least one terminal state
10- Evaluator configs have required fields for their type
11- No conflicting routing (shorthand vs full route)
12- Numeric fields in valid ranges (max_iterations > 0, backoff >= 0, timeout > 0)
13"""
15from __future__ import annotations
17import logging
18import re
19from collections import deque
20from dataclasses import dataclass
21from enum import Enum
22from pathlib import Path
23from typing import Any
25import yaml
27from little_loops.fsm.evaluators import _NUMERIC_OPERATORS
28from little_loops.fsm.fragments import resolve_flow, resolve_fragments, resolve_inheritance
29from little_loops.fsm.schema import EvaluateConfig, FSMLoop, ParameterSpec, StateConfig
31logger = logging.getLogger(__name__)
34class ValidationSeverity(Enum):
35 """Severity level for validation issues."""
37 ERROR = "error"
38 WARNING = "warning"
41@dataclass
42class ValidationError:
43 """Structured validation error.
45 Attributes:
46 message: Human-readable error description
47 path: Path to the problematic element (e.g., "states.check.route")
48 severity: Error severity (error or warning)
49 """
51 message: str
52 path: str | None = None
53 severity: ValidationSeverity = ValidationSeverity.ERROR
55 def __str__(self) -> str:
56 """Format error for display."""
57 prefix = f"[{self.severity.value.upper()}]"
58 if self.path:
59 return f"{prefix} {self.path}: {self.message}"
60 return f"{prefix} {self.message}"
63# Evaluator type to required fields mapping
64EVALUATOR_REQUIRED_FIELDS: dict[str, list[str]] = {
65 "exit_code": [],
66 "output_numeric": ["operator", "target"],
67 "output_json": ["path", "operator", "target"],
68 "output_contains": ["pattern"],
69 "convergence": ["target"],
70 "diff_stall": [],
71 "action_stall": [],
72 "llm_structured": [],
73 "mcp_result": [],
74 "harbor_scorer": [],
75 "comparator": ["baseline_path"],
76 "contract": ["pairs"],
77}
79# Non-LLM evaluator types: all evaluator types except llm_structured
80# Derived from EVALUATOR_REQUIRED_FIELDS so new types are automatically included
81NON_LLM_EVALUATOR_TYPES: frozenset[str] = frozenset(EVALUATOR_REQUIRED_FIELDS.keys()) - {
82 "llm_structured",
83 "comparator",
84 "contract",
85}
87# Meta-loop detector: action string patterns that indicate harness artifact writes
88_META_LOOP_ACTION_PATTERNS: tuple[re.Pattern[str], ...] = (
89 re.compile(r"loops/[\w-]+\.yaml"),
90 re.compile(r"skills/[\w-]+/SKILL\.md"),
91 re.compile(r"agents/[\w-]+\.md"),
92 re.compile(r"commands/[\w-]+\.md"),
93 re.compile(r"\.claude/(CLAUDE\.md|settings)"),
94)
96# Action string tokens that indicate meta-loop behavior
97_META_LOOP_ACTION_TOKENS: frozenset[str] = frozenset({"yaml_state_editor", "replace_action"})
99# Import paths that identify a loop as a meta-loop (harness optimization framework)
100_META_LOOP_IMPORT_TRIGGERS: frozenset[str] = frozenset({"lib/benchmark.yaml"})
102# MR-3: shared-tmp path detector. The runner injects ${context.run_dir} resolving
103# to .loops/runs/<loop>-<timestamp>/; loops that hardcode .loops/tmp/ instead
104# cause state corruption under concurrent runs (ll-parallel, retries, etc.).
105_SHARED_TMP_PATH_RE = re.compile(r"\.loops/tmp/[\w./-]+")
107# ENH-1819: Regex patterns for detecting multimodal evaluation in prompt actions
108_MULTIMODAL_EVAL_PATTERNS: tuple[re.Pattern[str], ...] = (
109 re.compile(r"Read the screenshot", re.IGNORECASE),
110 re.compile(r"view the (generated )?(website|page|image)", re.IGNORECASE),
111 re.compile(r"screenshot\.(png|jpg|jpeg|webp)"),
112 re.compile(r"\.(png|jpg|jpeg|webp)\b.*\b(read|view|evaluate|score|judge)", re.IGNORECASE),
113)
115# Valid comparison operators
116VALID_OPERATORS = {"eq", "ne", "lt", "le", "gt", "ge"}
118# All top-level keys recognized by FSMLoop.from_dict()
119KNOWN_TOP_LEVEL_KEYS: frozenset[str] = frozenset(
120 {
121 "name",
122 "description",
123 "initial",
124 "states",
125 "context",
126 "parameters",
127 "scope",
128 "max_iterations",
129 "on_max_iterations",
130 "max_edge_revisits",
131 "backoff",
132 "timeout",
133 "default_timeout",
134 "maintain",
135 "llm",
136 "on_handoff",
137 "input_key",
138 "required_inputs",
139 "config",
140 "category",
141 "labels",
142 "commands",
143 "targets",
144 "circuit",
145 "meta_self_eval_ok",
146 "shared_state_ok",
147 "partial_route_ok",
148 "import",
149 "fragments",
150 "from",
151 "flow",
152 "state_defs",
153 }
154)
156# Special tokens accepted as the `on_repeated_failure` target on the
157# stall detector — `"abort"` means terminate via _finish("stall_detected").
158STALL_SPECIAL_TOKENS: frozenset[str] = frozenset({"abort"})
160# Valid parameter types for the 'parameters:' block
161VALID_PARAMETER_TYPES: frozenset[str] = frozenset(
162 {"string", "integer", "number", "boolean", "enum", "path"}
163)
166def _validate_evaluator(state_name: str, evaluate: EvaluateConfig) -> list[ValidationError]:
167 """Validate evaluator configuration for type-specific requirements.
169 Args:
170 state_name: Name of the state containing this evaluator
171 evaluate: The evaluator configuration to validate
173 Returns:
174 List of validation errors found
175 """
176 errors: list[ValidationError] = []
177 path = f"states.{state_name}.evaluate"
179 # Check that evaluator type is recognized
180 valid_types = set(EVALUATOR_REQUIRED_FIELDS.keys())
181 if evaluate.type not in valid_types:
182 errors.append(
183 ValidationError(
184 message=f"Unknown evaluator type '{evaluate.type}'. "
185 f"Must be one of: {', '.join(sorted(valid_types))}",
186 path=path,
187 )
188 )
189 return errors # Can't check required fields for unknown type
191 # Check required fields for evaluator type
192 required = EVALUATOR_REQUIRED_FIELDS.get(evaluate.type, [])
193 for field_name in required:
194 value = getattr(evaluate, field_name, None)
195 if value is None:
196 errors.append(
197 ValidationError(
198 message=f"Evaluator type '{evaluate.type}' requires '{field_name}' field",
199 path=path,
200 )
201 )
203 # Validate operator if present
204 if evaluate.operator is not None and evaluate.operator not in VALID_OPERATORS:
205 errors.append(
206 ValidationError(
207 message=f"Invalid operator '{evaluate.operator}'. "
208 f"Must be one of: {', '.join(sorted(VALID_OPERATORS))}",
209 path=f"{path}.operator",
210 )
211 )
213 # Validate convergence-specific fields
214 if evaluate.type == "convergence":
215 if evaluate.direction not in ("minimize", "maximize"):
216 errors.append(
217 ValidationError(
218 message=f"Invalid direction '{evaluate.direction}'. "
219 "Must be 'minimize' or 'maximize'",
220 path=f"{path}.direction",
221 )
222 )
223 # Only validate tolerance if it's a numeric value (not an interpolation string)
224 if (
225 evaluate.tolerance is not None
226 and isinstance(evaluate.tolerance, (int, float))
227 and evaluate.tolerance < 0
228 ):
229 errors.append(
230 ValidationError(
231 message="Tolerance cannot be negative",
232 path=f"{path}.tolerance",
233 )
234 )
236 # Validate llm_structured-specific fields
237 if evaluate.type == "llm_structured":
238 if evaluate.min_confidence < 0 or evaluate.min_confidence > 1:
239 errors.append(
240 ValidationError(
241 message="min_confidence must be between 0 and 1",
242 path=f"{path}.min_confidence",
243 )
244 )
246 # Validate diff_stall-specific fields
247 if evaluate.type == "diff_stall":
248 if evaluate.max_stall < 1:
249 errors.append(
250 ValidationError(
251 message="max_stall must be >= 1",
252 path=f"{path}.max_stall",
253 )
254 )
256 # Validate action_stall-specific fields
257 if evaluate.type == "action_stall":
258 if evaluate.max_repeat < 1:
259 errors.append(
260 ValidationError(
261 message="max_repeat must be >= 1",
262 path=f"{path}.max_repeat",
263 )
264 )
266 return errors
269def _validate_parameters(fsm: FSMLoop) -> list[ValidationError]:
270 """Validate the loop's top-level parameters: block.
272 Args:
273 fsm: The FSM loop to validate
275 Returns:
276 List of validation errors found
277 """
278 errors: list[ValidationError] = []
280 for param_name, param_spec in fsm.parameters.items():
281 path = f"parameters.{param_name}"
283 if param_spec.type not in VALID_PARAMETER_TYPES:
284 errors.append(
285 ValidationError(
286 message=(
287 f"Unknown parameter type '{param_spec.type}'. "
288 f"Must be one of: {', '.join(sorted(VALID_PARAMETER_TYPES))}"
289 ),
290 path=path,
291 )
292 )
294 if param_spec.type == "enum" and not param_spec.values:
295 errors.append(
296 ValidationError(
297 message="Parameter type 'enum' requires a 'values' list",
298 path=path,
299 )
300 )
302 if param_spec.required and param_spec.default is not None:
303 errors.append(
304 ValidationError(
305 message="Parameter cannot be both 'required: true' and have a 'default' value",
306 path=path,
307 )
308 )
310 return errors
313def _check_param_type(value: Any, spec: ParameterSpec) -> str | None:
314 """Return an error message if value does not match spec.type, else None."""
315 if spec.type == "string" and not isinstance(value, str):
316 return f"expected string, got {type(value).__name__}"
317 if spec.type == "integer" and not isinstance(value, int):
318 return f"expected integer, got {type(value).__name__}"
319 if spec.type == "number" and not isinstance(value, (int, float)):
320 return f"expected number, got {type(value).__name__}"
321 if spec.type == "boolean" and not isinstance(value, bool):
322 return f"expected boolean, got {type(value).__name__}"
323 if spec.type == "enum" and spec.values and value not in spec.values:
324 return f"expected one of {spec.values!r}, got {value!r}"
325 return None
328def _validate_with_bindings(fsm: FSMLoop, loop_dir: Path) -> list[ValidationError]:
329 """Validate with: bindings against child loop parameter contracts.
331 Called from load_and_validate (not validate_fsm) because resolving child loops
332 requires file-system access via the loop directory path.
334 Args:
335 fsm: The parent FSM loop
336 loop_dir: Directory to resolve child loop paths from
338 Returns:
339 List of validation errors found
340 """
341 errors: list[ValidationError] = []
343 for state_name, state in fsm.states.items():
344 if state.loop is None or not state.with_:
345 continue
347 # Try to resolve and load the child loop; skip if unavailable
348 try:
349 from little_loops.cli.loop._helpers import resolve_loop_path
351 loop_path = resolve_loop_path(state.loop, loop_dir)
352 child_fsm, _ = load_and_validate(loop_path)
353 except Exception:
354 continue
356 if not child_fsm.parameters:
357 continue # Child has no declared contract — nothing to cross-validate
359 path = f"states.{state_name}"
361 # Unknown with: keys (not declared by child)
362 for key in state.with_:
363 if key not in child_fsm.parameters:
364 errors.append(
365 ValidationError(
366 message=(
367 f"'with.{key}' is not a declared parameter of loop '{state.loop}'. "
368 f"Declared: {', '.join(sorted(child_fsm.parameters))}"
369 ),
370 path=f"{path}.with.{key}",
371 )
372 )
374 # Required parameters not bound
375 for param_name, param_spec in child_fsm.parameters.items():
376 if param_spec.required and param_name not in state.with_:
377 errors.append(
378 ValidationError(
379 message=(
380 f"Required parameter '{param_name}' of loop '{state.loop}' "
381 f"is not bound in 'with'"
382 ),
383 path=f"{path}.with",
384 )
385 )
387 # Statically-detectable type mismatches (skip interpolation strings)
388 for param_name, value in state.with_.items():
389 if param_name not in child_fsm.parameters:
390 continue
391 if isinstance(value, str) and "${" in value:
392 continue
393 type_error = _check_param_type(value, child_fsm.parameters[param_name])
394 if type_error:
395 errors.append(
396 ValidationError(
397 message=f"Parameter '{param_name}': {type_error}",
398 path=f"{path}.with.{param_name}",
399 )
400 )
402 return errors
405def _validate_fragment_bindings(fsm: FSMLoop, loop_dir: Path) -> list[ValidationError]:
406 """Validate fragment with: bindings against fragment parameter contracts.
408 Called from load_and_validate (not validate_fsm) because fragment parameters
409 are populated by resolve_fragments which runs before dataclass parsing.
411 Args:
412 fsm: The FSM loop to validate
413 loop_dir: Directory containing the loop file (unused; kept for API symmetry with
414 _validate_with_bindings)
416 Returns:
417 List of validation errors found
418 """
419 # Runner-injected vars available at runtime but not at static analysis time
420 RUNNER_INJECTED = {"run_dir", "loop_name", "started_at"}
422 errors: list[ValidationError] = []
424 for state_name, state in fsm.states.items():
425 if not state.fragment_parameters:
426 continue # No declared contract — nothing to cross-validate
428 path = f"states.{state_name}"
430 # Unknown with: keys (not declared by fragment)
431 for key in state.fragment_bindings:
432 if key not in state.fragment_parameters:
433 errors.append(
434 ValidationError(
435 message=(
436 f"'with.{key}' is not a declared parameter of fragment "
437 f"'{state.fragment_name}'. "
438 f"Declared: {', '.join(sorted(state.fragment_parameters))}"
439 ),
440 path=f"{path}.with.{key}",
441 )
442 )
444 # Required parameters not bound (whitelist runner-injected vars)
445 for param_name, param_spec in state.fragment_parameters.items():
446 if param_spec.required and param_name not in state.fragment_bindings:
447 if param_name in RUNNER_INJECTED:
448 continue # Available at runtime; not a static error
449 errors.append(
450 ValidationError(
451 message=(
452 f"Required parameter '{param_name}' of fragment "
453 f"'{state.fragment_name}' is not bound in 'with'"
454 ),
455 path=f"{path}.with",
456 )
457 )
459 # Statically-detectable type mismatches (skip interpolation strings)
460 for param_name, value in state.fragment_bindings.items():
461 if param_name not in state.fragment_parameters:
462 continue
463 if isinstance(value, str) and "${" in value:
464 continue
465 type_error = _check_param_type(value, state.fragment_parameters[param_name])
466 if type_error:
467 errors.append(
468 ValidationError(
469 message=f"Parameter '{param_name}': {type_error}",
470 path=f"{path}.with.{param_name}",
471 )
472 )
474 return errors
477def _validate_state_action(state_name: str, state: StateConfig) -> list[ValidationError]:
478 """Validate state action configuration.
480 Args:
481 state_name: Name of the state to validate
482 state: The state configuration to validate
484 Returns:
485 List of validation errors found
486 """
487 errors: list[ValidationError] = []
488 path = f"states.{state_name}"
490 # append_to_messages must contain at least one ${...} interpolation expression
491 if state.append_to_messages is not None:
492 if "${" not in state.append_to_messages:
493 errors.append(
494 ValidationError(
495 message=(
496 "'append_to_messages' must contain a ${...} interpolation expression "
497 f"(e.g. '${{captured.{state_name}.output}}')"
498 ),
499 path=f"{path}.append_to_messages",
500 )
501 )
503 # params field is only valid for mcp_tool states
504 if state.params and state.action_type != "mcp_tool":
505 errors.append(
506 ValidationError(
507 message="'params' field is only valid when action_type is 'mcp_tool'",
508 path=f"{path}.params",
509 )
510 )
512 # loop and action are mutually exclusive
513 if state.loop is not None and state.action is not None:
514 errors.append(
515 ValidationError(
516 message="'loop' and 'action' are mutually exclusive — "
517 "a sub-loop state cannot also have an action",
518 path=f"{path}",
519 )
520 )
522 # with: requires loop: to be set
523 if state.with_ and state.loop is None:
524 errors.append(
525 ValidationError(
526 message="'with' is only valid when 'loop' is set",
527 path=f"{path}.with",
528 )
529 )
531 # FEAT-1283: type=learning requires a populated LearningConfig
532 if state.type == "learning" and state.learning is not None:
533 if not state.learning.targets and not state.learning.targets_csv:
534 errors.append(
535 ValidationError(
536 message="type=learning requires non-empty 'learning.targets' or 'learning.targets_csv'",
537 path=f"{path}.learning.targets",
538 )
539 )
540 if state.learning.max_retries < 0:
541 errors.append(
542 ValidationError(
543 message=(
544 f"learning.max_retries must be >= 0, got {state.learning.max_retries}"
545 ),
546 path=f"{path}.learning.max_retries",
547 )
548 )
549 if state.on_yes is None:
550 errors.append(
551 ValidationError(
552 message="type=learning requires 'on_yes' (target for all-proven)",
553 path=f"{path}.on_yes",
554 )
555 )
556 if state.on_blocked is None and state.on_no is None:
557 errors.append(
558 ValidationError(
559 message=(
560 "type=learning requires 'on_blocked' or 'on_no' "
561 "(target for refuted / retries_exhausted)"
562 ),
563 path=f"{path}",
564 )
565 )
567 # with: and context_passthrough are mutually exclusive
568 if state.with_ and state.context_passthrough:
569 errors.append(
570 ValidationError(
571 message=(
572 "'with' and 'context_passthrough' are mutually exclusive — "
573 "use 'with' for explicit parameter bindings or 'context_passthrough' "
574 "for legacy bulk passthrough, not both"
575 ),
576 path=f"{path}",
577 )
578 )
580 return errors
583def _validate_state_routing(state_name: str, state: StateConfig) -> list[ValidationError]:
584 """Validate state routing configuration.
586 Checks for conflicting routing definitions (shorthand vs full route).
588 Args:
589 state_name: Name of the state to validate
590 state: The state configuration to validate
592 Returns:
593 List of validation errors/warnings found
594 """
595 errors: list[ValidationError] = []
596 path = f"states.{state_name}"
598 has_shorthand = (
599 state.on_yes is not None
600 or state.on_no is not None
601 or state.on_error is not None
602 or state.on_partial is not None
603 or state.on_blocked is not None
604 or bool(state.extra_routes)
605 )
606 has_route = state.route is not None
608 # Warn about conflicting definitions
609 if has_shorthand and has_route:
610 errors.append(
611 ValidationError(
612 message="Both shorthand routing (on_yes/on_no/on_error) "
613 "and full route table defined. Route table will take precedence.",
614 path=path,
615 severity=ValidationSeverity.WARNING,
616 )
617 )
619 # Check for no valid transition definition
620 has_next = state.next is not None
621 has_terminal = state.terminal
622 has_loop = state.loop is not None
624 if not has_shorthand and not has_route and not has_next and not has_terminal and not has_loop:
625 errors.append(
626 ValidationError(
627 message="State has no transition defined. Add routing, 'next', "
628 "or mark as 'terminal: true'",
629 path=path,
630 )
631 )
633 # Validate retry field pairing: max_retries requires on_retry_exhausted and vice versa
634 if state.max_retries is not None and state.on_retry_exhausted is None:
635 errors.append(
636 ValidationError(
637 message="'max_retries' requires 'on_retry_exhausted' to also be set",
638 path=path,
639 )
640 )
641 if state.on_retry_exhausted is not None and state.max_retries is None:
642 errors.append(
643 ValidationError(
644 message="'on_retry_exhausted' requires 'max_retries' to also be set",
645 path=path,
646 )
647 )
648 if state.max_retries is not None and state.max_retries < 1:
649 errors.append(
650 ValidationError(
651 message=f"'max_retries' must be >= 1, got {state.max_retries}",
652 path=path,
653 )
654 )
656 # Validate retryable_exit_codes: requires on_error; all codes must be positive ints
657 if state.retryable_exit_codes is not None:
658 if state.on_error is None:
659 errors.append(
660 ValidationError(
661 message="'retryable_exit_codes' requires 'on_error' to also be set",
662 path=path,
663 )
664 )
665 for code in state.retryable_exit_codes:
666 if not isinstance(code, int) or code < 1:
667 errors.append(
668 ValidationError(
669 message=(
670 f"'retryable_exit_codes' entries must be positive "
671 f"integers, got {code!r}"
672 ),
673 path=f"{path}.retryable_exit_codes",
674 )
675 )
676 break
678 # Validate rate-limit retry field pairing (mirrors max_retries/on_retry_exhausted)
679 if state.max_rate_limit_retries is not None and state.on_rate_limit_exhausted is None:
680 errors.append(
681 ValidationError(
682 message="'max_rate_limit_retries' requires 'on_rate_limit_exhausted' to also be set",
683 path=path,
684 )
685 )
686 if state.on_rate_limit_exhausted is not None and state.max_rate_limit_retries is None:
687 errors.append(
688 ValidationError(
689 message="'on_rate_limit_exhausted' requires 'max_rate_limit_retries' to also be set",
690 path=path,
691 )
692 )
693 if state.max_rate_limit_retries is not None and state.max_rate_limit_retries < 1:
694 errors.append(
695 ValidationError(
696 message=f"'max_rate_limit_retries' must be >= 1, got {state.max_rate_limit_retries}",
697 path=path,
698 )
699 )
700 if (
701 state.rate_limit_backoff_base_seconds is not None
702 and state.rate_limit_backoff_base_seconds < 1
703 ):
704 errors.append(
705 ValidationError(
706 message=(
707 f"'rate_limit_backoff_base_seconds' must be >= 1, "
708 f"got {state.rate_limit_backoff_base_seconds}"
709 ),
710 path=path,
711 )
712 )
713 if state.rate_limit_max_wait_seconds is not None and state.rate_limit_max_wait_seconds < 1:
714 errors.append(
715 ValidationError(
716 message=(
717 f"'rate_limit_max_wait_seconds' must be >= 1, "
718 f"got {state.rate_limit_max_wait_seconds}"
719 ),
720 path=path,
721 )
722 )
723 if state.rate_limit_long_wait_ladder is not None:
724 if len(state.rate_limit_long_wait_ladder) == 0:
725 errors.append(
726 ValidationError(
727 message="'rate_limit_long_wait_ladder' must be non-empty if specified",
728 path=path,
729 )
730 )
731 else:
732 for idx, value in enumerate(state.rate_limit_long_wait_ladder):
733 if not isinstance(value, int) or value < 1:
734 errors.append(
735 ValidationError(
736 message=(
737 f"'rate_limit_long_wait_ladder[{idx}]' must be a "
738 f"positive integer, got {value!r}"
739 ),
740 path=path,
741 )
742 )
744 # Validate throttle config when present
745 if state.throttle is not None:
746 t = state.throttle
747 fields = {
748 "normal_max": t.normal_max,
749 "warn_max": t.warn_max,
750 "hard_max": t.hard_max,
751 }
752 for field_name, val in fields.items():
753 if val is not None and (not isinstance(val, int) or val < 1):
754 errors.append(
755 ValidationError(
756 message=f"'throttle.{field_name}' must be a positive integer, got {val!r}",
757 path=path,
758 )
759 )
760 # Enforce ordering when all three are set
761 if t.normal_max is not None and t.warn_max is not None and t.normal_max >= t.warn_max:
762 errors.append(
763 ValidationError(
764 message=(
765 f"'throttle.normal_max' ({t.normal_max}) must be less than "
766 f"'throttle.warn_max' ({t.warn_max})"
767 ),
768 path=path,
769 )
770 )
771 if t.warn_max is not None and t.hard_max is not None and t.warn_max >= t.hard_max:
772 errors.append(
773 ValidationError(
774 message=(
775 f"'throttle.warn_max' ({t.warn_max}) must be less than "
776 f"'throttle.hard_max' ({t.hard_max})"
777 ),
778 path=path,
779 )
780 )
782 return errors
785def _validate_targets(fsm: FSMLoop) -> list[ValidationError]:
786 """Validate top-level targets[] entries (ENH-1552).
788 Rejects any targets[].states[] entry whose sibling file: value does not
789 end with a .yaml extension.
790 """
791 errors: list[ValidationError] = []
792 for i, target in enumerate(fsm.targets):
793 if target.file is not None and not target.file.endswith(".yaml"):
794 errors.append(
795 ValidationError(
796 message=(f"targets[{i}].file must be a .yaml file, got '{target.file}'"),
797 path=f"targets[{i}].file",
798 )
799 )
800 return errors
803def _validate_failure_terminal_action(fsm: FSMLoop) -> list[ValidationError]:
804 """Warn when a failure-named terminal state has no diagnostic predecessor.
806 Failure terminals (failed, error, aborted) should have at least one
807 predecessor state with an action or sub-loop that provides diagnostic
808 output before termination. Otherwise the failure is silent — the
809 executor calls _finish("terminal") before any action on the terminal
810 itself can execute.
812 Severity is WARNING (not ERROR) so that existing loops with bare
813 failure terminals continue to load, and test_terminal_only_state_valid
814 (which filters by ERROR) passes without modification.
815 """
816 FAILURE_TERMINAL_NAMES: frozenset[str] = frozenset({"failed", "error", "aborted"})
817 errors: list[ValidationError] = []
819 terminal_states = fsm.get_terminal_states()
820 failure_terminals = terminal_states & FAILURE_TERMINAL_NAMES
822 for ft_name in failure_terminals:
823 has_diagnostic_predecessor = False
824 for state_name, state in fsm.states.items():
825 if state_name == ft_name:
826 continue
827 if ft_name in state.get_referenced_states():
828 if state.action is not None or state.loop is not None:
829 has_diagnostic_predecessor = True
830 break
832 if not has_diagnostic_predecessor:
833 errors.append(
834 ValidationError(
835 message=(
836 f"Failure-named terminal state '{ft_name}' has no predecessor "
837 "state with a diagnostic action. Add a non-terminal diagnostic "
838 "state (e.g. 'diagnose') with an action or sub-loop that routes "
839 f"to '{ft_name}'."
840 ),
841 path=f"states.{ft_name}",
842 severity=ValidationSeverity.WARNING,
843 )
844 )
846 return errors
849def validate_fsm(fsm: FSMLoop) -> list[ValidationError]:
850 """Validate FSM structure and return list of errors.
852 Performs comprehensive validation:
853 - Initial state exists
854 - All referenced states exist
855 - At least one terminal state
856 - Evaluator configurations are valid
857 - Routing configurations are valid
858 - Numeric fields are in valid ranges (max_iterations > 0, backoff >= 0, timeout > 0)
860 Args:
861 fsm: The FSM loop to validate
863 Returns:
864 List of validation errors (empty if valid)
865 """
866 errors: list[ValidationError] = []
867 defined_states = fsm.get_all_state_names()
869 # Warn when no top-level description: field is set. The field is optional
870 # for FSM execution but required for goal-alignment skills (debug-loop-run,
871 # audit-loop-run) and for ll-loop show --json to surface intent text.
872 if not fsm.description:
873 errors.append(
874 ValidationError(
875 path="<root>",
876 message=("No 'description' field defined. Add a top-level description: key."),
877 severity=ValidationSeverity.WARNING,
878 )
879 )
881 # Validate parameters block
882 errors.extend(_validate_parameters(fsm))
884 # Validate targets block (ENH-1552)
885 errors.extend(_validate_targets(fsm))
887 # Check initial state exists
888 if fsm.initial not in defined_states:
889 errors.append(
890 ValidationError(
891 message=f"Initial state '{fsm.initial}' not found in states",
892 path="initial",
893 )
894 )
896 # Check at least one terminal state
897 terminal_states = fsm.get_terminal_states()
898 if not terminal_states:
899 errors.append(
900 ValidationError(
901 message="No terminal state defined. At least one state must have 'terminal: true'",
902 path="states",
903 )
904 )
906 # Validate each state
907 for state_name, state in fsm.states.items():
908 # Check all referenced states exist
909 refs = state.get_referenced_states()
910 for ref in refs:
911 # $current is a special token for retry
912 if ref != "$current" and ref not in defined_states:
913 errors.append(
914 ValidationError(
915 message=f"References unknown state '{ref}'",
916 path=f"states.{state_name}",
917 )
918 )
920 # Validate action configuration
921 errors.extend(_validate_state_action(state_name, state))
923 # Validate evaluator if present
924 if state.evaluate is not None:
925 errors.extend(_validate_evaluator(state_name, state.evaluate))
927 # Validate routing configuration
928 errors.extend(_validate_state_routing(state_name, state))
930 # Check numeric field ranges
931 if fsm.max_iterations <= 0:
932 errors.append(
933 ValidationError(
934 message=f"max_iterations must be > 0, got {fsm.max_iterations}",
935 path="max_iterations",
936 )
937 )
938 if fsm.max_edge_revisits <= 0:
939 errors.append(
940 ValidationError(
941 message=f"max_edge_revisits must be > 0, got {fsm.max_edge_revisits}",
942 path="max_edge_revisits",
943 )
944 )
945 if fsm.backoff is not None and fsm.backoff < 0:
946 errors.append(
947 ValidationError(
948 message=f"backoff must be >= 0, got {fsm.backoff}",
949 path="backoff",
950 )
951 )
952 if fsm.timeout is not None and fsm.timeout <= 0:
953 errors.append(
954 ValidationError(
955 message=f"timeout must be > 0, got {fsm.timeout}",
956 path="timeout",
957 )
958 )
959 if fsm.llm.max_tokens <= 0:
960 errors.append(
961 ValidationError(
962 message=f"llm.max_tokens must be > 0, got {fsm.llm.max_tokens}",
963 path="llm.max_tokens",
964 )
965 )
966 if fsm.llm.timeout <= 0:
967 errors.append(
968 ValidationError(
969 message=f"llm.timeout must be > 0, got {fsm.llm.timeout}",
970 path="llm.timeout",
971 )
972 )
974 # Check for unreachable states (warning only)
975 reachable = _find_reachable_states(fsm)
976 unreachable = defined_states - reachable
977 for state_name in unreachable:
978 errors.append(
979 ValidationError(
980 message="State is not reachable from initial state",
981 path=f"states.{state_name}",
982 severity=ValidationSeverity.WARNING,
983 )
984 )
986 errors.extend(_validate_failure_terminal_action(fsm))
988 errors.extend(_validate_meta_loop_evaluation(fsm))
990 errors.extend(_validate_input_key_without_guard(fsm))
992 errors.extend(_validate_artifact_isolation(fsm))
994 errors.extend(_validate_harness_multimodal_evaluator_blind_spot(fsm))
996 errors.extend(_validate_partial_route_dead_end(fsm))
998 errors.extend(_validate_zero_retry_counter(fsm))
1000 errors.extend(_validate_on_max_iterations(fsm, defined_states))
1002 errors.extend(_validate_circuit(fsm, defined_states))
1004 errors.extend(_validate_progress_paths_isolation(fsm))
1006 return errors
1009def _is_meta_loop(fsm: FSMLoop) -> bool:
1010 """Return True if fsm is classified as a meta-loop.
1012 A loop is meta if ANY of the following match:
1013 1. Any state action string matches a harness-artifact path regex
1014 (writes another loop YAML, skill, agent, command, or project config)
1015 2. The loop's import list contains lib/benchmark.yaml
1016 3. Any state action references yaml_state_editor or replace_action
1017 """
1018 # Condition 2: imports lib/benchmark.yaml
1019 if any(imp in _META_LOOP_IMPORT_TRIGGERS for imp in fsm.imports):
1020 return True
1021 # Conditions 1 and 3: scan action strings
1022 for state in fsm.states.values():
1023 if state.action is None:
1024 continue
1025 for pattern in _META_LOOP_ACTION_PATTERNS:
1026 if pattern.search(state.action):
1027 return True
1028 for token in _META_LOOP_ACTION_TOKENS:
1029 if token in state.action:
1030 return True
1031 return False
1034def _validate_meta_loop_evaluation(fsm: FSMLoop) -> list[ValidationError]:
1035 """Validate meta-loop evaluation rules MR-1 and MR-2.
1037 MR-1 (ERROR): meta-loop must have at least one non-LLM evaluator.
1038 MR-2 (WARNING): meta-loop should reference a captured baseline in an evaluator.
1040 Both rules are suppressed by ``meta_self_eval_ok: true`` at the loop top-level.
1041 """
1042 errors: list[ValidationError] = []
1043 if fsm.meta_self_eval_ok or not _is_meta_loop(fsm):
1044 return errors
1046 # Collect all evaluator types used across all states
1047 evaluator_types: set[str] = set()
1048 for state in fsm.states.values():
1049 if state.evaluate is not None:
1050 evaluator_types.add(state.evaluate.type)
1052 # MR-1: must have at least one non-LLM evaluator
1053 if not evaluator_types & NON_LLM_EVALUATOR_TYPES:
1054 errors.append(
1055 ValidationError(
1056 message=(
1057 "Loop modifies harness artifacts but has no non-LLM evaluator. "
1058 "LLM self-grades on harness updates are unreliable (SHOR Table 1: "
1059 "33-55% accuracy). Pair every check_semantic state with at least one "
1060 "of: exit_code, output_numeric, convergence, diff_stall, action_stall, mcp_result. "
1061 "Note: llm_structured and comparator both use the LLM and do not satisfy MR-1. "
1062 "To suppress with justification, set `meta_self_eval_ok: true` at the "
1063 "loop top-level."
1064 ),
1065 path="<root>",
1066 severity=ValidationSeverity.ERROR,
1067 )
1068 )
1070 # MR-2: should reference a captured baseline in a later evaluator
1071 capture_names: set[str] = {state.capture for state in fsm.states.values() if state.capture}
1072 if capture_names and not _has_baseline_reference(fsm, capture_names):
1073 errors.append(
1074 ValidationError(
1075 message=(
1076 "Meta-loop appears to lack a measure→propose→apply→re-measure "
1077 "spine: no captured baseline value is referenced by a later evaluator. "
1078 "Meta-loops should compare a post-change score against a pre-change "
1079 "baseline (see loops/harness-optimize.yaml as reference template). "
1080 "To suppress, set `meta_self_eval_ok: true`."
1081 ),
1082 path="<root>",
1083 severity=ValidationSeverity.WARNING,
1084 )
1085 )
1087 return errors
1090# Regex patterns for detecting counter-increment actions.
1091# Must contain a printf/echo writing to a file AND an arithmetic increment.
1092_COUNTER_FILE_WRITE_RE = re.compile(r"(?:printf|echo)\s+.*>")
1093_COUNTER_INCREMENT_RE = re.compile(
1094 r"\$\(\(.*\+\s*1\s*\)\)" # $((N + 1)) or $((N+1))
1095 r"|\+\+" # C-style increment
1096 r"|\+=1" # compound assignment
1097 r"|awk\s+.*\+\+" # awk with increment
1098)
1101def _validate_zero_retry_counter(fsm: FSMLoop) -> list[ValidationError]:
1102 """Detect counter + output_numeric combos that yield zero effective retries.
1104 A common loop-authoring footgun: a state increments a counter file and then
1105 evaluates ``output_numeric`` with ``operator: lt, target: 1`` against it.
1106 After the first increment the counter is 1, ``1 < 1 == false``, so the
1107 retry budget is 0 by construction. Author almost always intended target=2.
1108 """
1109 errors: list[ValidationError] = []
1111 for state_name, state in fsm.states.items():
1112 if not state.action or not state.evaluate:
1113 continue
1115 ev = state.evaluate
1116 if ev.type != "output_numeric":
1117 continue
1118 if ev.operator is None or ev.target is None:
1119 continue
1121 # Must be a number-like target for numeric comparison
1122 try:
1123 target = float(ev.target)
1124 except (ValueError, TypeError):
1125 continue
1127 if not _is_counter_action(state.action):
1128 continue
1130 # Check: after first increment (0→1), does operator(1, target) already fail?
1131 op_fn = _NUMERIC_OPERATORS.get(ev.operator)
1132 if op_fn is None:
1133 continue
1135 if not op_fn(1.0, target):
1136 suggested_target = _suggested_target(ev.operator, target)
1137 errors.append(
1138 ValidationError(
1139 message=(
1140 f"Zero retry budget: operator={ev.operator} target={target} "
1141 f"means the first post-increment value (1) already fails "
1142 f"({ev.operator}(1, {target}) == False). "
1143 f"Did you mean target={suggested_target}?"
1144 ),
1145 path=f"states.{state_name}.evaluate",
1146 severity=ValidationSeverity.WARNING,
1147 )
1148 )
1150 return errors
1153def _is_counter_action(action: str) -> bool:
1154 """Return True if the action string contains a counter-increment pattern."""
1155 return bool(_COUNTER_FILE_WRITE_RE.search(action) and _COUNTER_INCREMENT_RE.search(action))
1158def _suggested_target(operator: str, target: float) -> str:
1159 """Suggest a target value that allows at least one retry."""
1160 # For lt/le with a too-low target, suggest target+1 so first post-increment passes
1161 if operator in ("lt", "le"):
1162 return str(int(target) + 1)
1163 # For eq with target=0, suggest 1 so the counter can eventually match
1164 if operator == "eq" and target == 0:
1165 return "1"
1166 # For other cases, suggest target+1 as a default nudge
1167 return str(int(target) + 1)
1170def _validate_harness_multimodal_evaluator_blind_spot(fsm: FSMLoop) -> list[ValidationError]:
1171 """Warn when harness loops use LLM multimodal eval as sole gate to terminal.
1173 LLMs can silently fall back to text-only analysis when reading images,
1174 producing verdicts based on incomplete information. The output_contains
1175 evaluator can verify the LLM wrote the pass string but not that it
1176 actually processed the image. This is the same class of failure as MR-1
1177 (LLM self-evaluation bias) applied to artifact evaluation rather than
1178 harness modification.
1180 Suppressed by ``meta_self_eval_ok: true`` at the loop top-level.
1181 """
1182 errors: list[ValidationError] = []
1183 if fsm.meta_self_eval_ok or fsm.category != "harness":
1184 return errors
1186 terminal_states = fsm.get_terminal_states()
1188 for state_name, state in fsm.states.items():
1189 if state.action_type != "prompt" or not state.action:
1190 continue
1191 if state.evaluate is None or state.evaluate.type != "output_contains":
1192 continue
1193 if not any(p.search(state.action) for p in _MULTIMODAL_EVAL_PATTERNS):
1194 continue
1195 if state.on_yes not in terminal_states:
1196 continue
1197 errors.append(
1198 ValidationError(
1199 message=(
1200 f"State '{state_name}' evaluates a screenshot/image via LLM "
1201 "prompt and routes directly to a terminal on success. The "
1202 "output_contains evaluator can verify the LLM wrote the pass "
1203 "string but not that the LLM actually processed the image. "
1204 "Consider adding a shell-action verification state (e.g., "
1205 "functional smoke test) between scoring and the terminal."
1206 ),
1207 path=f"states.{state_name}",
1208 severity=ValidationSeverity.WARNING,
1209 )
1210 )
1212 return errors
1215def _find_shared_tmp_writes(fsm: FSMLoop) -> list[tuple[str, str]]:
1216 """Return (state_name, matched_path) for every action referencing shared .loops/tmp/.
1218 Scans `state.action` only. Prompts and sub-loop bindings can also reference
1219 paths, but those are out of static-scan reach: action strings are the
1220 place where loop YAMLs directly encode artifact paths.
1221 """
1222 findings: list[tuple[str, str]] = []
1223 for state_name, state in fsm.states.items():
1224 if not state.action:
1225 continue
1226 for match in _SHARED_TMP_PATH_RE.finditer(state.action):
1227 findings.append((state_name, match.group(0)))
1228 return findings
1231def _validate_input_key_without_guard(fsm: FSMLoop) -> list[ValidationError]:
1232 """Warn when a loop sets a custom input_key but omits required_inputs.
1234 A loop that accepts a runtime input via a named key (e.g. input_key: description)
1235 but doesn't declare required_inputs will silently proceed with an empty value if the
1236 user forgets to pass one. Declaring required_inputs shifts that failure to start-time.
1237 """
1238 if fsm.input_key == "input":
1239 return []
1240 if fsm.required_inputs:
1241 return []
1242 return [
1243 ValidationError(
1244 message=(
1245 f"Loop sets input_key: '{fsm.input_key}' but does not declare "
1246 f"required_inputs. If this input is mandatory, add "
1247 f"'required_inputs: [\"{fsm.input_key}\"]' to make the runner "
1248 f"abort when no input is provided."
1249 ),
1250 path="required_inputs",
1251 severity=ValidationSeverity.WARNING,
1252 )
1253 ]
1256def _validate_artifact_isolation(fsm: FSMLoop) -> list[ValidationError]:
1257 """Validate rule MR-3: loops must isolate artifacts to ${context.run_dir}.
1259 The runner injects ${context.run_dir} pointing at .loops/runs/<loop>-<ts>/
1260 and creates the folder before execution. Loops that write intermediate
1261 state (queues, checkpoints, generated files) to shared .loops/tmp/ paths
1262 will corrupt each other under concurrent runs (ll-parallel workers, retries,
1263 repeated invocations).
1265 Suppressed by `shared_state_ok: true` at the loop top-level for loops that
1266 intentionally share state across runs.
1267 """
1268 if fsm.shared_state_ok:
1269 return []
1270 errors: list[ValidationError] = []
1271 for state_name, path in _find_shared_tmp_writes(fsm):
1272 errors.append(
1273 ValidationError(
1274 message=(
1275 f"State writes to shared '{path}' instead of "
1276 "'${context.run_dir}/...'. Concurrent runs of this loop "
1277 "(e.g., under ll-parallel) will corrupt each other's state. "
1278 "Use the runner-injected `${context.run_dir}` for per-run "
1279 "artifact paths, or set `shared_state_ok: true` at the loop "
1280 "top-level if cross-run sharing is intentional."
1281 ),
1282 path=f"states.{state_name}.action",
1283 severity=ValidationSeverity.WARNING,
1284 )
1285 )
1286 return errors
1289def _is_llm_judged(state: StateConfig) -> bool:
1290 """Return True if this state will be graded by the default LLM judge.
1292 Mirrors the action-mode detection in executor._action_mode() (not imported
1293 here because that is runtime code). A state is LLM-judged when:
1294 - it has no explicit evaluate block AND its action is a prompt or slash_command, OR
1295 - it has an explicit evaluate block of type llm_structured or check_semantic.
1296 """
1297 if state.evaluate is None:
1298 # Heuristic: explicit action_type wins; fall back to leading "/" on action string.
1299 action_type = state.action_type
1300 if action_type in ("prompt", "slash_command"):
1301 return True
1302 if action_type is None and state.action and state.action.lstrip().startswith("/"):
1303 return True
1304 return False
1305 return state.evaluate.type in ("llm_structured", "check_semantic")
1308def _validate_partial_route_dead_end(fsm: FSMLoop) -> list[ValidationError]:
1309 """Validate rule MR-4: LLM-judged states with only on_yes have a partial/no dead-end.
1311 A state gated by the default LLM judge can receive yes/no/partial verdicts.
1312 If only on_yes is mapped (and no on_no, on_partial, next, or route table with
1313 a default exist), a partial or no verdict returns None from _route and silently
1314 terminates the loop — the parent treats this as failed.
1316 Suppressed by `partial_route_ok: true` at the loop top-level for the rare
1317 case where dead-ending on a non-yes verdict is intentional.
1318 """
1319 if fsm.partial_route_ok:
1320 return []
1321 errors: list[ValidationError] = []
1322 for state_name, state in fsm.states.items():
1323 if not _is_llm_judged(state):
1324 continue
1325 # States with an unconditional next: or a full route: table are safe.
1326 if state.next is not None or state.route is not None:
1327 continue
1328 # Only flag when on_yes is set but at least one of on_no/on_partial is missing.
1329 if state.on_yes is None:
1330 continue
1331 missing = [v for v in ("no", "partial") if getattr(state, f"on_{v}") is None]
1332 if not missing:
1333 continue
1334 unrouted = " or ".join(f"`{v}`" for v in missing)
1335 errors.append(
1336 ValidationError(
1337 message=(
1338 f"[state: {state_name}] LLM-judged prompt routes only on_yes; "
1339 f"a {unrouted} verdict has no route and will dead-end the loop "
1340 "(parent reads this as failed). Add on_no/on_partial, use `next:` "
1341 "for an unconditional handoff, or a `route:` table with a default. "
1342 "Set `partial_route_ok: true` at the loop top-level to suppress "
1343 "if intentional. (ENH-1917)"
1344 ),
1345 path=f"states.{state_name}",
1346 severity=ValidationSeverity.WARNING,
1347 )
1348 )
1349 return errors
1352def _has_baseline_reference(fsm: FSMLoop, capture_names: set[str]) -> bool:
1353 """Return True if any evaluate block references a captured variable."""
1354 for state in fsm.states.values():
1355 ev = state.evaluate
1356 if ev is None:
1357 continue
1358 # Check string fields that may interpolate captured values
1359 candidates = [ev.previous, ev.source]
1360 if isinstance(ev.target, str):
1361 candidates.append(ev.target)
1362 for field_val in candidates:
1363 if not field_val:
1364 continue
1365 for name in capture_names:
1366 if f"captured.{name}" in field_val:
1367 return True
1368 return False
1371def _validate_on_max_iterations(fsm: FSMLoop, defined_states: set[str]) -> list[ValidationError]:
1372 """Validate the top-level `on_max_iterations` field (ENH-1631).
1374 Checks that the named state exists when `on_max_iterations` is set.
1375 """
1376 errors: list[ValidationError] = []
1377 if fsm.on_max_iterations is None:
1378 return errors
1379 if fsm.on_max_iterations not in defined_states:
1380 errors.append(
1381 ValidationError(
1382 message=(
1383 f"on_max_iterations references unknown state "
1384 f"'{fsm.on_max_iterations}' (must be a declared state)"
1385 ),
1386 path="on_max_iterations",
1387 )
1388 )
1389 return errors
1392def _validate_circuit(fsm: FSMLoop, defined_states: set[str]) -> list[ValidationError]:
1393 """Validate the top-level `circuit:` block (FEAT-1637).
1395 Checks:
1396 - `circuit.repeated_failure.window` is a positive integer.
1397 - `circuit.repeated_failure.on_repeated_failure` is either the special
1398 token ``"abort"`` or the name of a declared state.
1399 """
1400 errors: list[ValidationError] = []
1401 if fsm.circuit is None or fsm.circuit.repeated_failure is None:
1402 return errors
1404 rf = fsm.circuit.repeated_failure
1405 if rf.window < 1:
1406 errors.append(
1407 ValidationError(
1408 message=f"circuit.repeated_failure.window must be >= 1, got {rf.window}",
1409 path="circuit.repeated_failure.window",
1410 )
1411 )
1413 target = rf.on_repeated_failure
1414 if target not in STALL_SPECIAL_TOKENS and target not in defined_states:
1415 errors.append(
1416 ValidationError(
1417 message=(
1418 f"circuit.repeated_failure.on_repeated_failure references "
1419 f"unknown state '{target}' (must be a declared state or "
1420 f'the literal "abort")'
1421 ),
1422 path="circuit.repeated_failure.on_repeated_failure",
1423 )
1424 )
1426 return errors
1429# Matches common interpolation prefixes used in loop YAML paths so we can
1430# extract the portable relative component for action-string scanning.
1431_INTERPOLATION_PREFIX_RE = re.compile(r"^\$\{[^}]+\}/")
1434def _strip_interpolation_prefix(path: str) -> str:
1435 """Return the path with any leading ${...}/ prefix removed."""
1436 return _INTERPOLATION_PREFIX_RE.sub("", path)
1439def _validate_progress_paths_isolation(fsm: FSMLoop) -> list[ValidationError]:
1440 """Warn when a state's action writes to a file listed in progress_paths (BUG-1767).
1442 When a loop's own bookkeeping files appear in both progress_paths and the
1443 state action strings, every append to those files resets the stall window,
1444 silently disabling the BUG-1674 stall guard for that loop. Authors should
1445 move such files to exclude_paths so the stall detector can still fire.
1446 """
1447 if fsm.circuit is None or fsm.circuit.repeated_failure is None:
1448 return []
1449 rf = fsm.circuit.repeated_failure
1450 if not rf.progress_paths:
1451 return []
1453 # Build a set of the relative path components we need to look for.
1454 watched = {_strip_interpolation_prefix(p) for p in rf.progress_paths}
1455 # Exclude paths that are already in exclude_paths — author acknowledged.
1456 excluded = {_strip_interpolation_prefix(p) for p in rf.exclude_paths}
1457 active_watched = watched - excluded
1458 if not active_watched:
1459 return []
1461 errors: list[ValidationError] = []
1462 for state_name, state in fsm.states.items():
1463 if not state.action:
1464 continue
1465 for path_fragment in active_watched:
1466 if path_fragment in state.action:
1467 errors.append(
1468 ValidationError(
1469 message=(
1470 f"State action references '{path_fragment}', which is also "
1471 "listed in circuit.repeated_failure.progress_paths. Writes "
1472 "to this file will reset the stall window every cycle, "
1473 "silently disabling stall detection. Move it to "
1474 "circuit.repeated_failure.exclude_paths to separate "
1475 "bookkeeping files from real progress signals."
1476 ),
1477 path=f"states.{state_name}.action",
1478 severity=ValidationSeverity.WARNING,
1479 )
1480 )
1481 return errors
1484def _find_reachable_states(fsm: FSMLoop) -> set[str]:
1485 """Find all states reachable from the initial state.
1487 Uses breadth-first search to find all reachable states. Seeds the BFS
1488 with the initial state plus top-level transition targets that act as
1489 alternate entry points: ``on_max_iterations`` (fires when the iteration
1490 cap is hit) and ``circuit.repeated_failure.on_repeated_failure`` (fires
1491 when the circuit breaker trips). These are real edges the runtime can
1492 take, so states reached only through them are not orphans.
1494 Args:
1495 fsm: The FSM loop to analyze
1497 Returns:
1498 Set of reachable state names
1499 """
1500 reachable: set[str] = set()
1501 to_visit: deque[str] = deque([fsm.initial])
1502 if fsm.on_max_iterations is not None:
1503 to_visit.append(fsm.on_max_iterations)
1504 if fsm.circuit is not None and fsm.circuit.repeated_failure is not None:
1505 target = fsm.circuit.repeated_failure.on_repeated_failure
1506 if target not in STALL_SPECIAL_TOKENS:
1507 to_visit.append(target)
1509 while to_visit:
1510 current = to_visit.popleft()
1511 if current in reachable or current not in fsm.states:
1512 continue
1514 reachable.add(current)
1515 state = fsm.states[current]
1516 refs = state.get_referenced_states()
1518 for ref in refs:
1519 if ref != "$current" and ref not in reachable:
1520 to_visit.append(ref)
1522 return reachable
1525def is_runnable_loop(path: Path) -> bool:
1526 """Cheap check for whether a YAML file is a runnable FSM loop definition.
1528 Returns True iff the file parses as a YAML mapping with the required
1529 top-level keys ``name``, ``initial``, and either ``states`` or ``flow``
1530 (the shorthand resolved by :func:`resolve_flow`). This matches the
1531 required-fields gate in :func:`load_and_validate` (lines 885-894) so
1532 "counted by the verifier" stays in sync with "runnable by ll-loop validate".
1534 No fragment/inheritance resolution is performed — library fragments under
1535 ``loops/lib/`` that omit ``initial`` or ``states`` correctly return False.
1536 """
1537 try:
1538 data = yaml.safe_load(path.read_text())
1539 except (OSError, yaml.YAMLError):
1540 return False
1541 if not isinstance(data, dict):
1542 return False
1543 has_flow = "states" in data or "flow" in data
1544 return "name" in data and "initial" in data and has_flow
1547def load_and_validate(path: Path) -> tuple[FSMLoop, list[ValidationError]]:
1548 """Load YAML file and validate FSM structure.
1550 Args:
1551 path: Path to the YAML file to load
1553 Returns:
1554 Tuple of (validated FSMLoop instance, list of WARNING-severity ValidationErrors)
1556 Raises:
1557 FileNotFoundError: If the file doesn't exist
1558 yaml.YAMLError: If the file is not valid YAML
1559 ValueError: If validation fails (contains error details)
1560 """
1561 if not path.exists():
1562 raise FileNotFoundError(f"FSM file not found: {path}")
1564 with open(path) as f:
1565 data: dict[str, Any] = yaml.safe_load(f)
1567 if not isinstance(data, dict):
1568 raise ValueError(f"FSM file must contain a YAML mapping, got {type(data)}")
1570 # Resolve `from:` inheritance before any further checks, so a child loop
1571 # can omit fields its parent provides (including `initial`/`states`) and
1572 # so a parent's `import:`/`fragments:` blocks survive into the merged
1573 # result for the subsequent `resolve_fragments` pass.
1574 data = resolve_inheritance(data, path.parent)
1576 # Expand flow: linear shorthand into states: before required-fields check
1577 data = resolve_flow(data)
1579 # Check required fields before parsing
1580 missing = []
1581 for field in ["name", "initial"]:
1582 if field not in data:
1583 missing.append(field)
1584 if "states" not in data:
1585 missing.append("states (or flow)")
1587 if missing:
1588 raise ValueError(f"FSM file missing required fields: {', '.join(missing)}")
1590 # Check for unknown top-level keys before parsing
1591 unknown_key_warnings: list[ValidationError] = []
1592 unknown = set(data.keys()) - KNOWN_TOP_LEVEL_KEYS
1593 if unknown:
1594 unknown_key_warnings.append(
1595 ValidationError(
1596 path="<root>",
1597 message=f"Unknown top-level keys: {', '.join(sorted(unknown))}",
1598 severity=ValidationSeverity.WARNING,
1599 )
1600 )
1602 # Resolve fragment libraries before parsing into dataclass
1603 data = resolve_fragments(data, path.parent)
1605 # Parse into dataclass
1606 fsm = FSMLoop.from_dict(data)
1608 # Validate
1609 errors = validate_fsm(fsm)
1611 # Validate with: bindings against child loop parameters (requires file-system access)
1612 errors.extend(_validate_with_bindings(fsm, path.parent))
1613 errors.extend(_validate_fragment_bindings(fsm, path.parent))
1615 # Filter to errors only (not warnings) for raising
1616 error_list = [e for e in errors if e.severity == ValidationSeverity.ERROR]
1618 if error_list:
1619 error_messages = "\n ".join(str(e) for e in error_list)
1620 raise ValueError(f"FSM validation failed:\n {error_messages}")
1622 # Collect all warnings (unknown-key warnings + structural warnings)
1623 struct_warnings = [e for e in errors if e.severity == ValidationSeverity.WARNING]
1624 all_warnings = unknown_key_warnings + struct_warnings
1625 for warning in all_warnings:
1626 logger.warning(str(warning))
1628 return fsm, all_warnings