Coverage for little_loops / fsm / validation.py: 46%
670 statements
« prev ^ index » next coverage.py v7.12.0, created at 2026-06-08 15:34 -0500
1"""FSM loop validation logic.
3This module provides validation for FSM loop definitions, ensuring
4structural correctness and catching common configuration errors.
6Validation checks:
7- Initial state exists in states dict
8- All referenced states exist
9- At least one terminal state
10- Evaluator configs have required fields for their type
11- No conflicting routing (shorthand vs full route)
12- Numeric fields in valid ranges (max_iterations > 0, backoff >= 0, timeout > 0)
13"""
15from __future__ import annotations
17import logging
18import re
19from collections import deque
20from dataclasses import dataclass
21from enum import Enum
22from pathlib import Path
23from typing import Any
25import yaml
27from little_loops.fsm.evaluators import _NUMERIC_OPERATORS
28from little_loops.fsm.fragments import resolve_flow, resolve_fragments, resolve_inheritance
29from little_loops.fsm.schema import EvaluateConfig, FSMLoop, ParameterSpec, StateConfig
31logger = logging.getLogger(__name__)
34class ValidationSeverity(Enum):
35 """Severity level for validation issues."""
37 ERROR = "error"
38 WARNING = "warning"
41@dataclass
42class ValidationError:
43 """Structured validation error.
45 Attributes:
46 message: Human-readable error description
47 path: Path to the problematic element (e.g., "states.check.route")
48 severity: Error severity (error or warning)
49 """
51 message: str
52 path: str | None = None
53 severity: ValidationSeverity = ValidationSeverity.ERROR
55 def __str__(self) -> str:
56 """Format error for display."""
57 prefix = f"[{self.severity.value.upper()}]"
58 if self.path:
59 return f"{prefix} {self.path}: {self.message}"
60 return f"{prefix} {self.message}"
63# Evaluator type to required fields mapping
64EVALUATOR_REQUIRED_FIELDS: dict[str, list[str]] = {
65 "exit_code": [],
66 "output_numeric": ["operator", "target"],
67 "output_json": ["path", "operator", "target"],
68 "output_contains": ["pattern"],
69 "convergence": ["target"],
70 "diff_stall": [],
71 "action_stall": [],
72 "llm_structured": [],
73 "mcp_result": [],
74 "harbor_scorer": [],
75 "comparator": ["baseline_path"],
76 "contract": ["pairs"],
77}
79# Non-LLM evaluator types: all evaluator types except llm_structured
80# Derived from EVALUATOR_REQUIRED_FIELDS so new types are automatically included
81NON_LLM_EVALUATOR_TYPES: frozenset[str] = frozenset(EVALUATOR_REQUIRED_FIELDS.keys()) - {
82 "llm_structured",
83 "comparator",
84 "contract",
85}
87# Meta-loop detector: action string patterns that indicate harness artifact writes
88_META_LOOP_ACTION_PATTERNS: tuple[re.Pattern[str], ...] = (
89 re.compile(r"loops/[\w-]+\.yaml"),
90 re.compile(r"skills/[\w-]+/SKILL\.md"),
91 re.compile(r"agents/[\w-]+\.md"),
92 re.compile(r"commands/[\w-]+\.md"),
93 re.compile(r"\.claude/(CLAUDE\.md|settings)"),
94)
96# Action string tokens that indicate meta-loop behavior
97_META_LOOP_ACTION_TOKENS: frozenset[str] = frozenset({"yaml_state_editor", "replace_action"})
99# Import paths that identify a loop as a meta-loop (harness optimization framework)
100_META_LOOP_IMPORT_TRIGGERS: frozenset[str] = frozenset({"lib/benchmark.yaml"})
102# MR-3: shared-tmp path detector. The runner injects ${context.run_dir} resolving
103# to .loops/runs/<loop>-<timestamp>/; loops that hardcode .loops/tmp/ instead
104# cause state corruption under concurrent runs (ll-parallel, retries, etc.).
105_SHARED_TMP_PATH_RE = re.compile(r"\.loops/tmp/[\w./-]+")
107# ENH-1961: Regex for extracting captured variable names from ${captured.<var>.*} references
108_CAPTURED_REF_RE = re.compile(r"\$\{captured\.(\w+)")
110# ENH-1819: Regex patterns for detecting multimodal evaluation in prompt actions
111_MULTIMODAL_EVAL_PATTERNS: tuple[re.Pattern[str], ...] = (
112 re.compile(r"Read the screenshot", re.IGNORECASE),
113 re.compile(r"view the (generated )?(website|page|image)", re.IGNORECASE),
114 re.compile(r"screenshot\.(png|jpg|jpeg|webp)"),
115 re.compile(r"\.(png|jpg|jpeg|webp)\b.*\b(read|view|evaluate|score|judge)", re.IGNORECASE),
116)
118# Valid comparison operators
119VALID_OPERATORS = {"eq", "ne", "lt", "le", "gt", "ge"}
121# All top-level keys recognized by FSMLoop.from_dict()
122KNOWN_TOP_LEVEL_KEYS: frozenset[str] = frozenset(
123 {
124 "name",
125 "description",
126 "initial",
127 "states",
128 "context",
129 "parameters",
130 "scope",
131 "max_iterations",
132 "on_max_iterations",
133 "max_edge_revisits",
134 "backoff",
135 "timeout",
136 "default_timeout",
137 "maintain",
138 "llm",
139 "on_handoff",
140 "input_key",
141 "required_inputs",
142 "config",
143 "category",
144 "labels",
145 "commands",
146 "targets",
147 "circuit",
148 "meta_self_eval_ok",
149 "shared_state_ok",
150 "partial_route_ok",
151 "artifact_versioning",
152 "artifact_versioning_ok",
153 "import",
154 "fragments",
155 "from",
156 "flow",
157 "state_defs",
158 }
159)
161# Special tokens accepted as the `on_repeated_failure` target on the
162# stall detector — `"abort"` means terminate via _finish("stall_detected").
163STALL_SPECIAL_TOKENS: frozenset[str] = frozenset({"abort"})
165# Valid parameter types for the 'parameters:' block
166VALID_PARAMETER_TYPES: frozenset[str] = frozenset(
167 {"string", "integer", "number", "boolean", "enum", "path"}
168)
171def _validate_evaluator(state_name: str, evaluate: EvaluateConfig) -> list[ValidationError]:
172 """Validate evaluator configuration for type-specific requirements.
174 Args:
175 state_name: Name of the state containing this evaluator
176 evaluate: The evaluator configuration to validate
178 Returns:
179 List of validation errors found
180 """
181 errors: list[ValidationError] = []
182 path = f"states.{state_name}.evaluate"
184 # Check that evaluator type is recognized
185 valid_types = set(EVALUATOR_REQUIRED_FIELDS.keys())
186 if evaluate.type not in valid_types:
187 errors.append(
188 ValidationError(
189 message=f"Unknown evaluator type '{evaluate.type}'. "
190 f"Must be one of: {', '.join(sorted(valid_types))}",
191 path=path,
192 )
193 )
194 return errors # Can't check required fields for unknown type
196 # Check required fields for evaluator type
197 required = EVALUATOR_REQUIRED_FIELDS.get(evaluate.type, [])
198 for field_name in required:
199 value = getattr(evaluate, field_name, None)
200 if value is None:
201 errors.append(
202 ValidationError(
203 message=f"Evaluator type '{evaluate.type}' requires '{field_name}' field",
204 path=path,
205 )
206 )
208 # Validate operator if present
209 if evaluate.operator is not None and evaluate.operator not in VALID_OPERATORS:
210 errors.append(
211 ValidationError(
212 message=f"Invalid operator '{evaluate.operator}'. "
213 f"Must be one of: {', '.join(sorted(VALID_OPERATORS))}",
214 path=f"{path}.operator",
215 )
216 )
218 # Validate convergence-specific fields
219 if evaluate.type == "convergence":
220 if evaluate.direction not in ("minimize", "maximize"):
221 errors.append(
222 ValidationError(
223 message=f"Invalid direction '{evaluate.direction}'. "
224 "Must be 'minimize' or 'maximize'",
225 path=f"{path}.direction",
226 )
227 )
228 # Only validate tolerance if it's a numeric value (not an interpolation string)
229 if (
230 evaluate.tolerance is not None
231 and isinstance(evaluate.tolerance, (int, float))
232 and evaluate.tolerance < 0
233 ):
234 errors.append(
235 ValidationError(
236 message="Tolerance cannot be negative",
237 path=f"{path}.tolerance",
238 )
239 )
241 # Validate llm_structured-specific fields
242 if evaluate.type == "llm_structured":
243 if evaluate.min_confidence < 0 or evaluate.min_confidence > 1:
244 errors.append(
245 ValidationError(
246 message="min_confidence must be between 0 and 1",
247 path=f"{path}.min_confidence",
248 )
249 )
251 # Validate diff_stall-specific fields
252 if evaluate.type == "diff_stall":
253 if evaluate.max_stall < 1:
254 errors.append(
255 ValidationError(
256 message="max_stall must be >= 1",
257 path=f"{path}.max_stall",
258 )
259 )
261 # Validate action_stall-specific fields
262 if evaluate.type == "action_stall":
263 if evaluate.max_repeat < 1:
264 errors.append(
265 ValidationError(
266 message="max_repeat must be >= 1",
267 path=f"{path}.max_repeat",
268 )
269 )
271 return errors
274def _validate_parameters(fsm: FSMLoop) -> list[ValidationError]:
275 """Validate the loop's top-level parameters: block.
277 Args:
278 fsm: The FSM loop to validate
280 Returns:
281 List of validation errors found
282 """
283 errors: list[ValidationError] = []
285 for param_name, param_spec in fsm.parameters.items():
286 path = f"parameters.{param_name}"
288 if param_spec.type not in VALID_PARAMETER_TYPES:
289 errors.append(
290 ValidationError(
291 message=(
292 f"Unknown parameter type '{param_spec.type}'. "
293 f"Must be one of: {', '.join(sorted(VALID_PARAMETER_TYPES))}"
294 ),
295 path=path,
296 )
297 )
299 if param_spec.type == "enum" and not param_spec.values:
300 errors.append(
301 ValidationError(
302 message="Parameter type 'enum' requires a 'values' list",
303 path=path,
304 )
305 )
307 if param_spec.required and param_spec.default is not None:
308 errors.append(
309 ValidationError(
310 message="Parameter cannot be both 'required: true' and have a 'default' value",
311 path=path,
312 )
313 )
315 return errors
318def _check_param_type(value: Any, spec: ParameterSpec) -> str | None:
319 """Return an error message if value does not match spec.type, else None."""
320 if spec.type == "string" and not isinstance(value, str):
321 return f"expected string, got {type(value).__name__}"
322 if spec.type == "integer" and not isinstance(value, int):
323 return f"expected integer, got {type(value).__name__}"
324 if spec.type == "number" and not isinstance(value, (int, float)):
325 return f"expected number, got {type(value).__name__}"
326 if spec.type == "boolean" and not isinstance(value, bool):
327 return f"expected boolean, got {type(value).__name__}"
328 if spec.type == "enum" and spec.values and value not in spec.values:
329 return f"expected one of {spec.values!r}, got {value!r}"
330 return None
333def _validate_with_bindings(fsm: FSMLoop, loop_dir: Path) -> list[ValidationError]:
334 """Validate with: bindings against child loop parameter contracts.
336 Called from load_and_validate (not validate_fsm) because resolving child loops
337 requires file-system access via the loop directory path.
339 Args:
340 fsm: The parent FSM loop
341 loop_dir: Directory to resolve child loop paths from
343 Returns:
344 List of validation errors found
345 """
346 errors: list[ValidationError] = []
348 for state_name, state in fsm.states.items():
349 if state.loop is None or not state.with_:
350 continue
352 # Try to resolve and load the child loop; skip if unavailable
353 try:
354 from little_loops.cli.loop._helpers import resolve_loop_path
356 loop_path = resolve_loop_path(state.loop, loop_dir)
357 child_fsm, _ = load_and_validate(loop_path)
358 except Exception:
359 continue
361 if not child_fsm.parameters:
362 continue # Child has no declared contract — nothing to cross-validate
364 path = f"states.{state_name}"
366 # Unknown with: keys (not declared by child)
367 for key in state.with_:
368 if key not in child_fsm.parameters:
369 errors.append(
370 ValidationError(
371 message=(
372 f"'with.{key}' is not a declared parameter of loop '{state.loop}'. "
373 f"Declared: {', '.join(sorted(child_fsm.parameters))}"
374 ),
375 path=f"{path}.with.{key}",
376 )
377 )
379 # Required parameters not bound
380 for param_name, param_spec in child_fsm.parameters.items():
381 if param_spec.required and param_name not in state.with_:
382 errors.append(
383 ValidationError(
384 message=(
385 f"Required parameter '{param_name}' of loop '{state.loop}' "
386 f"is not bound in 'with'"
387 ),
388 path=f"{path}.with",
389 )
390 )
392 # Statically-detectable type mismatches (skip interpolation strings)
393 for param_name, value in state.with_.items():
394 if param_name not in child_fsm.parameters:
395 continue
396 if isinstance(value, str) and "${" in value:
397 continue
398 type_error = _check_param_type(value, child_fsm.parameters[param_name])
399 if type_error:
400 errors.append(
401 ValidationError(
402 message=f"Parameter '{param_name}': {type_error}",
403 path=f"{path}.with.{param_name}",
404 )
405 )
407 return errors
410def _validate_fragment_bindings(fsm: FSMLoop, loop_dir: Path) -> list[ValidationError]:
411 """Validate fragment with: bindings against fragment parameter contracts.
413 Called from load_and_validate (not validate_fsm) because fragment parameters
414 are populated by resolve_fragments which runs before dataclass parsing.
416 Args:
417 fsm: The FSM loop to validate
418 loop_dir: Directory containing the loop file (unused; kept for API symmetry with
419 _validate_with_bindings)
421 Returns:
422 List of validation errors found
423 """
424 # Runner-injected vars available at runtime but not at static analysis time
425 RUNNER_INJECTED = {"run_dir", "loop_name", "started_at", "input_hash"}
427 errors: list[ValidationError] = []
429 for state_name, state in fsm.states.items():
430 if not state.fragment_parameters:
431 continue # No declared contract — nothing to cross-validate
433 path = f"states.{state_name}"
435 # Unknown with: keys (not declared by fragment)
436 for key in state.fragment_bindings:
437 if key not in state.fragment_parameters:
438 errors.append(
439 ValidationError(
440 message=(
441 f"'with.{key}' is not a declared parameter of fragment "
442 f"'{state.fragment_name}'. "
443 f"Declared: {', '.join(sorted(state.fragment_parameters))}"
444 ),
445 path=f"{path}.with.{key}",
446 )
447 )
449 # Required parameters not bound (whitelist runner-injected vars)
450 for param_name, param_spec in state.fragment_parameters.items():
451 if param_spec.required and param_name not in state.fragment_bindings:
452 if param_name in RUNNER_INJECTED:
453 continue # Available at runtime; not a static error
454 errors.append(
455 ValidationError(
456 message=(
457 f"Required parameter '{param_name}' of fragment "
458 f"'{state.fragment_name}' is not bound in 'with'"
459 ),
460 path=f"{path}.with",
461 )
462 )
464 # Statically-detectable type mismatches (skip interpolation strings)
465 for param_name, value in state.fragment_bindings.items():
466 if param_name not in state.fragment_parameters:
467 continue
468 if isinstance(value, str) and "${" in value:
469 continue
470 type_error = _check_param_type(value, state.fragment_parameters[param_name])
471 if type_error:
472 errors.append(
473 ValidationError(
474 message=f"Parameter '{param_name}': {type_error}",
475 path=f"{path}.with.{param_name}",
476 )
477 )
479 return errors
482def _validate_state_action(state_name: str, state: StateConfig) -> list[ValidationError]:
483 """Validate state action configuration.
485 Args:
486 state_name: Name of the state to validate
487 state: The state configuration to validate
489 Returns:
490 List of validation errors found
491 """
492 errors: list[ValidationError] = []
493 path = f"states.{state_name}"
495 # append_to_messages must contain at least one ${...} interpolation expression
496 if state.append_to_messages is not None:
497 if "${" not in state.append_to_messages:
498 errors.append(
499 ValidationError(
500 message=(
501 "'append_to_messages' must contain a ${...} interpolation expression "
502 f"(e.g. '${{captured.{state_name}.output}}')"
503 ),
504 path=f"{path}.append_to_messages",
505 )
506 )
508 # params field is only valid for mcp_tool states
509 if state.params and state.action_type != "mcp_tool":
510 errors.append(
511 ValidationError(
512 message="'params' field is only valid when action_type is 'mcp_tool'",
513 path=f"{path}.params",
514 )
515 )
517 # loop and action are mutually exclusive
518 if state.loop is not None and state.action is not None:
519 errors.append(
520 ValidationError(
521 message="'loop' and 'action' are mutually exclusive — "
522 "a sub-loop state cannot also have an action",
523 path=f"{path}",
524 )
525 )
527 # with: requires loop: to be set
528 if state.with_ and state.loop is None:
529 errors.append(
530 ValidationError(
531 message="'with' is only valid when 'loop' is set",
532 path=f"{path}.with",
533 )
534 )
536 # FEAT-1283: type=learning requires a populated LearningConfig
537 if state.type == "learning" and state.learning is not None:
538 if not state.learning.targets and not state.learning.targets_csv:
539 errors.append(
540 ValidationError(
541 message="type=learning requires non-empty 'learning.targets' or 'learning.targets_csv'",
542 path=f"{path}.learning.targets",
543 )
544 )
545 if state.learning.max_retries < 0:
546 errors.append(
547 ValidationError(
548 message=(
549 f"learning.max_retries must be >= 0, got {state.learning.max_retries}"
550 ),
551 path=f"{path}.learning.max_retries",
552 )
553 )
554 if state.on_yes is None:
555 errors.append(
556 ValidationError(
557 message="type=learning requires 'on_yes' (target for all-proven)",
558 path=f"{path}.on_yes",
559 )
560 )
561 if state.on_blocked is None and state.on_no is None:
562 errors.append(
563 ValidationError(
564 message=(
565 "type=learning requires 'on_blocked' or 'on_no' "
566 "(target for refuted / retries_exhausted)"
567 ),
568 path=f"{path}",
569 )
570 )
572 # with: and context_passthrough are mutually exclusive
573 if state.with_ and state.context_passthrough:
574 errors.append(
575 ValidationError(
576 message=(
577 "'with' and 'context_passthrough' are mutually exclusive — "
578 "use 'with' for explicit parameter bindings or 'context_passthrough' "
579 "for legacy bulk passthrough, not both"
580 ),
581 path=f"{path}",
582 )
583 )
585 return errors
588def _validate_state_routing(state_name: str, state: StateConfig) -> list[ValidationError]:
589 """Validate state routing configuration.
591 Checks for conflicting routing definitions (shorthand vs full route).
593 Args:
594 state_name: Name of the state to validate
595 state: The state configuration to validate
597 Returns:
598 List of validation errors/warnings found
599 """
600 errors: list[ValidationError] = []
601 path = f"states.{state_name}"
603 has_shorthand = (
604 state.on_yes is not None
605 or state.on_no is not None
606 or state.on_error is not None
607 or state.on_partial is not None
608 or state.on_blocked is not None
609 or bool(state.extra_routes)
610 )
611 has_route = state.route is not None
613 # Warn about conflicting definitions
614 if has_shorthand and has_route:
615 errors.append(
616 ValidationError(
617 message="Both shorthand routing (on_yes/on_no/on_error) "
618 "and full route table defined. Route table will take precedence.",
619 path=path,
620 severity=ValidationSeverity.WARNING,
621 )
622 )
624 # Check for no valid transition definition
625 has_next = state.next is not None
626 has_terminal = state.terminal
627 has_loop = state.loop is not None
629 if not has_shorthand and not has_route and not has_next and not has_terminal and not has_loop:
630 errors.append(
631 ValidationError(
632 message="State has no transition defined. Add routing, 'next', "
633 "or mark as 'terminal: true'",
634 path=path,
635 )
636 )
638 # Validate retry field pairing: max_retries requires on_retry_exhausted and vice versa
639 if state.max_retries is not None and state.on_retry_exhausted is None:
640 errors.append(
641 ValidationError(
642 message="'max_retries' requires 'on_retry_exhausted' to also be set",
643 path=path,
644 )
645 )
646 if state.on_retry_exhausted is not None and state.max_retries is None:
647 errors.append(
648 ValidationError(
649 message="'on_retry_exhausted' requires 'max_retries' to also be set",
650 path=path,
651 )
652 )
653 if state.max_retries is not None and state.max_retries < 1:
654 errors.append(
655 ValidationError(
656 message=f"'max_retries' must be >= 1, got {state.max_retries}",
657 path=path,
658 )
659 )
661 # Validate retryable_exit_codes: requires on_error; all codes must be positive ints
662 if state.retryable_exit_codes is not None:
663 if state.on_error is None:
664 errors.append(
665 ValidationError(
666 message="'retryable_exit_codes' requires 'on_error' to also be set",
667 path=path,
668 )
669 )
670 for code in state.retryable_exit_codes:
671 if not isinstance(code, int) or code < 1:
672 errors.append(
673 ValidationError(
674 message=(
675 f"'retryable_exit_codes' entries must be positive "
676 f"integers, got {code!r}"
677 ),
678 path=f"{path}.retryable_exit_codes",
679 )
680 )
681 break
683 # Validate rate-limit retry field pairing (mirrors max_retries/on_retry_exhausted)
684 if state.max_rate_limit_retries is not None and state.on_rate_limit_exhausted is None:
685 errors.append(
686 ValidationError(
687 message="'max_rate_limit_retries' requires 'on_rate_limit_exhausted' to also be set",
688 path=path,
689 )
690 )
691 if state.on_rate_limit_exhausted is not None and state.max_rate_limit_retries is None:
692 errors.append(
693 ValidationError(
694 message="'on_rate_limit_exhausted' requires 'max_rate_limit_retries' to also be set",
695 path=path,
696 )
697 )
698 if state.max_rate_limit_retries is not None and state.max_rate_limit_retries < 1:
699 errors.append(
700 ValidationError(
701 message=f"'max_rate_limit_retries' must be >= 1, got {state.max_rate_limit_retries}",
702 path=path,
703 )
704 )
705 if (
706 state.rate_limit_backoff_base_seconds is not None
707 and state.rate_limit_backoff_base_seconds < 1
708 ):
709 errors.append(
710 ValidationError(
711 message=(
712 f"'rate_limit_backoff_base_seconds' must be >= 1, "
713 f"got {state.rate_limit_backoff_base_seconds}"
714 ),
715 path=path,
716 )
717 )
718 if state.rate_limit_max_wait_seconds is not None and state.rate_limit_max_wait_seconds < 1:
719 errors.append(
720 ValidationError(
721 message=(
722 f"'rate_limit_max_wait_seconds' must be >= 1, "
723 f"got {state.rate_limit_max_wait_seconds}"
724 ),
725 path=path,
726 )
727 )
728 if state.rate_limit_long_wait_ladder is not None:
729 if len(state.rate_limit_long_wait_ladder) == 0:
730 errors.append(
731 ValidationError(
732 message="'rate_limit_long_wait_ladder' must be non-empty if specified",
733 path=path,
734 )
735 )
736 else:
737 for idx, value in enumerate(state.rate_limit_long_wait_ladder):
738 if not isinstance(value, int) or value < 1:
739 errors.append(
740 ValidationError(
741 message=(
742 f"'rate_limit_long_wait_ladder[{idx}]' must be a "
743 f"positive integer, got {value!r}"
744 ),
745 path=path,
746 )
747 )
749 # Validate throttle config when present
750 if state.throttle is not None:
751 t = state.throttle
752 fields = {
753 "normal_max": t.normal_max,
754 "warn_max": t.warn_max,
755 "hard_max": t.hard_max,
756 }
757 for field_name, val in fields.items():
758 if val is not None and (not isinstance(val, int) or val < 1):
759 errors.append(
760 ValidationError(
761 message=f"'throttle.{field_name}' must be a positive integer, got {val!r}",
762 path=path,
763 )
764 )
765 # Enforce ordering when all three are set
766 if t.normal_max is not None and t.warn_max is not None and t.normal_max >= t.warn_max:
767 errors.append(
768 ValidationError(
769 message=(
770 f"'throttle.normal_max' ({t.normal_max}) must be less than "
771 f"'throttle.warn_max' ({t.warn_max})"
772 ),
773 path=path,
774 )
775 )
776 if t.warn_max is not None and t.hard_max is not None and t.warn_max >= t.hard_max:
777 errors.append(
778 ValidationError(
779 message=(
780 f"'throttle.warn_max' ({t.warn_max}) must be less than "
781 f"'throttle.hard_max' ({t.hard_max})"
782 ),
783 path=path,
784 )
785 )
787 return errors
790def _validate_targets(fsm: FSMLoop) -> list[ValidationError]:
791 """Validate top-level targets[] entries (ENH-1552).
793 Rejects any targets[].states[] entry whose sibling file: value does not
794 end with a .yaml extension.
795 """
796 errors: list[ValidationError] = []
797 for i, target in enumerate(fsm.targets):
798 if target.file is not None and not target.file.endswith(".yaml"):
799 errors.append(
800 ValidationError(
801 message=(f"targets[{i}].file must be a .yaml file, got '{target.file}'"),
802 path=f"targets[{i}].file",
803 )
804 )
805 return errors
808def _validate_failure_terminal_action(fsm: FSMLoop) -> list[ValidationError]:
809 """Warn when a failure-named terminal state has no diagnostic predecessor.
811 Failure terminals (failed, error, aborted) should have at least one
812 predecessor state with an action or sub-loop that provides diagnostic
813 output before termination. Otherwise the failure is silent — the
814 executor calls _finish("terminal") before any action on the terminal
815 itself can execute.
817 Severity is WARNING (not ERROR) so that existing loops with bare
818 failure terminals continue to load, and test_terminal_only_state_valid
819 (which filters by ERROR) passes without modification.
820 """
821 FAILURE_TERMINAL_NAMES: frozenset[str] = frozenset({"failed", "error", "aborted"})
822 errors: list[ValidationError] = []
824 terminal_states = fsm.get_terminal_states()
825 failure_terminals = terminal_states & FAILURE_TERMINAL_NAMES
827 for ft_name in failure_terminals:
828 has_diagnostic_predecessor = False
829 for state_name, state in fsm.states.items():
830 if state_name == ft_name:
831 continue
832 if ft_name in state.get_referenced_states():
833 if state.action is not None or state.loop is not None:
834 has_diagnostic_predecessor = True
835 break
837 if not has_diagnostic_predecessor:
838 errors.append(
839 ValidationError(
840 message=(
841 f"Failure-named terminal state '{ft_name}' has no predecessor "
842 "state with a diagnostic action. Add a non-terminal diagnostic "
843 "state (e.g. 'diagnose') with an action or sub-loop that routes "
844 f"to '{ft_name}'."
845 ),
846 path=f"states.{ft_name}",
847 severity=ValidationSeverity.WARNING,
848 )
849 )
851 return errors
854def validate_fsm(fsm: FSMLoop) -> list[ValidationError]:
855 """Validate FSM structure and return list of errors.
857 Performs comprehensive validation:
858 - Initial state exists
859 - All referenced states exist
860 - At least one terminal state
861 - Evaluator configurations are valid
862 - Routing configurations are valid
863 - Numeric fields are in valid ranges (max_iterations > 0, backoff >= 0, timeout > 0)
865 Args:
866 fsm: The FSM loop to validate
868 Returns:
869 List of validation errors (empty if valid)
870 """
871 errors: list[ValidationError] = []
872 defined_states = fsm.get_all_state_names()
874 # Warn when no top-level description: field is set. The field is optional
875 # for FSM execution but required for goal-alignment skills (debug-loop-run,
876 # audit-loop-run) and for ll-loop show --json to surface intent text.
877 if not fsm.description:
878 errors.append(
879 ValidationError(
880 path="<root>",
881 message=("No 'description' field defined. Add a top-level description: key."),
882 severity=ValidationSeverity.WARNING,
883 )
884 )
886 # Validate parameters block
887 errors.extend(_validate_parameters(fsm))
889 # Validate targets block (ENH-1552)
890 errors.extend(_validate_targets(fsm))
892 # Check initial state exists
893 if fsm.initial not in defined_states:
894 errors.append(
895 ValidationError(
896 message=f"Initial state '{fsm.initial}' not found in states",
897 path="initial",
898 )
899 )
901 # Check at least one terminal state
902 terminal_states = fsm.get_terminal_states()
903 if not terminal_states:
904 errors.append(
905 ValidationError(
906 message="No terminal state defined. At least one state must have 'terminal: true'",
907 path="states",
908 )
909 )
911 # Validate each state
912 for state_name, state in fsm.states.items():
913 # Check all referenced states exist
914 refs = state.get_referenced_states()
915 for ref in refs:
916 # $current is a special token for retry
917 if ref != "$current" and ref not in defined_states:
918 errors.append(
919 ValidationError(
920 message=f"References unknown state '{ref}'",
921 path=f"states.{state_name}",
922 )
923 )
925 # Validate action configuration
926 errors.extend(_validate_state_action(state_name, state))
928 # Validate evaluator if present
929 if state.evaluate is not None:
930 errors.extend(_validate_evaluator(state_name, state.evaluate))
932 # Validate routing configuration
933 errors.extend(_validate_state_routing(state_name, state))
935 # Check numeric field ranges
936 if fsm.max_iterations <= 0:
937 errors.append(
938 ValidationError(
939 message=f"max_iterations must be > 0, got {fsm.max_iterations}",
940 path="max_iterations",
941 )
942 )
943 if fsm.max_edge_revisits <= 0:
944 errors.append(
945 ValidationError(
946 message=f"max_edge_revisits must be > 0, got {fsm.max_edge_revisits}",
947 path="max_edge_revisits",
948 )
949 )
950 if fsm.backoff is not None and fsm.backoff < 0:
951 errors.append(
952 ValidationError(
953 message=f"backoff must be >= 0, got {fsm.backoff}",
954 path="backoff",
955 )
956 )
957 if fsm.timeout is not None and fsm.timeout <= 0:
958 errors.append(
959 ValidationError(
960 message=f"timeout must be > 0, got {fsm.timeout}",
961 path="timeout",
962 )
963 )
964 if fsm.llm.max_tokens <= 0:
965 errors.append(
966 ValidationError(
967 message=f"llm.max_tokens must be > 0, got {fsm.llm.max_tokens}",
968 path="llm.max_tokens",
969 )
970 )
971 if fsm.llm.timeout <= 0:
972 errors.append(
973 ValidationError(
974 message=f"llm.timeout must be > 0, got {fsm.llm.timeout}",
975 path="llm.timeout",
976 )
977 )
979 # Check for unreachable states (warning only)
980 reachable = _find_reachable_states(fsm)
981 unreachable = defined_states - reachable
982 for state_name in unreachable:
983 errors.append(
984 ValidationError(
985 message="State is not reachable from initial state",
986 path=f"states.{state_name}",
987 severity=ValidationSeverity.WARNING,
988 )
989 )
991 errors.extend(_validate_failure_terminal_action(fsm))
993 errors.extend(_validate_meta_loop_evaluation(fsm))
995 errors.extend(_validate_input_key_without_guard(fsm))
997 errors.extend(_validate_artifact_isolation(fsm))
999 errors.extend(_validate_harness_multimodal_evaluator_blind_spot(fsm))
1001 errors.extend(_validate_partial_route_dead_end(fsm))
1003 errors.extend(_validate_artifact_overwrite(fsm))
1005 errors.extend(_validate_zero_retry_counter(fsm))
1007 errors.extend(_validate_on_max_iterations(fsm, defined_states))
1009 errors.extend(_validate_circuit(fsm, defined_states))
1011 errors.extend(_validate_progress_paths_isolation(fsm))
1013 errors.extend(_validate_capture_reachability(fsm))
1015 return errors
1018def _is_meta_loop(fsm: FSMLoop) -> bool:
1019 """Return True if fsm is classified as a meta-loop.
1021 A loop is meta if ANY of the following match:
1022 1. Any state action string matches a harness-artifact path regex
1023 (writes another loop YAML, skill, agent, command, or project config)
1024 2. The loop's import list contains lib/benchmark.yaml
1025 3. Any state action references yaml_state_editor or replace_action
1026 """
1027 # Condition 2: imports lib/benchmark.yaml
1028 if any(imp in _META_LOOP_IMPORT_TRIGGERS for imp in fsm.imports):
1029 return True
1030 # Conditions 1 and 3: scan action strings
1031 for state in fsm.states.values():
1032 if state.action is None:
1033 continue
1034 for pattern in _META_LOOP_ACTION_PATTERNS:
1035 if pattern.search(state.action):
1036 return True
1037 for token in _META_LOOP_ACTION_TOKENS:
1038 if token in state.action:
1039 return True
1040 return False
1043def _validate_meta_loop_evaluation(fsm: FSMLoop) -> list[ValidationError]:
1044 """Validate meta-loop evaluation rules MR-1 and MR-2.
1046 MR-1 (ERROR): meta-loop must have at least one non-LLM evaluator.
1047 MR-2 (WARNING): meta-loop should reference a captured baseline in an evaluator.
1049 Both rules are suppressed by ``meta_self_eval_ok: true`` at the loop top-level.
1050 """
1051 errors: list[ValidationError] = []
1052 if fsm.meta_self_eval_ok or not _is_meta_loop(fsm):
1053 return errors
1055 # Collect all evaluator types used across all states
1056 evaluator_types: set[str] = set()
1057 for state in fsm.states.values():
1058 if state.evaluate is not None:
1059 evaluator_types.add(state.evaluate.type)
1061 # MR-1: must have at least one non-LLM evaluator
1062 if not evaluator_types & NON_LLM_EVALUATOR_TYPES:
1063 errors.append(
1064 ValidationError(
1065 message=(
1066 "Loop modifies harness artifacts but has no non-LLM evaluator. "
1067 "LLM self-grades on harness updates are unreliable (SHOR Table 1: "
1068 "33-55% accuracy). Pair every check_semantic state with at least one "
1069 "of: exit_code, output_numeric, convergence, diff_stall, action_stall, mcp_result. "
1070 "Note: llm_structured and comparator both use the LLM and do not satisfy MR-1. "
1071 "To suppress with justification, set `meta_self_eval_ok: true` at the "
1072 "loop top-level."
1073 ),
1074 path="<root>",
1075 severity=ValidationSeverity.ERROR,
1076 )
1077 )
1079 # MR-2: should reference a captured baseline in a later evaluator
1080 capture_names: set[str] = {state.capture for state in fsm.states.values() if state.capture}
1081 if capture_names and not _has_baseline_reference(fsm, capture_names):
1082 errors.append(
1083 ValidationError(
1084 message=(
1085 "Meta-loop appears to lack a measure→propose→apply→re-measure "
1086 "spine: no captured baseline value is referenced by a later evaluator. "
1087 "Meta-loops should compare a post-change score against a pre-change "
1088 "baseline (see loops/harness-optimize.yaml as reference template). "
1089 "To suppress, set `meta_self_eval_ok: true`."
1090 ),
1091 path="<root>",
1092 severity=ValidationSeverity.WARNING,
1093 )
1094 )
1096 return errors
1099# Regex patterns for detecting counter-increment actions.
1100# Must contain a printf/echo writing to a file AND an arithmetic increment.
1101_COUNTER_FILE_WRITE_RE = re.compile(r"(?:printf|echo)\s+.*>")
1102_COUNTER_INCREMENT_RE = re.compile(
1103 r"\$\(\(.*\+\s*1\s*\)\)" # $((N + 1)) or $((N+1))
1104 r"|\+\+" # C-style increment
1105 r"|\+=1" # compound assignment
1106 r"|awk\s+.*\+\+" # awk with increment
1107)
1110def _validate_zero_retry_counter(fsm: FSMLoop) -> list[ValidationError]:
1111 """Detect counter + output_numeric combos that yield zero effective retries.
1113 A common loop-authoring footgun: a state increments a counter file and then
1114 evaluates ``output_numeric`` with ``operator: lt, target: 1`` against it.
1115 After the first increment the counter is 1, ``1 < 1 == false``, so the
1116 retry budget is 0 by construction. Author almost always intended target=2.
1117 """
1118 errors: list[ValidationError] = []
1120 for state_name, state in fsm.states.items():
1121 if not state.action or not state.evaluate:
1122 continue
1124 ev = state.evaluate
1125 if ev.type != "output_numeric":
1126 continue
1127 if ev.operator is None or ev.target is None:
1128 continue
1130 # Must be a number-like target for numeric comparison
1131 try:
1132 target = float(ev.target)
1133 except (ValueError, TypeError):
1134 continue
1136 if not _is_counter_action(state.action):
1137 continue
1139 # Check: after first increment (0→1), does operator(1, target) already fail?
1140 op_fn = _NUMERIC_OPERATORS.get(ev.operator)
1141 if op_fn is None:
1142 continue
1144 if not op_fn(1.0, target):
1145 suggested_target = _suggested_target(ev.operator, target)
1146 errors.append(
1147 ValidationError(
1148 message=(
1149 f"Zero retry budget: operator={ev.operator} target={target} "
1150 f"means the first post-increment value (1) already fails "
1151 f"({ev.operator}(1, {target}) == False). "
1152 f"Did you mean target={suggested_target}?"
1153 ),
1154 path=f"states.{state_name}.evaluate",
1155 severity=ValidationSeverity.WARNING,
1156 )
1157 )
1159 return errors
1162def _is_counter_action(action: str) -> bool:
1163 """Return True if the action string contains a counter-increment pattern."""
1164 return bool(_COUNTER_FILE_WRITE_RE.search(action) and _COUNTER_INCREMENT_RE.search(action))
1167def _suggested_target(operator: str, target: float) -> str:
1168 """Suggest a target value that allows at least one retry."""
1169 # For lt/le with a too-low target, suggest target+1 so first post-increment passes
1170 if operator in ("lt", "le"):
1171 return str(int(target) + 1)
1172 # For eq with target=0, suggest 1 so the counter can eventually match
1173 if operator == "eq" and target == 0:
1174 return "1"
1175 # For other cases, suggest target+1 as a default nudge
1176 return str(int(target) + 1)
1179def _validate_harness_multimodal_evaluator_blind_spot(fsm: FSMLoop) -> list[ValidationError]:
1180 """Warn when harness loops use LLM multimodal eval as sole gate to terminal.
1182 LLMs can silently fall back to text-only analysis when reading images,
1183 producing verdicts based on incomplete information. The output_contains
1184 evaluator can verify the LLM wrote the pass string but not that it
1185 actually processed the image. This is the same class of failure as MR-1
1186 (LLM self-evaluation bias) applied to artifact evaluation rather than
1187 harness modification.
1189 Suppressed by ``meta_self_eval_ok: true`` at the loop top-level.
1190 """
1191 errors: list[ValidationError] = []
1192 if fsm.meta_self_eval_ok or fsm.category != "harness":
1193 return errors
1195 terminal_states = fsm.get_terminal_states()
1197 for state_name, state in fsm.states.items():
1198 if state.action_type != "prompt" or not state.action:
1199 continue
1200 if state.evaluate is None or state.evaluate.type != "output_contains":
1201 continue
1202 if not any(p.search(state.action) for p in _MULTIMODAL_EVAL_PATTERNS):
1203 continue
1204 if state.on_yes not in terminal_states:
1205 continue
1206 errors.append(
1207 ValidationError(
1208 message=(
1209 f"State '{state_name}' evaluates a screenshot/image via LLM "
1210 "prompt and routes directly to a terminal on success. The "
1211 "output_contains evaluator can verify the LLM wrote the pass "
1212 "string but not that the LLM actually processed the image. "
1213 "Consider adding a shell-action verification state (e.g., "
1214 "functional smoke test) between scoring and the terminal."
1215 ),
1216 path=f"states.{state_name}",
1217 severity=ValidationSeverity.WARNING,
1218 )
1219 )
1221 return errors
1224def _find_shared_tmp_writes(fsm: FSMLoop) -> list[tuple[str, str]]:
1225 """Return (state_name, matched_path) for every action referencing shared .loops/tmp/.
1227 Scans `state.action` only. Prompts and sub-loop bindings can also reference
1228 paths, but those are out of static-scan reach: action strings are the
1229 place where loop YAMLs directly encode artifact paths.
1230 """
1231 findings: list[tuple[str, str]] = []
1232 for state_name, state in fsm.states.items():
1233 if not state.action:
1234 continue
1235 for match in _SHARED_TMP_PATH_RE.finditer(state.action):
1236 findings.append((state_name, match.group(0)))
1237 return findings
1240def _validate_input_key_without_guard(fsm: FSMLoop) -> list[ValidationError]:
1241 """Warn when a loop sets a custom input_key but omits required_inputs.
1243 A loop that accepts a runtime input via a named key (e.g. input_key: description)
1244 but doesn't declare required_inputs will silently proceed with an empty value if the
1245 user forgets to pass one. Declaring required_inputs shifts that failure to start-time.
1246 """
1247 if fsm.input_key == "input":
1248 return []
1249 if fsm.required_inputs:
1250 return []
1251 return [
1252 ValidationError(
1253 message=(
1254 f"Loop sets input_key: '{fsm.input_key}' but does not declare "
1255 f"required_inputs. If this input is mandatory, add "
1256 f"'required_inputs: [\"{fsm.input_key}\"]' to make the runner "
1257 f"abort when no input is provided."
1258 ),
1259 path="required_inputs",
1260 severity=ValidationSeverity.WARNING,
1261 )
1262 ]
1265def _validate_artifact_isolation(fsm: FSMLoop) -> list[ValidationError]:
1266 """Validate rule MR-3: loops must isolate artifacts to ${context.run_dir}.
1268 The runner injects ${context.run_dir} pointing at .loops/runs/<loop>-<ts>/
1269 and creates the folder before execution. Loops that write intermediate
1270 state (queues, checkpoints, generated files) to shared .loops/tmp/ paths
1271 will corrupt each other under concurrent runs (ll-parallel workers, retries,
1272 repeated invocations).
1274 Suppressed by `shared_state_ok: true` at the loop top-level for loops that
1275 intentionally share state across runs.
1276 """
1277 if fsm.shared_state_ok:
1278 return []
1279 errors: list[ValidationError] = []
1280 for state_name, path in _find_shared_tmp_writes(fsm):
1281 errors.append(
1282 ValidationError(
1283 message=(
1284 f"State writes to shared '{path}' instead of "
1285 "'${context.run_dir}/...'. Concurrent runs of this loop "
1286 "(e.g., under ll-parallel) will corrupt each other's state. "
1287 "Use the runner-injected `${context.run_dir}` for per-run "
1288 "artifact paths, or set `shared_state_ok: true` at the loop "
1289 "top-level if cross-run sharing is intentional."
1290 ),
1291 path=f"states.{state_name}.action",
1292 severity=ValidationSeverity.WARNING,
1293 )
1294 )
1295 return errors
1298def _is_llm_judged(state: StateConfig) -> bool:
1299 """Return True if this state will be graded by the default LLM judge.
1301 Mirrors the action-mode detection in executor._action_mode() (not imported
1302 here because that is runtime code). A state is LLM-judged when:
1303 - it has no explicit evaluate block AND its action is a prompt or slash_command, OR
1304 - it has an explicit evaluate block of type llm_structured or check_semantic.
1305 """
1306 if state.evaluate is None:
1307 # Heuristic: explicit action_type wins; fall back to leading "/" on action string.
1308 action_type = state.action_type
1309 if action_type in ("prompt", "slash_command"):
1310 return True
1311 if action_type is None and state.action and state.action.lstrip().startswith("/"):
1312 return True
1313 return False
1314 return state.evaluate.type in ("llm_structured", "check_semantic")
1317def _validate_partial_route_dead_end(fsm: FSMLoop) -> list[ValidationError]:
1318 """Validate rule MR-4: LLM-judged states with only on_yes have a partial/no dead-end.
1320 A state gated by the default LLM judge can receive yes/no/partial verdicts.
1321 If only on_yes is mapped (and no on_no, on_partial, next, or route table with
1322 a default exist), a partial or no verdict returns None from _route and silently
1323 terminates the loop — the parent treats this as failed.
1325 Suppressed by `partial_route_ok: true` at the loop top-level for the rare
1326 case where dead-ending on a non-yes verdict is intentional.
1327 """
1328 if fsm.partial_route_ok:
1329 return []
1330 errors: list[ValidationError] = []
1331 for state_name, state in fsm.states.items():
1332 if not _is_llm_judged(state):
1333 continue
1334 # States with an unconditional next: or a full route: table are safe.
1335 if state.next is not None or state.route is not None:
1336 continue
1337 # Only flag when on_yes is set but at least one of on_no/on_partial is missing.
1338 if state.on_yes is None:
1339 continue
1340 missing = [v for v in ("no", "partial") if getattr(state, f"on_{v}") is None]
1341 if not missing:
1342 continue
1343 unrouted = " or ".join(f"`{v}`" for v in missing)
1344 errors.append(
1345 ValidationError(
1346 message=(
1347 f"[state: {state_name}] LLM-judged prompt routes only on_yes; "
1348 f"a {unrouted} verdict has no route and will dead-end the loop "
1349 "(parent reads this as failed). Add on_no/on_partial, use `next:` "
1350 "for an unconditional handoff, or a `route:` table with a default. "
1351 "Set `partial_route_ok: true` at the loop top-level to suppress "
1352 "if intentional. (ENH-1917)"
1353 ),
1354 path=f"states.{state_name}",
1355 severity=ValidationSeverity.WARNING,
1356 )
1357 )
1358 return errors
1361def _validate_artifact_overwrite(fsm: FSMLoop) -> list[ValidationError]:
1362 """Validate rule MR-5 (ENH-1957): harness loops should version artifacts per iteration.
1364 A harness-category loop that iteratively generates and overwrites a flat artifact
1365 path (e.g. ``${context.run_dir}/image.svg``) loses all intermediate versions.
1366 Only the final iteration survives. This rule flags iterative generate→evaluate→generate
1367 cycles that write to artifact paths without declaring versioning intent.
1369 Suppressed by ``artifact_versioning: true`` (loop snapshots per-iteration artifacts)
1370 or ``artifact_versioning_ok: true`` (intentional overwrite, e.g. artifact varies
1371 by task).
1372 """
1373 if fsm.artifact_versioning or fsm.artifact_versioning_ok:
1374 return []
1375 if fsm.category not in ("harness",):
1376 return []
1378 errors: list[ValidationError] = []
1380 # Find states that write to artifact paths (shell actions with file output)
1381 writers: dict[str, set[str]] = {} # state_name -> set of artifact paths
1382 for state_name, state in fsm.states.items():
1383 if not state.action or state.action_type not in ("shell", None):
1384 continue
1385 # Skip sub-loop delegation states
1386 if state.action_type == "loop":
1387 continue
1388 action = state.action
1389 # Find run_dir-based artifact writes: ${context.run_dir}/<path> or $RUNDIR/<path>
1390 import re
1392 # Match patterns like: ${context.run_dir}/output.svg, $RUNDIR/image.png, > path
1393 # We look for output redirections or cp/mv commands writing to run_dir
1394 artifact_refs = set()
1395 for pattern in (
1396 r'\$\{context\.run_dir\}/([^\s"\';&]+)',
1397 r'\$\{captured\.[^}]+\}/([^\s"\';&]+)',
1398 ):
1399 for m in re.finditer(pattern, action):
1400 artifact_refs.add(m.group(1))
1401 # Also detect explicit cp/mv writing to run_dir paths
1402 for m in re.finditer(r'(?:cp|mv)\s+.*\s+\$\{context\.run_dir\}/([^\s"\';&]+)', action):
1403 artifact_refs.add(m.group(1))
1404 if artifact_refs:
1405 writers[state_name] = artifact_refs
1407 if not writers:
1408 return []
1410 # Detect iterative cycles: a writer state that is reachable from itself
1411 # via a non-trivial path through other states (generate → evaluate → generate)
1412 for state_name in writers:
1413 refs = fsm.states[state_name].get_referenced_states()
1414 # Check if this writer or its downstream states loop back to this writer
1415 visited: set[str] = set()
1416 to_visit = list(refs - {"$current"})
1417 while to_visit:
1418 target = to_visit.pop()
1419 if target in visited or target not in fsm.states:
1420 continue
1421 visited.add(target)
1422 if target == state_name:
1423 # Found a cycle: this writer state is reachable from itself
1424 artifact_list = ", ".join(sorted(writers[state_name]))
1425 errors.append(
1426 ValidationError(
1427 message=(
1428 f"[state: {state_name}] Harness loop writes artifact(s) "
1429 f"({artifact_list}) to a flat path in an iterative cycle "
1430 f"({state_name} → ... → {state_name}). Per-iteration versions "
1431 "are lost; only the final output survives. Add per-iteration "
1432 "snapshots (see oracle generator-evaluator for pattern) and "
1433 "declare `artifact_versioning: true`, or set "
1434 "`artifact_versioning_ok: true` if intentional. (ENH-1957)"
1435 ),
1436 path=f"states.{state_name}",
1437 severity=ValidationSeverity.WARNING,
1438 )
1439 )
1440 break
1441 target_state = fsm.states.get(target)
1442 if target_state is not None:
1443 target_refs = target_state.get_referenced_states()
1444 for r in target_refs:
1445 if r != "$current" and r not in visited:
1446 to_visit.append(r)
1448 return errors
1451def _has_baseline_reference(fsm: FSMLoop, capture_names: set[str]) -> bool:
1452 """Return True if any evaluate block references a captured variable."""
1453 for state in fsm.states.values():
1454 ev = state.evaluate
1455 if ev is None:
1456 continue
1457 # Check string fields that may interpolate captured values
1458 candidates = [ev.previous, ev.source]
1459 if isinstance(ev.target, str):
1460 candidates.append(ev.target)
1461 for field_val in candidates:
1462 if not field_val:
1463 continue
1464 for name in capture_names:
1465 if f"captured.{name}" in field_val:
1466 return True
1467 return False
1470def _validate_on_max_iterations(fsm: FSMLoop, defined_states: set[str]) -> list[ValidationError]:
1471 """Validate the top-level `on_max_iterations` field (ENH-1631).
1473 Checks that the named state exists when `on_max_iterations` is set.
1474 """
1475 errors: list[ValidationError] = []
1476 if fsm.on_max_iterations is None:
1477 return errors
1478 if fsm.on_max_iterations not in defined_states:
1479 errors.append(
1480 ValidationError(
1481 message=(
1482 f"on_max_iterations references unknown state "
1483 f"'{fsm.on_max_iterations}' (must be a declared state)"
1484 ),
1485 path="on_max_iterations",
1486 )
1487 )
1488 return errors
1491def _validate_circuit(fsm: FSMLoop, defined_states: set[str]) -> list[ValidationError]:
1492 """Validate the top-level `circuit:` block (FEAT-1637).
1494 Checks:
1495 - `circuit.repeated_failure.window` is a positive integer.
1496 - `circuit.repeated_failure.on_repeated_failure` is either the special
1497 token ``"abort"`` or the name of a declared state.
1498 """
1499 errors: list[ValidationError] = []
1500 if fsm.circuit is None or fsm.circuit.repeated_failure is None:
1501 return errors
1503 rf = fsm.circuit.repeated_failure
1504 if rf.window < 1:
1505 errors.append(
1506 ValidationError(
1507 message=f"circuit.repeated_failure.window must be >= 1, got {rf.window}",
1508 path="circuit.repeated_failure.window",
1509 )
1510 )
1512 target = rf.on_repeated_failure
1513 if target not in STALL_SPECIAL_TOKENS and target not in defined_states:
1514 errors.append(
1515 ValidationError(
1516 message=(
1517 f"circuit.repeated_failure.on_repeated_failure references "
1518 f"unknown state '{target}' (must be a declared state or "
1519 f'the literal "abort")'
1520 ),
1521 path="circuit.repeated_failure.on_repeated_failure",
1522 )
1523 )
1525 return errors
1528# Matches common interpolation prefixes used in loop YAML paths so we can
1529# extract the portable relative component for action-string scanning.
1530_INTERPOLATION_PREFIX_RE = re.compile(r"^\$\{[^}]+\}/")
1533def _strip_interpolation_prefix(path: str) -> str:
1534 """Return the path with any leading ${...}/ prefix removed."""
1535 return _INTERPOLATION_PREFIX_RE.sub("", path)
1538def _validate_progress_paths_isolation(fsm: FSMLoop) -> list[ValidationError]:
1539 """Warn when a state's action writes to a file listed in progress_paths (BUG-1767).
1541 When a loop's own bookkeeping files appear in both progress_paths and the
1542 state action strings, every append to those files resets the stall window,
1543 silently disabling the BUG-1674 stall guard for that loop. Authors should
1544 move such files to exclude_paths so the stall detector can still fire.
1545 """
1546 if fsm.circuit is None or fsm.circuit.repeated_failure is None:
1547 return []
1548 rf = fsm.circuit.repeated_failure
1549 if not rf.progress_paths:
1550 return []
1552 # Build a set of the relative path components we need to look for.
1553 watched = {_strip_interpolation_prefix(p) for p in rf.progress_paths}
1554 # Exclude paths that are already in exclude_paths — author acknowledged.
1555 excluded = {_strip_interpolation_prefix(p) for p in rf.exclude_paths}
1556 active_watched = watched - excluded
1557 if not active_watched:
1558 return []
1560 errors: list[ValidationError] = []
1561 for state_name, state in fsm.states.items():
1562 if not state.action:
1563 continue
1564 for path_fragment in active_watched:
1565 if path_fragment in state.action:
1566 errors.append(
1567 ValidationError(
1568 message=(
1569 f"State action references '{path_fragment}', which is also "
1570 "listed in circuit.repeated_failure.progress_paths. Writes "
1571 "to this file will reset the stall window every cycle, "
1572 "silently disabling stall detection. Move it to "
1573 "circuit.repeated_failure.exclude_paths to separate "
1574 "bookkeeping files from real progress signals."
1575 ),
1576 path=f"states.{state_name}.action",
1577 severity=ValidationSeverity.WARNING,
1578 )
1579 )
1580 return errors
1583def _dominated_by_any(fsm: FSMLoop, dominators: set[str], dominated: str) -> bool:
1584 """Return True if the set ``dominators`` collectively dominates ``dominated``.
1586 Group domination: every path from the initial state to ``dominated`` must
1587 pass through at least one state in ``dominators``. Checked by removing all
1588 dominator states from the graph and testing whether ``dominated`` is still
1589 reachable from the initial state.
1591 This generalizes single-state domination — used when a capture variable is
1592 produced by more than one state on mutually-exclusive branches, where the
1593 reference is safe as long as *some* capturing state runs on every path.
1595 Args:
1596 fsm: The FSM loop to analyze
1597 dominators: Names of the states that should collectively dominate
1598 dominated: Name of the state that should be dominated
1600 Returns:
1601 True if the dominators collectively dominate ``dominated``
1602 """
1603 if dominated in dominators:
1604 return True
1605 if dominated not in fsm.states:
1606 return False
1608 visited: set[str] = set()
1609 to_visit: deque[str] = deque([fsm.initial])
1611 while to_visit:
1612 current = to_visit.popleft()
1613 if current in visited or current not in fsm.states:
1614 continue
1615 if current in dominators:
1616 continue # Block this node (simulate removal)
1618 visited.add(current)
1620 if current == dominated:
1621 # Reached dominated without going through any dominator
1622 return False
1624 state = fsm.states[current]
1625 for ref in state.get_referenced_states():
1626 if ref != "$current" and ref not in visited:
1627 to_visit.append(ref)
1629 # Dominated not reachable without the dominators → they dominate
1630 return True
1633def _dominates(fsm: FSMLoop, dominator: str, dominated: str) -> bool:
1634 """Return True if dominator dominates dominated in the FSM graph.
1636 A state D dominates S if every path from the initial state to S must pass
1637 through D. Thin single-state wrapper around :func:`_dominated_by_any`.
1638 """
1639 if dominator not in fsm.states:
1640 return False
1641 return _dominated_by_any(fsm, {dominator}, dominated)
1644def _find_bypass_path_any(fsm: FSMLoop, dominators: set[str], dominated: str) -> list[str]:
1645 """Find an example path from initial to dominated that bypasses all dominators.
1647 Uses BFS to find the shortest path that avoids every state in ``dominators``.
1648 Returns empty list if no bypass exists (should not happen when called after
1649 :func:`_dominated_by_any` returns False).
1650 """
1651 parent: dict[str, str] = {}
1652 to_visit: deque[str] = deque([fsm.initial])
1653 visited: set[str] = set()
1655 while to_visit:
1656 current = to_visit.popleft()
1657 if current in visited or current not in fsm.states:
1658 continue
1659 if current in dominators:
1660 continue
1662 visited.add(current)
1664 if current == dominated:
1665 # Reconstruct path
1666 path = [dominated]
1667 while path[-1] in parent:
1668 path.append(parent[path[-1]])
1669 path.reverse()
1670 return path
1672 state = fsm.states[current]
1673 for ref in state.get_referenced_states():
1674 if ref != "$current" and ref not in visited:
1675 if ref not in parent:
1676 parent[ref] = current
1677 to_visit.append(ref)
1679 return []
1682def _find_bypass_path(fsm: FSMLoop, dominator: str, dominated: str) -> list[str]:
1683 """Find an example path from initial to dominated that bypasses dominator.
1685 Thin single-state wrapper around :func:`_find_bypass_path_any`.
1686 """
1687 return _find_bypass_path_any(fsm, {dominator}, dominated)
1690def _has_sub_loop_state(fsm: FSMLoop) -> bool:
1691 """Return True if any state in the FSM has ``loop:`` set (delegates to a child loop).
1693 Used by ENH-1961 to distinguish "capture lives in a sub-loop" from "capture is missing".
1694 """
1695 return any(state.loop is not None for state in fsm.states.values())
1698def _validate_capture_reachability(fsm: FSMLoop) -> list[ValidationError]:
1699 """Validate that ``${captured.*}`` references are dominated by their capturing states.
1701 ENH-1961: For each state that references ``${captured.<var>.*}`` in its action
1702 or evaluate source, checks that the capturing state dominates the referencing
1703 state (i.e., all paths from the initial state pass through the capture state).
1705 Emits:
1706 - WARNING when a capture state does not dominate a referencing state
1707 (the reference may crash at runtime on paths that bypass the capture).
1708 - ERROR when a referenced capture variable has no capturing state at all
1709 in this FSM (excluding sub-loop captures which live in child namespaces).
1710 """
1711 errors: list[ValidationError] = []
1713 # Step 1: Build capture map (var_name → set of capturing state names).
1714 # A variable may be captured by more than one state on mutually-exclusive
1715 # branches (e.g. fifo_pop vs select_next dispatched by schedule_mode); the
1716 # reference is safe as long as the *set* collectively dominates it.
1717 capture_map: dict[str, set[str]] = {}
1718 for state_name, state in fsm.states.items():
1719 if state.capture:
1720 capture_map.setdefault(state.capture, set()).add(state_name)
1722 # Step 2: Build reference map (state_name → set of captured var names referenced)
1723 reference_map: dict[str, set[str]] = {}
1724 for state_name, state in fsm.states.items():
1725 # Skip sub-loop delegation states — their action is a loop name,
1726 # and captured vars belong to the child loop's namespace.
1727 if state.loop is not None:
1728 continue
1730 refs: set[str] = set()
1731 if state.action:
1732 refs.update(_CAPTURED_REF_RE.findall(state.action))
1733 if state.evaluate is not None and state.evaluate.source:
1734 refs.update(_CAPTURED_REF_RE.findall(state.evaluate.source))
1735 if refs:
1736 reference_map[state_name] = refs
1738 if not reference_map:
1739 return errors
1741 # Step 3: For each reference, check dominance of capturing state
1742 for ref_state_name, ref_vars in reference_map.items():
1743 for var_name in ref_vars:
1744 if var_name not in capture_map:
1745 # Referenced capture variable has no capturing state in this FSM.
1746 # ENH-1998: downgrade to WARNING (not silent skip) when sub-loops
1747 # are present — the capture may live in a child namespace, but a
1748 # typo'd name should still surface rather than go completely dark.
1749 if _has_sub_loop_state(fsm):
1750 errors.append(
1751 ValidationError(
1752 message=(
1753 f"References ${{captured.{var_name}.*}} but no state in "
1754 f"this loop captures '{var_name}'. "
1755 f"If '{var_name}' is produced by a sub-loop, this may be "
1756 f"intentional; otherwise add 'capture: {var_name}' to the "
1757 f"state that produces this value."
1758 ),
1759 path=f"states.{ref_state_name}.action",
1760 severity=ValidationSeverity.WARNING,
1761 )
1762 )
1763 continue
1764 # No sub-loops: this is genuinely missing.
1765 errors.append(
1766 ValidationError(
1767 message=(
1768 f"References ${{captured.{var_name}.*}} but no state in "
1769 f"this loop captures '{var_name}'. Add 'capture: {var_name}' "
1770 f"to the state that produces this value."
1771 ),
1772 path=f"states.{ref_state_name}.action",
1773 severity=ValidationSeverity.ERROR,
1774 )
1775 )
1776 continue
1778 # Capturing states present in this FSM (shouldn't drop any normally)
1779 cap_states = {s for s in capture_map[var_name] if s in fsm.states}
1780 if not cap_states:
1781 continue
1783 # Group dominance check: do the capturing states collectively
1784 # dominate ref_state_name (does at least one run on every path)?
1785 if not _dominated_by_any(fsm, cap_states, ref_state_name):
1786 bypass_path = _find_bypass_path_any(fsm, cap_states, ref_state_name)
1787 path_str = " → ".join(bypass_path) if bypass_path else "unknown path"
1789 if len(cap_states) == 1:
1790 captured_by = f"state '{next(iter(cap_states))}' which may not"
1791 else:
1792 names = ", ".join(f"'{s}'" for s in sorted(cap_states))
1793 captured_by = f"states {names}, none of which"
1795 errors.append(
1796 ValidationError(
1797 message=(
1798 f"References ${{captured.{var_name}.*}} but '{var_name}' "
1799 f"is captured by {captured_by} "
1800 f"execute on all paths to '{ref_state_name}'. "
1801 f"Path(s) bypassing capture: {path_str}"
1802 ),
1803 path=f"states.{ref_state_name}.action",
1804 severity=ValidationSeverity.WARNING,
1805 )
1806 )
1808 return errors
1811def _find_reachable_states(fsm: FSMLoop) -> set[str]:
1812 """Find all states reachable from the initial state.
1814 Uses breadth-first search to find all reachable states. Seeds the BFS
1815 with the initial state plus top-level transition targets that act as
1816 alternate entry points: ``on_max_iterations`` (fires when the iteration
1817 cap is hit) and ``circuit.repeated_failure.on_repeated_failure`` (fires
1818 when the circuit breaker trips). These are real edges the runtime can
1819 take, so states reached only through them are not orphans.
1821 Args:
1822 fsm: The FSM loop to analyze
1824 Returns:
1825 Set of reachable state names
1826 """
1827 reachable: set[str] = set()
1828 to_visit: deque[str] = deque([fsm.initial])
1829 if fsm.on_max_iterations is not None:
1830 to_visit.append(fsm.on_max_iterations)
1831 if fsm.circuit is not None and fsm.circuit.repeated_failure is not None:
1832 target = fsm.circuit.repeated_failure.on_repeated_failure
1833 if target not in STALL_SPECIAL_TOKENS:
1834 to_visit.append(target)
1836 while to_visit:
1837 current = to_visit.popleft()
1838 if current in reachable or current not in fsm.states:
1839 continue
1841 reachable.add(current)
1842 state = fsm.states[current]
1843 refs = state.get_referenced_states()
1845 for ref in refs:
1846 if ref != "$current" and ref not in reachable:
1847 to_visit.append(ref)
1849 return reachable
1852def is_runnable_loop(path: Path) -> bool:
1853 """Cheap check for whether a YAML file is a runnable FSM loop definition.
1855 Returns True iff the file parses as a YAML mapping with the required
1856 top-level keys ``name``, ``initial``, and either ``states`` or ``flow``
1857 (the shorthand resolved by :func:`resolve_flow`). This matches the
1858 required-fields gate in :func:`load_and_validate` (lines 885-894) so
1859 "counted by the verifier" stays in sync with "runnable by ll-loop validate".
1861 No fragment/inheritance resolution is performed — library fragments under
1862 ``loops/lib/`` that omit ``initial`` or ``states`` correctly return False.
1863 """
1864 try:
1865 data = yaml.safe_load(path.read_text())
1866 except (OSError, yaml.YAMLError):
1867 return False
1868 if not isinstance(data, dict):
1869 return False
1870 has_flow = "states" in data or "flow" in data
1871 return "name" in data and "initial" in data and has_flow
1874def load_and_validate(path: Path) -> tuple[FSMLoop, list[ValidationError]]:
1875 """Load YAML file and validate FSM structure.
1877 Args:
1878 path: Path to the YAML file to load
1880 Returns:
1881 Tuple of (validated FSMLoop instance, list of WARNING-severity ValidationErrors)
1883 Raises:
1884 FileNotFoundError: If the file doesn't exist
1885 yaml.YAMLError: If the file is not valid YAML
1886 ValueError: If validation fails (contains error details)
1887 """
1888 if not path.exists():
1889 raise FileNotFoundError(f"FSM file not found: {path}")
1891 with open(path) as f:
1892 data: dict[str, Any] = yaml.safe_load(f)
1894 if not isinstance(data, dict):
1895 raise ValueError(f"FSM file must contain a YAML mapping, got {type(data)}")
1897 # Resolve `from:` inheritance before any further checks, so a child loop
1898 # can omit fields its parent provides (including `initial`/`states`) and
1899 # so a parent's `import:`/`fragments:` blocks survive into the merged
1900 # result for the subsequent `resolve_fragments` pass.
1901 data = resolve_inheritance(data, path.parent)
1903 # Expand flow: linear shorthand into states: before required-fields check
1904 data = resolve_flow(data)
1906 # Check required fields before parsing
1907 missing = []
1908 for field in ["name", "initial"]:
1909 if field not in data:
1910 missing.append(field)
1911 if "states" not in data:
1912 missing.append("states (or flow)")
1914 if missing:
1915 raise ValueError(f"FSM file missing required fields: {', '.join(missing)}")
1917 # Check for unknown top-level keys before parsing
1918 unknown_key_warnings: list[ValidationError] = []
1919 unknown = set(data.keys()) - KNOWN_TOP_LEVEL_KEYS
1920 if unknown:
1921 unknown_key_warnings.append(
1922 ValidationError(
1923 path="<root>",
1924 message=f"Unknown top-level keys: {', '.join(sorted(unknown))}",
1925 severity=ValidationSeverity.WARNING,
1926 )
1927 )
1929 # Resolve fragment libraries before parsing into dataclass
1930 data = resolve_fragments(data, path.parent)
1932 # Parse into dataclass
1933 fsm = FSMLoop.from_dict(data)
1935 # Validate
1936 errors = validate_fsm(fsm)
1938 # Validate with: bindings against child loop parameters (requires file-system access)
1939 errors.extend(_validate_with_bindings(fsm, path.parent))
1940 errors.extend(_validate_fragment_bindings(fsm, path.parent))
1942 # Filter to errors only (not warnings) for raising
1943 error_list = [e for e in errors if e.severity == ValidationSeverity.ERROR]
1945 if error_list:
1946 error_messages = "\n ".join(str(e) for e in error_list)
1947 raise ValueError(f"FSM validation failed:\n {error_messages}")
1949 # Collect all warnings (unknown-key warnings + structural warnings)
1950 struct_warnings = [e for e in errors if e.severity == ValidationSeverity.WARNING]
1951 all_warnings = unknown_key_warnings + struct_warnings
1952 for warning in all_warnings:
1953 logger.warning(str(warning))
1955 return fsm, all_warnings