Coverage for little_loops / fsm / validation.py: 43%
737 statements
« prev ^ index » next coverage.py v7.12.0, created at 2026-06-16 13:12 -0500
1"""FSM loop validation logic.
3This module provides validation for FSM loop definitions, ensuring
4structural correctness and catching common configuration errors.
6Validation checks:
7- Initial state exists in states dict
8- All referenced states exist
9- At least one terminal state
10- Evaluator configs have required fields for their type
11- No conflicting routing (shorthand vs full route)
12- Numeric fields in valid ranges (max_iterations > 0, backoff >= 0, timeout > 0)
13"""
15from __future__ import annotations
17import logging
18import re
19from collections import deque
20from dataclasses import dataclass
21from enum import Enum
22from pathlib import Path
23from typing import Any
25import yaml
27from little_loops.fsm.evaluators import _NUMERIC_OPERATORS
28from little_loops.fsm.fragments import resolve_flow, resolve_fragments, resolve_inheritance
29from little_loops.fsm.schema import EvaluateConfig, FSMLoop, ParameterSpec, StateConfig
31logger = logging.getLogger(__name__)
34class ValidationSeverity(Enum):
35 """Severity level for validation issues."""
37 ERROR = "error"
38 WARNING = "warning"
41@dataclass
42class ValidationError:
43 """Structured validation error.
45 Attributes:
46 message: Human-readable error description
47 path: Path to the problematic element (e.g., "states.check.route")
48 severity: Error severity (error or warning)
49 """
51 message: str
52 path: str | None = None
53 severity: ValidationSeverity = ValidationSeverity.ERROR
55 def __str__(self) -> str:
56 """Format error for display."""
57 prefix = f"[{self.severity.value.upper()}]"
58 if self.path:
59 return f"{prefix} {self.path}: {self.message}"
60 return f"{prefix} {self.message}"
63# Evaluator type to required fields mapping
64EVALUATOR_REQUIRED_FIELDS: dict[str, list[str]] = {
65 "exit_code": [],
66 "output_numeric": ["operator", "target"],
67 "output_json": ["path", "operator", "target"],
68 "output_contains": ["pattern"],
69 "convergence": ["target"],
70 "diff_stall": [],
71 "action_stall": [],
72 "llm_structured": [],
73 "mcp_result": [],
74 "harbor_scorer": [],
75 "comparator": ["baseline_path"],
76 "contract": ["pairs"],
77 "classify": [],
78}
80# Non-LLM evaluator types: all evaluator types except llm_structured
81# Derived from EVALUATOR_REQUIRED_FIELDS so new types are automatically included
82NON_LLM_EVALUATOR_TYPES: frozenset[str] = frozenset(EVALUATOR_REQUIRED_FIELDS.keys()) - {
83 "llm_structured",
84 "comparator",
85 "contract",
86}
88# Meta-loop detector: action string patterns that indicate harness artifact writes
89_META_LOOP_ACTION_PATTERNS: tuple[re.Pattern[str], ...] = (
90 re.compile(r"loops/[\w-]+\.yaml"),
91 re.compile(r"skills/[\w-]+/SKILL\.md"),
92 re.compile(r"agents/[\w-]+\.md"),
93 re.compile(r"commands/[\w-]+\.md"),
94 re.compile(r"\.claude/(CLAUDE\.md|settings)"),
95)
97# Action string tokens that indicate meta-loop behavior
98_META_LOOP_ACTION_TOKENS: frozenset[str] = frozenset({"yaml_state_editor", "replace_action"})
100# Import paths that identify a loop as a meta-loop (harness optimization framework)
101_META_LOOP_IMPORT_TRIGGERS: frozenset[str] = frozenset({"lib/benchmark.yaml"})
103# MR-3: shared-tmp path detector. The runner injects ${context.run_dir} resolving
104# to .loops/runs/<loop>-<timestamp>/; loops that hardcode .loops/tmp/ instead
105# cause state corruption under concurrent runs (ll-parallel, retries, etc.).
106_SHARED_TMP_PATH_RE = re.compile(r"\.loops/tmp/[\w./-]+")
108# ENH-1961: Regex for extracting captured variable names from ${captured.<var>.*} references
109_CAPTURED_REF_RE = re.compile(r"\$\{captured\.(\w+)")
111# Full-reference form, capturing the var name and the remainder up to the closing
112# brace so we can detect a `:default=` guard. A reference written as
113# `${captured.x.output:default=...}` is provably safe even on paths that bypass
114# the capturing state — the interpolation engine (interpolation.py) substitutes
115# the default when the path is missing — so it must NOT be flagged by the
116# capture-reachability check. _CAPTURED_REF_RE alone can't see the guard.
117_CAPTURED_REF_FULL_RE = re.compile(r"\$\{captured\.(\w+)([^}]*)\}")
120def _unguarded_captured_refs(text: str) -> set[str]:
121 """Return captured var names that have at least one reference WITHOUT a
122 `:default=` guard. Vars referenced only via `${captured.x...:default=...}`
123 are omitted: the default makes a missing value safe, so they should not
124 trigger missing-capture or bypass-path diagnostics.
125 """
126 refs: set[str] = set()
127 for var_name, remainder in _CAPTURED_REF_FULL_RE.findall(text):
128 if ":default=" not in remainder:
129 refs.add(var_name)
130 return refs
133# ENH-1819: Regex patterns for detecting multimodal evaluation in prompt actions
134_MULTIMODAL_EVAL_PATTERNS: tuple[re.Pattern[str], ...] = (
135 re.compile(r"Read the screenshot", re.IGNORECASE),
136 re.compile(r"view the (generated )?(website|page|image)", re.IGNORECASE),
137 re.compile(r"screenshot\.(png|jpg|jpeg|webp)"),
138 re.compile(r"\.(png|jpg|jpeg|webp)\b.*\b(read|view|evaluate|score|judge)", re.IGNORECASE),
139)
141# Valid comparison operators
142VALID_OPERATORS = {"eq", "ne", "lt", "le", "gt", "ge"}
144# Valid values for the top-level `visibility:` field (audience axis for
145# `ll-loop list` filtering). "public" is the default when the field is absent.
146VALID_VISIBILITY: frozenset[str] = frozenset({"public", "internal", "example"})
148# All top-level keys recognized by FSMLoop.from_dict()
149KNOWN_TOP_LEVEL_KEYS: frozenset[str] = frozenset(
150 {
151 "name",
152 "description",
153 "initial",
154 "states",
155 "context",
156 "parameters",
157 "scope",
158 "max_iterations",
159 "on_max_iterations",
160 "max_edge_revisits",
161 "backoff",
162 "timeout",
163 "default_timeout",
164 "maintain",
165 "llm",
166 "on_handoff",
167 "input_key",
168 "required_inputs",
169 "config",
170 "category",
171 "labels",
172 "visibility",
173 "commands",
174 "targets",
175 "circuit",
176 "meta_self_eval_ok",
177 "shared_state_ok",
178 "partial_route_ok",
179 "artifact_versioning",
180 "artifact_versioning_ok",
181 "generator_fix_ok",
182 "import",
183 "fragments",
184 "from",
185 "flow",
186 "state_defs",
187 }
188)
190# Special tokens accepted as the `on_repeated_failure` target on the
191# stall detector — `"abort"` means terminate via _finish("stall_detected").
192STALL_SPECIAL_TOKENS: frozenset[str] = frozenset({"abort"})
194# Valid parameter types for the 'parameters:' block
195VALID_PARAMETER_TYPES: frozenset[str] = frozenset(
196 {"string", "integer", "number", "boolean", "enum", "path"}
197)
200def _validate_evaluator(state_name: str, evaluate: EvaluateConfig) -> list[ValidationError]:
201 """Validate evaluator configuration for type-specific requirements.
203 Args:
204 state_name: Name of the state containing this evaluator
205 evaluate: The evaluator configuration to validate
207 Returns:
208 List of validation errors found
209 """
210 errors: list[ValidationError] = []
211 path = f"states.{state_name}.evaluate"
213 # Check that evaluator type is recognized
214 valid_types = set(EVALUATOR_REQUIRED_FIELDS.keys())
215 if evaluate.type not in valid_types:
216 errors.append(
217 ValidationError(
218 message=f"Unknown evaluator type '{evaluate.type}'. "
219 f"Must be one of: {', '.join(sorted(valid_types))}",
220 path=path,
221 )
222 )
223 return errors # Can't check required fields for unknown type
225 # Check required fields for evaluator type
226 required = EVALUATOR_REQUIRED_FIELDS.get(evaluate.type, [])
227 for field_name in required:
228 value = getattr(evaluate, field_name, None)
229 if value is None:
230 errors.append(
231 ValidationError(
232 message=f"Evaluator type '{evaluate.type}' requires '{field_name}' field",
233 path=path,
234 )
235 )
237 # Validate operator if present
238 if evaluate.operator is not None and evaluate.operator not in VALID_OPERATORS:
239 errors.append(
240 ValidationError(
241 message=f"Invalid operator '{evaluate.operator}'. "
242 f"Must be one of: {', '.join(sorted(VALID_OPERATORS))}",
243 path=f"{path}.operator",
244 )
245 )
247 # Validate convergence-specific fields
248 if evaluate.type == "convergence":
249 if evaluate.direction not in ("minimize", "maximize"):
250 errors.append(
251 ValidationError(
252 message=f"Invalid direction '{evaluate.direction}'. "
253 "Must be 'minimize' or 'maximize'",
254 path=f"{path}.direction",
255 )
256 )
257 # Only validate tolerance if it's a numeric value (not an interpolation string)
258 if (
259 evaluate.tolerance is not None
260 and isinstance(evaluate.tolerance, (int, float))
261 and evaluate.tolerance < 0
262 ):
263 errors.append(
264 ValidationError(
265 message="Tolerance cannot be negative",
266 path=f"{path}.tolerance",
267 )
268 )
270 # Validate llm_structured-specific fields
271 if evaluate.type == "llm_structured":
272 if evaluate.min_confidence < 0 or evaluate.min_confidence > 1:
273 errors.append(
274 ValidationError(
275 message="min_confidence must be between 0 and 1",
276 path=f"{path}.min_confidence",
277 )
278 )
280 # Validate diff_stall-specific fields
281 if evaluate.type == "diff_stall":
282 if evaluate.max_stall < 1:
283 errors.append(
284 ValidationError(
285 message="max_stall must be >= 1",
286 path=f"{path}.max_stall",
287 )
288 )
290 # Validate action_stall-specific fields
291 if evaluate.type == "action_stall":
292 if evaluate.max_repeat < 1:
293 errors.append(
294 ValidationError(
295 message="max_repeat must be >= 1",
296 path=f"{path}.max_repeat",
297 )
298 )
300 return errors
303def _validate_parameters(fsm: FSMLoop) -> list[ValidationError]:
304 """Validate the loop's top-level parameters: block.
306 Args:
307 fsm: The FSM loop to validate
309 Returns:
310 List of validation errors found
311 """
312 errors: list[ValidationError] = []
314 for param_name, param_spec in fsm.parameters.items():
315 path = f"parameters.{param_name}"
317 if param_spec.type not in VALID_PARAMETER_TYPES:
318 errors.append(
319 ValidationError(
320 message=(
321 f"Unknown parameter type '{param_spec.type}'. "
322 f"Must be one of: {', '.join(sorted(VALID_PARAMETER_TYPES))}"
323 ),
324 path=path,
325 )
326 )
328 if param_spec.type == "enum" and not param_spec.values:
329 errors.append(
330 ValidationError(
331 message="Parameter type 'enum' requires a 'values' list",
332 path=path,
333 )
334 )
336 if param_spec.required and param_spec.default is not None:
337 errors.append(
338 ValidationError(
339 message="Parameter cannot be both 'required: true' and have a 'default' value",
340 path=path,
341 )
342 )
344 return errors
347def _check_param_type(value: Any, spec: ParameterSpec) -> str | None:
348 """Return an error message if value does not match spec.type, else None."""
349 if spec.type == "string" and not isinstance(value, str):
350 return f"expected string, got {type(value).__name__}"
351 if spec.type == "integer" and not isinstance(value, int):
352 return f"expected integer, got {type(value).__name__}"
353 if spec.type == "number" and not isinstance(value, (int, float)):
354 return f"expected number, got {type(value).__name__}"
355 if spec.type == "boolean" and not isinstance(value, bool):
356 return f"expected boolean, got {type(value).__name__}"
357 if spec.type == "enum" and spec.values and value not in spec.values:
358 return f"expected one of {spec.values!r}, got {value!r}"
359 return None
362def _validate_with_bindings(fsm: FSMLoop, loop_dir: Path) -> list[ValidationError]:
363 """Validate with: bindings against child loop parameter contracts.
365 Called from load_and_validate (not validate_fsm) because resolving child loops
366 requires file-system access via the loop directory path.
368 Args:
369 fsm: The parent FSM loop
370 loop_dir: Directory to resolve child loop paths from
372 Returns:
373 List of validation errors found
374 """
375 errors: list[ValidationError] = []
377 for state_name, state in fsm.states.items():
378 if state.loop is None or not state.with_:
379 continue
381 # Try to resolve and load the child loop; skip if unavailable
382 try:
383 from little_loops.cli.loop._helpers import resolve_loop_path
385 loop_path = resolve_loop_path(state.loop, loop_dir)
386 child_fsm, _ = load_and_validate(loop_path)
387 except Exception:
388 continue
390 if not child_fsm.parameters:
391 continue # Child has no declared contract — nothing to cross-validate
393 path = f"states.{state_name}"
395 # Unknown with: keys (not declared by child)
396 for key in state.with_:
397 if key not in child_fsm.parameters:
398 errors.append(
399 ValidationError(
400 message=(
401 f"'with.{key}' is not a declared parameter of loop '{state.loop}'. "
402 f"Declared: {', '.join(sorted(child_fsm.parameters))}"
403 ),
404 path=f"{path}.with.{key}",
405 )
406 )
408 # Required parameters not bound
409 for param_name, param_spec in child_fsm.parameters.items():
410 if param_spec.required and param_name not in state.with_:
411 errors.append(
412 ValidationError(
413 message=(
414 f"Required parameter '{param_name}' of loop '{state.loop}' "
415 f"is not bound in 'with'"
416 ),
417 path=f"{path}.with",
418 )
419 )
421 # Statically-detectable type mismatches (skip interpolation strings)
422 for param_name, value in state.with_.items():
423 if param_name not in child_fsm.parameters:
424 continue
425 if isinstance(value, str) and "${" in value:
426 continue
427 type_error = _check_param_type(value, child_fsm.parameters[param_name])
428 if type_error:
429 errors.append(
430 ValidationError(
431 message=f"Parameter '{param_name}': {type_error}",
432 path=f"{path}.with.{param_name}",
433 )
434 )
436 return errors
439def _validate_fragment_bindings(fsm: FSMLoop, loop_dir: Path) -> list[ValidationError]:
440 """Validate fragment with: bindings against fragment parameter contracts.
442 Called from load_and_validate (not validate_fsm) because fragment parameters
443 are populated by resolve_fragments which runs before dataclass parsing.
445 Args:
446 fsm: The FSM loop to validate
447 loop_dir: Directory containing the loop file (unused; kept for API symmetry with
448 _validate_with_bindings)
450 Returns:
451 List of validation errors found
452 """
453 # Runner-injected vars available at runtime but not at static analysis time
454 RUNNER_INJECTED = {"run_dir", "loop_name", "started_at", "input_hash"}
456 errors: list[ValidationError] = []
458 for state_name, state in fsm.states.items():
459 if not state.fragment_parameters:
460 continue # No declared contract — nothing to cross-validate
462 path = f"states.{state_name}"
464 # Unknown with: keys (not declared by fragment)
465 for key in state.fragment_bindings:
466 if key not in state.fragment_parameters:
467 errors.append(
468 ValidationError(
469 message=(
470 f"'with.{key}' is not a declared parameter of fragment "
471 f"'{state.fragment_name}'. "
472 f"Declared: {', '.join(sorted(state.fragment_parameters))}"
473 ),
474 path=f"{path}.with.{key}",
475 )
476 )
478 # Required parameters not bound (whitelist runner-injected vars)
479 for param_name, param_spec in state.fragment_parameters.items():
480 if param_spec.required and param_name not in state.fragment_bindings:
481 if param_name in RUNNER_INJECTED:
482 continue # Available at runtime; not a static error
483 errors.append(
484 ValidationError(
485 message=(
486 f"Required parameter '{param_name}' of fragment "
487 f"'{state.fragment_name}' is not bound in 'with'"
488 ),
489 path=f"{path}.with",
490 )
491 )
493 # Statically-detectable type mismatches (skip interpolation strings)
494 for param_name, value in state.fragment_bindings.items():
495 if param_name not in state.fragment_parameters:
496 continue
497 if isinstance(value, str) and "${" in value:
498 continue
499 type_error = _check_param_type(value, state.fragment_parameters[param_name])
500 if type_error:
501 errors.append(
502 ValidationError(
503 message=f"Parameter '{param_name}': {type_error}",
504 path=f"{path}.with.{param_name}",
505 )
506 )
508 return errors
511def _validate_state_action(state_name: str, state: StateConfig) -> list[ValidationError]:
512 """Validate state action configuration.
514 Args:
515 state_name: Name of the state to validate
516 state: The state configuration to validate
518 Returns:
519 List of validation errors found
520 """
521 errors: list[ValidationError] = []
522 path = f"states.{state_name}"
524 # append_to_messages must contain at least one ${...} interpolation expression
525 if state.append_to_messages is not None:
526 if "${" not in state.append_to_messages:
527 errors.append(
528 ValidationError(
529 message=(
530 "'append_to_messages' must contain a ${...} interpolation expression "
531 f"(e.g. '${{captured.{state_name}.output}}')"
532 ),
533 path=f"{path}.append_to_messages",
534 )
535 )
537 # model: override is silently ignored for non-prompt states (host CLI is not invoked)
538 if state.model is not None and state.action_type not in ("prompt", "slash_command", None):
539 errors.append(
540 ValidationError(
541 message="model: override is ignored for shell/mcp_tool/contract states",
542 path=f"{path}.model",
543 severity=ValidationSeverity.WARNING,
544 )
545 )
547 # params field is only valid for mcp_tool states
548 if state.params and state.action_type != "mcp_tool":
549 errors.append(
550 ValidationError(
551 message="'params' field is only valid when action_type is 'mcp_tool'",
552 path=f"{path}.params",
553 )
554 )
556 # loop and action are mutually exclusive
557 if state.loop is not None and state.action is not None:
558 errors.append(
559 ValidationError(
560 message="'loop' and 'action' are mutually exclusive — "
561 "a sub-loop state cannot also have an action",
562 path=f"{path}",
563 )
564 )
566 # with: requires loop: to be set
567 if state.with_ and state.loop is None:
568 errors.append(
569 ValidationError(
570 message="'with' is only valid when 'loop' is set",
571 path=f"{path}.with",
572 )
573 )
575 # FEAT-1283: type=learning requires a populated LearningConfig
576 if state.type == "learning" and state.learning is not None:
577 if not state.learning.targets and not state.learning.targets_csv:
578 errors.append(
579 ValidationError(
580 message="type=learning requires non-empty 'learning.targets' or 'learning.targets_csv'",
581 path=f"{path}.learning.targets",
582 )
583 )
584 if state.learning.max_retries < 0:
585 errors.append(
586 ValidationError(
587 message=(
588 f"learning.max_retries must be >= 0, got {state.learning.max_retries}"
589 ),
590 path=f"{path}.learning.max_retries",
591 )
592 )
593 if state.on_yes is None:
594 errors.append(
595 ValidationError(
596 message="type=learning requires 'on_yes' (target for all-proven)",
597 path=f"{path}.on_yes",
598 )
599 )
600 if state.on_blocked is None and state.on_no is None:
601 errors.append(
602 ValidationError(
603 message=(
604 "type=learning requires 'on_blocked' or 'on_no' "
605 "(target for refuted / retries_exhausted)"
606 ),
607 path=f"{path}",
608 )
609 )
611 # with: and context_passthrough are mutually exclusive
612 if state.with_ and state.context_passthrough:
613 errors.append(
614 ValidationError(
615 message=(
616 "'with' and 'context_passthrough' are mutually exclusive — "
617 "use 'with' for explicit parameter bindings or 'context_passthrough' "
618 "for legacy bulk passthrough, not both"
619 ),
620 path=f"{path}",
621 )
622 )
624 return errors
627def _validate_state_routing(state_name: str, state: StateConfig) -> list[ValidationError]:
628 """Validate state routing configuration.
630 Checks for conflicting routing definitions (shorthand vs full route).
632 Args:
633 state_name: Name of the state to validate
634 state: The state configuration to validate
636 Returns:
637 List of validation errors/warnings found
638 """
639 errors: list[ValidationError] = []
640 path = f"states.{state_name}"
642 has_shorthand = (
643 state.on_yes is not None
644 or state.on_no is not None
645 or state.on_error is not None
646 or state.on_partial is not None
647 or state.on_blocked is not None
648 or bool(state.extra_routes)
649 )
650 has_route = state.route is not None
652 # Warn about conflicting definitions
653 if has_shorthand and has_route:
654 errors.append(
655 ValidationError(
656 message="Both shorthand routing (on_yes/on_no/on_error) "
657 "and full route table defined. Route table will take precedence.",
658 path=path,
659 severity=ValidationSeverity.WARNING,
660 )
661 )
663 # Check for no valid transition definition
664 has_next = state.next is not None
665 has_terminal = state.terminal
666 has_loop = state.loop is not None
668 if not has_shorthand and not has_route and not has_next and not has_terminal and not has_loop:
669 errors.append(
670 ValidationError(
671 message="State has no transition defined. Add routing, 'next', "
672 "or mark as 'terminal: true'",
673 path=path,
674 )
675 )
677 # Validate retry field pairing: max_retries requires on_retry_exhausted and vice versa
678 if state.max_retries is not None and state.on_retry_exhausted is None:
679 errors.append(
680 ValidationError(
681 message="'max_retries' requires 'on_retry_exhausted' to also be set",
682 path=path,
683 )
684 )
685 if state.on_retry_exhausted is not None and state.max_retries is None:
686 errors.append(
687 ValidationError(
688 message="'on_retry_exhausted' requires 'max_retries' to also be set",
689 path=path,
690 )
691 )
692 if state.max_retries is not None and state.max_retries < 1:
693 errors.append(
694 ValidationError(
695 message=f"'max_retries' must be >= 1, got {state.max_retries}",
696 path=path,
697 )
698 )
700 # Validate retryable_exit_codes: requires on_error; all codes must be positive ints
701 if state.retryable_exit_codes is not None:
702 if state.on_error is None:
703 errors.append(
704 ValidationError(
705 message="'retryable_exit_codes' requires 'on_error' to also be set",
706 path=path,
707 )
708 )
709 for code in state.retryable_exit_codes:
710 if not isinstance(code, int) or code < 1:
711 errors.append(
712 ValidationError(
713 message=(
714 f"'retryable_exit_codes' entries must be positive "
715 f"integers, got {code!r}"
716 ),
717 path=f"{path}.retryable_exit_codes",
718 )
719 )
720 break
722 # Validate rate-limit retry field pairing (mirrors max_retries/on_retry_exhausted)
723 if state.max_rate_limit_retries is not None and state.on_rate_limit_exhausted is None:
724 errors.append(
725 ValidationError(
726 message="'max_rate_limit_retries' requires 'on_rate_limit_exhausted' to also be set",
727 path=path,
728 )
729 )
730 if state.on_rate_limit_exhausted is not None and state.max_rate_limit_retries is None:
731 errors.append(
732 ValidationError(
733 message="'on_rate_limit_exhausted' requires 'max_rate_limit_retries' to also be set",
734 path=path,
735 )
736 )
737 if state.max_rate_limit_retries is not None and state.max_rate_limit_retries < 1:
738 errors.append(
739 ValidationError(
740 message=f"'max_rate_limit_retries' must be >= 1, got {state.max_rate_limit_retries}",
741 path=path,
742 )
743 )
744 if (
745 state.rate_limit_backoff_base_seconds is not None
746 and state.rate_limit_backoff_base_seconds < 1
747 ):
748 errors.append(
749 ValidationError(
750 message=(
751 f"'rate_limit_backoff_base_seconds' must be >= 1, "
752 f"got {state.rate_limit_backoff_base_seconds}"
753 ),
754 path=path,
755 )
756 )
757 if state.rate_limit_max_wait_seconds is not None and state.rate_limit_max_wait_seconds < 1:
758 errors.append(
759 ValidationError(
760 message=(
761 f"'rate_limit_max_wait_seconds' must be >= 1, "
762 f"got {state.rate_limit_max_wait_seconds}"
763 ),
764 path=path,
765 )
766 )
767 if state.rate_limit_long_wait_ladder is not None:
768 if len(state.rate_limit_long_wait_ladder) == 0:
769 errors.append(
770 ValidationError(
771 message="'rate_limit_long_wait_ladder' must be non-empty if specified",
772 path=path,
773 )
774 )
775 else:
776 for idx, value in enumerate(state.rate_limit_long_wait_ladder):
777 if not isinstance(value, int) or value < 1:
778 errors.append(
779 ValidationError(
780 message=(
781 f"'rate_limit_long_wait_ladder[{idx}]' must be a "
782 f"positive integer, got {value!r}"
783 ),
784 path=path,
785 )
786 )
788 # Validate throttle config when present
789 if state.throttle is not None:
790 t = state.throttle
791 fields = {
792 "normal_max": t.normal_max,
793 "warn_max": t.warn_max,
794 "hard_max": t.hard_max,
795 }
796 for field_name, val in fields.items():
797 if val is not None and (not isinstance(val, int) or val < 1):
798 errors.append(
799 ValidationError(
800 message=f"'throttle.{field_name}' must be a positive integer, got {val!r}",
801 path=path,
802 )
803 )
804 # Enforce ordering when all three are set
805 if t.normal_max is not None and t.warn_max is not None and t.normal_max >= t.warn_max:
806 errors.append(
807 ValidationError(
808 message=(
809 f"'throttle.normal_max' ({t.normal_max}) must be less than "
810 f"'throttle.warn_max' ({t.warn_max})"
811 ),
812 path=path,
813 )
814 )
815 if t.warn_max is not None and t.hard_max is not None and t.warn_max >= t.hard_max:
816 errors.append(
817 ValidationError(
818 message=(
819 f"'throttle.warn_max' ({t.warn_max}) must be less than "
820 f"'throttle.hard_max' ({t.hard_max})"
821 ),
822 path=path,
823 )
824 )
826 return errors
829def _validate_targets(fsm: FSMLoop) -> list[ValidationError]:
830 """Validate top-level targets[] entries (ENH-1552).
832 Rejects any targets[].states[] entry whose sibling file: value does not
833 end with a .yaml extension.
834 """
835 errors: list[ValidationError] = []
836 for i, target in enumerate(fsm.targets):
837 if target.file is not None and not target.file.endswith(".yaml"):
838 errors.append(
839 ValidationError(
840 message=(f"targets[{i}].file must be a .yaml file, got '{target.file}'"),
841 path=f"targets[{i}].file",
842 )
843 )
844 return errors
847def _validate_failure_terminal_action(fsm: FSMLoop) -> list[ValidationError]:
848 """Warn when a failure-named terminal state has no diagnostic predecessor.
850 Failure terminals (failed, error, aborted) should have at least one
851 predecessor state with an action or sub-loop that provides diagnostic
852 output before termination. Otherwise the failure is silent — the
853 executor calls _finish("terminal") before any action on the terminal
854 itself can execute.
856 Severity is WARNING (not ERROR) so that existing loops with bare
857 failure terminals continue to load, and test_terminal_only_state_valid
858 (which filters by ERROR) passes without modification.
859 """
860 FAILURE_TERMINAL_NAMES: frozenset[str] = frozenset({"failed", "error", "aborted"})
861 errors: list[ValidationError] = []
863 terminal_states = fsm.get_terminal_states()
864 failure_terminals = terminal_states & FAILURE_TERMINAL_NAMES
866 for ft_name in failure_terminals:
867 has_diagnostic_predecessor = False
868 for state_name, state in fsm.states.items():
869 if state_name == ft_name:
870 continue
871 if ft_name in state.get_referenced_states():
872 if state.action is not None or state.loop is not None:
873 has_diagnostic_predecessor = True
874 break
876 if not has_diagnostic_predecessor:
877 errors.append(
878 ValidationError(
879 message=(
880 f"Failure-named terminal state '{ft_name}' has no predecessor "
881 "state with a diagnostic action. Add a non-terminal diagnostic "
882 "state (e.g. 'diagnose') with an action or sub-loop that routes "
883 f"to '{ft_name}'."
884 ),
885 path=f"states.{ft_name}",
886 severity=ValidationSeverity.WARNING,
887 )
888 )
890 return errors
893def validate_fsm(fsm: FSMLoop) -> list[ValidationError]:
894 """Validate FSM structure and return list of errors.
896 Performs comprehensive validation:
897 - Initial state exists
898 - All referenced states exist
899 - At least one terminal state
900 - Evaluator configurations are valid
901 - Routing configurations are valid
902 - Numeric fields are in valid ranges (max_iterations > 0, backoff >= 0, timeout > 0)
904 Args:
905 fsm: The FSM loop to validate
907 Returns:
908 List of validation errors (empty if valid)
909 """
910 errors: list[ValidationError] = []
911 defined_states = fsm.get_all_state_names()
913 # Warn when no top-level description: field is set. The field is optional
914 # for FSM execution but required for goal-alignment skills (debug-loop-run,
915 # audit-loop-run) and for ll-loop show --json to surface intent text.
916 if not fsm.description:
917 errors.append(
918 ValidationError(
919 path="<root>",
920 message=("No 'description' field defined. Add a top-level description: key."),
921 severity=ValidationSeverity.WARNING,
922 )
923 )
925 # Validate parameters block
926 errors.extend(_validate_parameters(fsm))
928 # Validate targets block (ENH-1552)
929 errors.extend(_validate_targets(fsm))
931 # Check initial state exists
932 if fsm.initial not in defined_states:
933 errors.append(
934 ValidationError(
935 message=f"Initial state '{fsm.initial}' not found in states",
936 path="initial",
937 )
938 )
940 # Check at least one terminal state
941 terminal_states = fsm.get_terminal_states()
942 if not terminal_states:
943 errors.append(
944 ValidationError(
945 message="No terminal state defined. At least one state must have 'terminal: true'",
946 path="states",
947 )
948 )
950 # Validate each state
951 for state_name, state in fsm.states.items():
952 # Check all referenced states exist
953 refs = state.get_referenced_states()
954 for ref in refs:
955 # $current is a special token for retry
956 if ref != "$current" and ref not in defined_states:
957 errors.append(
958 ValidationError(
959 message=f"References unknown state '{ref}'",
960 path=f"states.{state_name}",
961 )
962 )
964 # Validate action configuration
965 errors.extend(_validate_state_action(state_name, state))
967 # Validate evaluator if present
968 if state.evaluate is not None:
969 errors.extend(_validate_evaluator(state_name, state.evaluate))
971 # Validate routing configuration
972 errors.extend(_validate_state_routing(state_name, state))
974 # Check numeric field ranges
975 if fsm.max_iterations <= 0:
976 errors.append(
977 ValidationError(
978 message=f"max_iterations must be > 0, got {fsm.max_iterations}",
979 path="max_iterations",
980 )
981 )
982 if fsm.max_edge_revisits <= 0:
983 errors.append(
984 ValidationError(
985 message=f"max_edge_revisits must be > 0, got {fsm.max_edge_revisits}",
986 path="max_edge_revisits",
987 )
988 )
989 if fsm.backoff is not None and fsm.backoff < 0:
990 errors.append(
991 ValidationError(
992 message=f"backoff must be >= 0, got {fsm.backoff}",
993 path="backoff",
994 )
995 )
996 if fsm.timeout is not None and fsm.timeout <= 0:
997 errors.append(
998 ValidationError(
999 message=f"timeout must be > 0, got {fsm.timeout}",
1000 path="timeout",
1001 )
1002 )
1003 if fsm.llm.max_tokens <= 0:
1004 errors.append(
1005 ValidationError(
1006 message=f"llm.max_tokens must be > 0, got {fsm.llm.max_tokens}",
1007 path="llm.max_tokens",
1008 )
1009 )
1010 if fsm.llm.timeout <= 0:
1011 errors.append(
1012 ValidationError(
1013 message=f"llm.timeout must be > 0, got {fsm.llm.timeout}",
1014 path="llm.timeout",
1015 )
1016 )
1018 # Check for unreachable states (warning only)
1019 reachable = _find_reachable_states(fsm)
1020 unreachable = defined_states - reachable
1021 for state_name in unreachable:
1022 errors.append(
1023 ValidationError(
1024 message="State is not reachable from initial state",
1025 path=f"states.{state_name}",
1026 severity=ValidationSeverity.WARNING,
1027 )
1028 )
1030 errors.extend(_validate_failure_terminal_action(fsm))
1032 errors.extend(_validate_meta_loop_evaluation(fsm))
1034 errors.extend(_validate_input_key_without_guard(fsm))
1036 errors.extend(_validate_artifact_isolation(fsm))
1038 errors.extend(_validate_harness_multimodal_evaluator_blind_spot(fsm))
1040 errors.extend(_validate_partial_route_dead_end(fsm))
1042 errors.extend(_validate_artifact_overwrite(fsm))
1044 errors.extend(_validate_generator_fix_discipline(fsm))
1046 errors.extend(_validate_classify_route_default(fsm))
1048 errors.extend(_validate_zero_retry_counter(fsm))
1050 errors.extend(_validate_on_max_iterations(fsm, defined_states))
1052 errors.extend(_validate_circuit(fsm, defined_states))
1054 errors.extend(_validate_progress_paths_isolation(fsm))
1056 errors.extend(_validate_capture_reachability(fsm))
1058 return errors
1061def _is_meta_loop(fsm: FSMLoop) -> bool:
1062 """Return True if fsm is classified as a meta-loop.
1064 A loop is meta if ANY of the following match:
1065 1. Any state action string matches a harness-artifact path regex
1066 (writes another loop YAML, skill, agent, command, or project config)
1067 2. The loop's import list contains lib/benchmark.yaml
1068 3. Any state action references yaml_state_editor or replace_action
1069 """
1070 # Condition 2: imports lib/benchmark.yaml
1071 if any(imp in _META_LOOP_IMPORT_TRIGGERS for imp in fsm.imports):
1072 return True
1073 # Conditions 1 and 3: scan action strings
1074 for state in fsm.states.values():
1075 if state.action is None:
1076 continue
1077 for pattern in _META_LOOP_ACTION_PATTERNS:
1078 if pattern.search(state.action):
1079 return True
1080 for token in _META_LOOP_ACTION_TOKENS:
1081 if token in state.action:
1082 return True
1083 return False
1086def _validate_meta_loop_evaluation(fsm: FSMLoop) -> list[ValidationError]:
1087 """Validate meta-loop evaluation rules MR-1 and MR-2.
1089 MR-1 (ERROR): meta-loop must have at least one non-LLM evaluator.
1090 MR-2 (WARNING): meta-loop should reference a captured baseline in an evaluator.
1092 Both rules are suppressed by ``meta_self_eval_ok: true`` at the loop top-level.
1093 """
1094 errors: list[ValidationError] = []
1095 if fsm.meta_self_eval_ok or not _is_meta_loop(fsm):
1096 return errors
1098 # Collect all evaluator types used across all states
1099 evaluator_types: set[str] = set()
1100 for state in fsm.states.values():
1101 if state.evaluate is not None:
1102 evaluator_types.add(state.evaluate.type)
1104 # MR-1: must have at least one non-LLM evaluator
1105 if not evaluator_types & NON_LLM_EVALUATOR_TYPES:
1106 errors.append(
1107 ValidationError(
1108 message=(
1109 "Loop modifies harness artifacts but has no non-LLM evaluator. "
1110 "LLM self-grades on harness updates are unreliable (SHOR Table 1: "
1111 "33-55% accuracy). Pair every check_semantic state with at least one "
1112 "of: exit_code, output_numeric, convergence, diff_stall, action_stall, mcp_result. "
1113 "Note: llm_structured and comparator both use the LLM and do not satisfy MR-1. "
1114 "To suppress with justification, set `meta_self_eval_ok: true` at the "
1115 "loop top-level."
1116 ),
1117 path="<root>",
1118 severity=ValidationSeverity.ERROR,
1119 )
1120 )
1122 # MR-2: should reference a captured baseline in a later evaluator
1123 capture_names: set[str] = {state.capture for state in fsm.states.values() if state.capture}
1124 if capture_names and not _has_baseline_reference(fsm, capture_names):
1125 errors.append(
1126 ValidationError(
1127 message=(
1128 "Meta-loop appears to lack a measure→propose→apply→re-measure "
1129 "spine: no captured baseline value is referenced by a later evaluator. "
1130 "Meta-loops should compare a post-change score against a pre-change "
1131 "baseline (see loops/harness-optimize.yaml as reference template). "
1132 "To suppress, set `meta_self_eval_ok: true`."
1133 ),
1134 path="<root>",
1135 severity=ValidationSeverity.WARNING,
1136 )
1137 )
1139 return errors
1142# Regex patterns for detecting counter-increment actions.
1143# Must contain a printf/echo writing to a file AND an arithmetic increment.
1144_COUNTER_FILE_WRITE_RE = re.compile(r"(?:printf|echo)\s+.*>")
1145_COUNTER_INCREMENT_RE = re.compile(
1146 r"\$\(\(.*\+\s*1\s*\)\)" # $((N + 1)) or $((N+1))
1147 r"|\+\+" # C-style increment
1148 r"|\+=1" # compound assignment
1149 r"|awk\s+.*\+\+" # awk with increment
1150)
1153def _validate_zero_retry_counter(fsm: FSMLoop) -> list[ValidationError]:
1154 """Detect counter + output_numeric combos that yield zero effective retries.
1156 A common loop-authoring footgun: a state increments a counter file and then
1157 evaluates ``output_numeric`` with ``operator: lt, target: 1`` against it.
1158 After the first increment the counter is 1, ``1 < 1 == false``, so the
1159 retry budget is 0 by construction. Author almost always intended target=2.
1160 """
1161 errors: list[ValidationError] = []
1163 for state_name, state in fsm.states.items():
1164 if not state.action or not state.evaluate:
1165 continue
1167 ev = state.evaluate
1168 if ev.type != "output_numeric":
1169 continue
1170 if ev.operator is None or ev.target is None:
1171 continue
1173 # Must be a number-like target for numeric comparison
1174 try:
1175 target = float(ev.target)
1176 except (ValueError, TypeError):
1177 continue
1179 if not _is_counter_action(state.action):
1180 continue
1182 # Check: after first increment (0→1), does operator(1, target) already fail?
1183 op_fn = _NUMERIC_OPERATORS.get(ev.operator)
1184 if op_fn is None:
1185 continue
1187 if not op_fn(1.0, target):
1188 suggested_target = _suggested_target(ev.operator, target)
1189 errors.append(
1190 ValidationError(
1191 message=(
1192 f"Zero retry budget: operator={ev.operator} target={target} "
1193 f"means the first post-increment value (1) already fails "
1194 f"({ev.operator}(1, {target}) == False). "
1195 f"Did you mean target={suggested_target}?"
1196 ),
1197 path=f"states.{state_name}.evaluate",
1198 severity=ValidationSeverity.WARNING,
1199 )
1200 )
1202 return errors
1205def _is_counter_action(action: str) -> bool:
1206 """Return True if the action string contains a counter-increment pattern."""
1207 return bool(_COUNTER_FILE_WRITE_RE.search(action) and _COUNTER_INCREMENT_RE.search(action))
1210def _suggested_target(operator: str, target: float) -> str:
1211 """Suggest a target value that allows at least one retry."""
1212 # For lt/le with a too-low target, suggest target+1 so first post-increment passes
1213 if operator in ("lt", "le"):
1214 return str(int(target) + 1)
1215 # For eq with target=0, suggest 1 so the counter can eventually match
1216 if operator == "eq" and target == 0:
1217 return "1"
1218 # For other cases, suggest target+1 as a default nudge
1219 return str(int(target) + 1)
1222def _validate_harness_multimodal_evaluator_blind_spot(fsm: FSMLoop) -> list[ValidationError]:
1223 """Warn when harness loops use LLM multimodal eval as sole gate to terminal.
1225 LLMs can silently fall back to text-only analysis when reading images,
1226 producing verdicts based on incomplete information. The output_contains
1227 evaluator can verify the LLM wrote the pass string but not that it
1228 actually processed the image. This is the same class of failure as MR-1
1229 (LLM self-evaluation bias) applied to artifact evaluation rather than
1230 harness modification.
1232 Suppressed by ``meta_self_eval_ok: true`` at the loop top-level.
1233 """
1234 errors: list[ValidationError] = []
1235 if fsm.meta_self_eval_ok or fsm.category != "harness":
1236 return errors
1238 terminal_states = fsm.get_terminal_states()
1240 for state_name, state in fsm.states.items():
1241 if state.action_type != "prompt" or not state.action:
1242 continue
1243 if state.evaluate is None or state.evaluate.type != "output_contains":
1244 continue
1245 if not any(p.search(state.action) for p in _MULTIMODAL_EVAL_PATTERNS):
1246 continue
1247 if state.on_yes not in terminal_states:
1248 continue
1249 errors.append(
1250 ValidationError(
1251 message=(
1252 f"State '{state_name}' evaluates a screenshot/image via LLM "
1253 "prompt and routes directly to a terminal on success. The "
1254 "output_contains evaluator can verify the LLM wrote the pass "
1255 "string but not that the LLM actually processed the image. "
1256 "Consider adding a shell-action verification state (e.g., "
1257 "functional smoke test) between scoring and the terminal."
1258 ),
1259 path=f"states.{state_name}",
1260 severity=ValidationSeverity.WARNING,
1261 )
1262 )
1264 return errors
1267def _find_shared_tmp_writes(fsm: FSMLoop) -> list[tuple[str, str]]:
1268 """Return (state_name, matched_path) for every action referencing shared .loops/tmp/.
1270 Scans `state.action` only. Prompts and sub-loop bindings can also reference
1271 paths, but those are out of static-scan reach: action strings are the
1272 place where loop YAMLs directly encode artifact paths.
1273 """
1274 findings: list[tuple[str, str]] = []
1275 for state_name, state in fsm.states.items():
1276 if not state.action:
1277 continue
1278 for match in _SHARED_TMP_PATH_RE.finditer(state.action):
1279 findings.append((state_name, match.group(0)))
1280 return findings
1283def _validate_input_key_without_guard(fsm: FSMLoop) -> list[ValidationError]:
1284 """Warn when a loop sets a custom input_key but omits required_inputs.
1286 A loop that accepts a runtime input via a named key (e.g. input_key: description)
1287 but doesn't declare required_inputs will silently proceed with an empty value if the
1288 user forgets to pass one. Declaring required_inputs shifts that failure to start-time.
1289 """
1290 if fsm.input_key == "input":
1291 return []
1292 if fsm.required_inputs:
1293 return []
1294 return [
1295 ValidationError(
1296 message=(
1297 f"Loop sets input_key: '{fsm.input_key}' but does not declare "
1298 f"required_inputs. If this input is mandatory, add "
1299 f"'required_inputs: [\"{fsm.input_key}\"]' to make the runner "
1300 f"abort when no input is provided."
1301 ),
1302 path="required_inputs",
1303 severity=ValidationSeverity.WARNING,
1304 )
1305 ]
1308def _validate_artifact_isolation(fsm: FSMLoop) -> list[ValidationError]:
1309 """Validate rule MR-3: loops must isolate artifacts to ${context.run_dir}.
1311 The runner injects ${context.run_dir} pointing at .loops/runs/<loop>-<ts>/
1312 and creates the folder before execution. Loops that write intermediate
1313 state (queues, checkpoints, generated files) to shared .loops/tmp/ paths
1314 will corrupt each other under concurrent runs (ll-parallel workers, retries,
1315 repeated invocations).
1317 Suppressed by `shared_state_ok: true` at the loop top-level for loops that
1318 intentionally share state across runs.
1319 """
1320 if fsm.shared_state_ok:
1321 return []
1322 errors: list[ValidationError] = []
1323 for state_name, path in _find_shared_tmp_writes(fsm):
1324 errors.append(
1325 ValidationError(
1326 message=(
1327 f"State writes to shared '{path}' instead of "
1328 "'${context.run_dir}/...'. Concurrent runs of this loop "
1329 "(e.g., under ll-parallel) will corrupt each other's state. "
1330 "Use the runner-injected `${context.run_dir}` for per-run "
1331 "artifact paths, or set `shared_state_ok: true` at the loop "
1332 "top-level if cross-run sharing is intentional."
1333 ),
1334 path=f"states.{state_name}.action",
1335 severity=ValidationSeverity.WARNING,
1336 )
1337 )
1338 return errors
1341def _is_llm_judged(state: StateConfig) -> bool:
1342 """Return True if this state will be graded by the default LLM judge.
1344 Mirrors the action-mode detection in executor._action_mode() (not imported
1345 here because that is runtime code). A state is LLM-judged when:
1346 - it has no explicit evaluate block AND its action is a prompt or slash_command, OR
1347 - it has an explicit evaluate block of type llm_structured or check_semantic.
1348 """
1349 if state.evaluate is None:
1350 # Heuristic: explicit action_type wins; fall back to leading "/" on action string.
1351 action_type = state.action_type
1352 if action_type in ("prompt", "slash_command"):
1353 return True
1354 if action_type is None and state.action and state.action.lstrip().startswith("/"):
1355 return True
1356 return False
1357 return state.evaluate.type in ("llm_structured", "check_semantic")
1360def _validate_partial_route_dead_end(fsm: FSMLoop) -> list[ValidationError]:
1361 """Validate rule MR-4: LLM-judged states with only on_yes have a partial/no dead-end.
1363 A state gated by the default LLM judge can receive yes/no/partial verdicts.
1364 If only on_yes is mapped (and no on_no, on_partial, next, or route table with
1365 a default exist), a partial or no verdict returns None from _route and silently
1366 terminates the loop — the parent treats this as failed.
1368 Suppressed by `partial_route_ok: true` at the loop top-level for the rare
1369 case where dead-ending on a non-yes verdict is intentional.
1370 """
1371 if fsm.partial_route_ok:
1372 return []
1373 errors: list[ValidationError] = []
1374 for state_name, state in fsm.states.items():
1375 if not _is_llm_judged(state):
1376 continue
1377 # States with an unconditional next: or a full route: table are safe.
1378 if state.next is not None or state.route is not None:
1379 continue
1380 # Only flag when on_yes is set but at least one of on_no/on_partial is missing.
1381 if state.on_yes is None:
1382 continue
1383 missing = [v for v in ("no", "partial") if getattr(state, f"on_{v}") is None]
1384 if not missing:
1385 continue
1386 unrouted = " or ".join(f"`{v}`" for v in missing)
1387 errors.append(
1388 ValidationError(
1389 message=(
1390 f"[state: {state_name}] LLM-judged prompt routes only on_yes; "
1391 f"a {unrouted} verdict has no route and will dead-end the loop "
1392 "(parent reads this as failed). Add on_no/on_partial, use `next:` "
1393 "for an unconditional handoff, or a `route:` table with a default. "
1394 "Set `partial_route_ok: true` at the loop top-level to suppress "
1395 "if intentional. (ENH-1917)"
1396 ),
1397 path=f"states.{state_name}",
1398 severity=ValidationSeverity.WARNING,
1399 )
1400 )
1401 return errors
1404def _validate_artifact_overwrite(fsm: FSMLoop) -> list[ValidationError]:
1405 """Validate rule MR-5 (ENH-1957): harness loops should version artifacts per iteration.
1407 A harness-category loop that iteratively generates and overwrites a flat artifact
1408 path (e.g. ``${context.run_dir}/image.svg``) loses all intermediate versions.
1409 Only the final iteration survives. This rule flags iterative generate→evaluate→generate
1410 cycles that write to artifact paths without declaring versioning intent.
1412 Suppressed by ``artifact_versioning: true`` (loop snapshots per-iteration artifacts)
1413 or ``artifact_versioning_ok: true`` (intentional overwrite, e.g. artifact varies
1414 by task).
1415 """
1416 if fsm.artifact_versioning or fsm.artifact_versioning_ok:
1417 return []
1418 if fsm.category not in ("harness",):
1419 return []
1421 errors: list[ValidationError] = []
1423 # Find states that write to artifact paths (shell actions with file output)
1424 writers: dict[str, set[str]] = {} # state_name -> set of artifact paths
1425 for state_name, state in fsm.states.items():
1426 if not state.action or state.action_type not in ("shell", None):
1427 continue
1428 # Skip sub-loop delegation states
1429 if state.action_type == "loop":
1430 continue
1431 action = state.action
1432 # Find run_dir-based artifact writes: ${context.run_dir}/<path> or $RUNDIR/<path>
1433 import re
1435 # Match patterns like: ${context.run_dir}/output.svg, $RUNDIR/image.png, > path
1436 # We look for output redirections or cp/mv commands writing to run_dir
1437 artifact_refs = set()
1438 for pattern in (
1439 r'\$\{context\.run_dir\}/([^\s"\';&]+)',
1440 r'\$\{captured\.[^}]+\}/([^\s"\';&]+)',
1441 ):
1442 for m in re.finditer(pattern, action):
1443 artifact_refs.add(m.group(1))
1444 # Also detect explicit cp/mv writing to run_dir paths
1445 for m in re.finditer(r'(?:cp|mv)\s+.*\s+\$\{context\.run_dir\}/([^\s"\';&]+)', action):
1446 artifact_refs.add(m.group(1))
1447 if artifact_refs:
1448 writers[state_name] = artifact_refs
1450 if not writers:
1451 return []
1453 # Detect iterative cycles: a writer state that is reachable from itself
1454 # via a non-trivial path through other states (generate → evaluate → generate)
1455 for state_name in writers:
1456 refs = fsm.states[state_name].get_referenced_states()
1457 # Check if this writer or its downstream states loop back to this writer
1458 visited: set[str] = set()
1459 to_visit = list(refs - {"$current"})
1460 while to_visit:
1461 target = to_visit.pop()
1462 if target in visited or target not in fsm.states:
1463 continue
1464 visited.add(target)
1465 if target == state_name:
1466 # Found a cycle: this writer state is reachable from itself
1467 artifact_list = ", ".join(sorted(writers[state_name]))
1468 errors.append(
1469 ValidationError(
1470 message=(
1471 f"[state: {state_name}] Harness loop writes artifact(s) "
1472 f"({artifact_list}) to a flat path in an iterative cycle "
1473 f"({state_name} → ... → {state_name}). Per-iteration versions "
1474 "are lost; only the final output survives. Add per-iteration "
1475 "snapshots (see oracle generator-evaluator for pattern) and "
1476 "declare `artifact_versioning: true`, or set "
1477 "`artifact_versioning_ok: true` if intentional. (ENH-1957)"
1478 ),
1479 path=f"states.{state_name}",
1480 severity=ValidationSeverity.WARNING,
1481 )
1482 )
1483 break
1484 target_state = fsm.states.get(target)
1485 if target_state is not None:
1486 target_refs = target_state.get_referenced_states()
1487 for r in target_refs:
1488 if r != "$current" and r not in visited:
1489 to_visit.append(r)
1491 return errors
1494def _validate_generator_fix_discipline(fsm: FSMLoop) -> list[ValidationError]:
1495 """Validate rule MR-6 (ENH-2079): meta-loops should not hand-patch generator artifacts.
1497 Detects the hand-patching anti-pattern: a ``shell``-type state that writes to the
1498 same file path as a non-shell (LLM-type) generator state in the same loop.
1499 Hand-patching creates fragile output that diverges from the generator on the next
1500 run; the stable fix is to update the generator action instead.
1502 Suppressed by ``generator_fix_ok: true`` at the loop top-level for intentional
1503 post-processing cases.
1504 """
1505 if fsm.generator_fix_ok or not _is_meta_loop(fsm):
1506 return []
1508 # Markers that indicate a prompt/slash_command state is generating file artifacts
1509 _GENERATOR_MARKERS = ("yaml_state_editor", "replace_action", "to_file:")
1511 _PATH_PATTERNS = (
1512 re.compile(r'\$\{context\.run_dir\}/([^\s"\';&|]+)'),
1513 re.compile(r'\$\{captured\.[^}]+\}/([^\s"\';&|]+)'),
1514 )
1516 def _extract_paths(action: str) -> set[str]:
1517 paths: set[str] = set()
1518 for pat in _PATH_PATTERNS:
1519 for m in pat.finditer(action):
1520 paths.add(m.group(1).rstrip("/"))
1521 return paths
1523 shell_targets: dict[str, set[str]] = {} # state_name -> set of file paths
1524 generator_targets: dict[str, set[str]] = {} # state_name -> set of file paths
1526 for state_name, state in fsm.states.items():
1527 if not state.action:
1528 continue
1529 action = state.action
1530 paths = _extract_paths(action)
1531 if not paths:
1532 continue
1533 action_type = state.action_type
1534 if action_type in ("shell", None):
1535 shell_targets[state_name] = paths
1536 elif action_type in ("prompt", "slash_command"):
1537 if any(marker in action for marker in _GENERATOR_MARKERS):
1538 generator_targets[state_name] = paths
1540 errors: list[ValidationError] = []
1541 for gen_name, gen_paths in generator_targets.items():
1542 for shell_name, shell_paths in shell_targets.items():
1543 overlap = gen_paths & shell_paths
1544 if overlap:
1545 artifact_list = ", ".join(sorted(overlap))
1546 errors.append(
1547 ValidationError(
1548 message=(
1549 f"[states: {gen_name}, {shell_name}] Hand-patching anti-pattern: "
1550 f"LLM-generator state '{gen_name}' and shell state '{shell_name}' "
1551 f"both write to ({artifact_list}). Move the fix into the generator "
1552 "action so every run produces correct output automatically. "
1553 "Set `generator_fix_ok: true` to suppress for intentional "
1554 "post-processing. (ENH-2079)"
1555 ),
1556 path=f"states.{shell_name}",
1557 severity=ValidationSeverity.WARNING,
1558 )
1559 )
1561 return errors
1564def _validate_classify_route_default(fsm: FSMLoop) -> list[ValidationError]:
1565 """Validate that classify states with a route: table include a default: fallback.
1567 A ``classify`` state whose ``route:`` table has no ``default:`` will dead-end
1568 whenever the action emits a token not listed in the table. This rule flags
1569 that gap as a WARNING so loop authors add a catch-all branch.
1571 Suppressed by ``partial_route_ok: true`` at the loop top-level when a
1572 dead-end on an unlisted token is intentional.
1573 """
1574 if fsm.partial_route_ok:
1575 return []
1576 errors: list[ValidationError] = []
1577 for state_name, state in fsm.states.items():
1578 if state.evaluate is None or state.evaluate.type != "classify":
1579 continue
1580 if state.route is None or state.route.default is not None:
1581 continue
1582 errors.append(
1583 ValidationError(
1584 message=(
1585 f"[state: {state_name}] classify route: table has no default: — "
1586 "unknown tokens will dead-end the loop. Add a default: catch-all, "
1587 "or set `partial_route_ok: true` at the loop top-level to suppress."
1588 ),
1589 path=f"states.{state_name}",
1590 severity=ValidationSeverity.WARNING,
1591 )
1592 )
1593 return errors
1596def _has_baseline_reference(fsm: FSMLoop, capture_names: set[str]) -> bool:
1597 """Return True if any evaluate block references a captured variable."""
1598 for state in fsm.states.values():
1599 ev = state.evaluate
1600 if ev is None:
1601 continue
1602 # Check string fields that may interpolate captured values
1603 candidates = [ev.previous, ev.source]
1604 if isinstance(ev.target, str):
1605 candidates.append(ev.target)
1606 for field_val in candidates:
1607 if not field_val:
1608 continue
1609 for name in capture_names:
1610 if f"captured.{name}" in field_val:
1611 return True
1612 return False
1615def _validate_on_max_iterations(fsm: FSMLoop, defined_states: set[str]) -> list[ValidationError]:
1616 """Validate the top-level `on_max_iterations` field (ENH-1631).
1618 Checks that the named state exists when `on_max_iterations` is set.
1619 """
1620 errors: list[ValidationError] = []
1621 if fsm.on_max_iterations is None:
1622 return errors
1623 if fsm.on_max_iterations not in defined_states:
1624 errors.append(
1625 ValidationError(
1626 message=(
1627 f"on_max_iterations references unknown state "
1628 f"'{fsm.on_max_iterations}' (must be a declared state)"
1629 ),
1630 path="on_max_iterations",
1631 )
1632 )
1633 return errors
1636def _validate_circuit(fsm: FSMLoop, defined_states: set[str]) -> list[ValidationError]:
1637 """Validate the top-level `circuit:` block (FEAT-1637).
1639 Checks:
1640 - `circuit.repeated_failure.window` is a positive integer.
1641 - `circuit.repeated_failure.on_repeated_failure` is either the special
1642 token ``"abort"`` or the name of a declared state.
1643 """
1644 errors: list[ValidationError] = []
1645 if fsm.circuit is None or fsm.circuit.repeated_failure is None:
1646 return errors
1648 rf = fsm.circuit.repeated_failure
1649 if rf.window < 1:
1650 errors.append(
1651 ValidationError(
1652 message=f"circuit.repeated_failure.window must be >= 1, got {rf.window}",
1653 path="circuit.repeated_failure.window",
1654 )
1655 )
1657 target = rf.on_repeated_failure
1658 if target not in STALL_SPECIAL_TOKENS and target not in defined_states:
1659 errors.append(
1660 ValidationError(
1661 message=(
1662 f"circuit.repeated_failure.on_repeated_failure references "
1663 f"unknown state '{target}' (must be a declared state or "
1664 f'the literal "abort")'
1665 ),
1666 path="circuit.repeated_failure.on_repeated_failure",
1667 )
1668 )
1670 return errors
1673# Matches common interpolation prefixes used in loop YAML paths so we can
1674# extract the portable relative component for action-string scanning.
1675_INTERPOLATION_PREFIX_RE = re.compile(r"^\$\{[^}]+\}/")
1678def _strip_interpolation_prefix(path: str) -> str:
1679 """Return the path with any leading ${...}/ prefix removed."""
1680 return _INTERPOLATION_PREFIX_RE.sub("", path)
1683def _validate_progress_paths_isolation(fsm: FSMLoop) -> list[ValidationError]:
1684 """Warn when a state's action writes to a file listed in progress_paths (BUG-1767).
1686 When a loop's own bookkeeping files appear in both progress_paths and the
1687 state action strings, every append to those files resets the stall window,
1688 silently disabling the BUG-1674 stall guard for that loop. Authors should
1689 move such files to exclude_paths so the stall detector can still fire.
1690 """
1691 if fsm.circuit is None or fsm.circuit.repeated_failure is None:
1692 return []
1693 rf = fsm.circuit.repeated_failure
1694 if not rf.progress_paths:
1695 return []
1697 # Build a set of the relative path components we need to look for.
1698 watched = {_strip_interpolation_prefix(p) for p in rf.progress_paths}
1699 # Exclude paths that are already in exclude_paths — author acknowledged.
1700 excluded = {_strip_interpolation_prefix(p) for p in rf.exclude_paths}
1701 active_watched = watched - excluded
1702 if not active_watched:
1703 return []
1705 errors: list[ValidationError] = []
1706 for state_name, state in fsm.states.items():
1707 if not state.action:
1708 continue
1709 for path_fragment in active_watched:
1710 if path_fragment in state.action:
1711 errors.append(
1712 ValidationError(
1713 message=(
1714 f"State action references '{path_fragment}', which is also "
1715 "listed in circuit.repeated_failure.progress_paths. Writes "
1716 "to this file will reset the stall window every cycle, "
1717 "silently disabling stall detection. Move it to "
1718 "circuit.repeated_failure.exclude_paths to separate "
1719 "bookkeeping files from real progress signals."
1720 ),
1721 path=f"states.{state_name}.action",
1722 severity=ValidationSeverity.WARNING,
1723 )
1724 )
1725 return errors
1728def _dominated_by_any(fsm: FSMLoop, dominators: set[str], dominated: str) -> bool:
1729 """Return True if the set ``dominators`` collectively dominates ``dominated``.
1731 Group domination: every path from the initial state to ``dominated`` must
1732 pass through at least one state in ``dominators``. Checked by removing all
1733 dominator states from the graph and testing whether ``dominated`` is still
1734 reachable from the initial state.
1736 This generalizes single-state domination — used when a capture variable is
1737 produced by more than one state on mutually-exclusive branches, where the
1738 reference is safe as long as *some* capturing state runs on every path.
1740 Args:
1741 fsm: The FSM loop to analyze
1742 dominators: Names of the states that should collectively dominate
1743 dominated: Name of the state that should be dominated
1745 Returns:
1746 True if the dominators collectively dominate ``dominated``
1747 """
1748 if dominated in dominators:
1749 return True
1750 if dominated not in fsm.states:
1751 return False
1753 visited: set[str] = set()
1754 to_visit: deque[str] = deque([fsm.initial])
1756 while to_visit:
1757 current = to_visit.popleft()
1758 if current in visited or current not in fsm.states:
1759 continue
1760 if current in dominators:
1761 continue # Block this node (simulate removal)
1763 visited.add(current)
1765 if current == dominated:
1766 # Reached dominated without going through any dominator
1767 return False
1769 state = fsm.states[current]
1770 for ref in state.get_referenced_states():
1771 if ref != "$current" and ref not in visited:
1772 to_visit.append(ref)
1774 # Dominated not reachable without the dominators → they dominate
1775 return True
1778def _dominates(fsm: FSMLoop, dominator: str, dominated: str) -> bool:
1779 """Return True if dominator dominates dominated in the FSM graph.
1781 A state D dominates S if every path from the initial state to S must pass
1782 through D. Thin single-state wrapper around :func:`_dominated_by_any`.
1783 """
1784 if dominator not in fsm.states:
1785 return False
1786 return _dominated_by_any(fsm, {dominator}, dominated)
1789def _find_bypass_path_any(fsm: FSMLoop, dominators: set[str], dominated: str) -> list[str]:
1790 """Find an example path from initial to dominated that bypasses all dominators.
1792 Uses BFS to find the shortest path that avoids every state in ``dominators``.
1793 Returns empty list if no bypass exists (should not happen when called after
1794 :func:`_dominated_by_any` returns False).
1795 """
1796 parent: dict[str, str] = {}
1797 to_visit: deque[str] = deque([fsm.initial])
1798 visited: set[str] = set()
1800 while to_visit:
1801 current = to_visit.popleft()
1802 if current in visited or current not in fsm.states:
1803 continue
1804 if current in dominators:
1805 continue
1807 visited.add(current)
1809 if current == dominated:
1810 # Reconstruct path
1811 path = [dominated]
1812 while path[-1] in parent:
1813 path.append(parent[path[-1]])
1814 path.reverse()
1815 return path
1817 state = fsm.states[current]
1818 for ref in state.get_referenced_states():
1819 if ref != "$current" and ref not in visited:
1820 if ref not in parent:
1821 parent[ref] = current
1822 to_visit.append(ref)
1824 return []
1827def _find_bypass_path(fsm: FSMLoop, dominator: str, dominated: str) -> list[str]:
1828 """Find an example path from initial to dominated that bypasses dominator.
1830 Thin single-state wrapper around :func:`_find_bypass_path_any`.
1831 """
1832 return _find_bypass_path_any(fsm, {dominator}, dominated)
1835def _has_sub_loop_state(fsm: FSMLoop) -> bool:
1836 """Return True if any state in the FSM has ``loop:`` set (delegates to a child loop).
1838 Used by ENH-1961 to distinguish "capture lives in a sub-loop" from "capture is missing".
1839 """
1840 return any(state.loop is not None for state in fsm.states.values())
1843def _validate_capture_reachability(fsm: FSMLoop) -> list[ValidationError]:
1844 """Validate that ``${captured.*}`` references are dominated by their capturing states.
1846 ENH-1961: For each state that references ``${captured.<var>.*}`` in its action
1847 or evaluate source, checks that the capturing state dominates the referencing
1848 state (i.e., all paths from the initial state pass through the capture state).
1850 Emits:
1851 - WARNING when a capture state does not dominate a referencing state
1852 (the reference may crash at runtime on paths that bypass the capture).
1853 - ERROR when a referenced capture variable has no capturing state at all
1854 in this FSM (excluding sub-loop captures which live in child namespaces).
1855 """
1856 errors: list[ValidationError] = []
1858 # Step 1: Build capture map (var_name → set of capturing state names).
1859 # A variable may be captured by more than one state on mutually-exclusive
1860 # branches (e.g. fifo_pop vs select_next dispatched by schedule_mode); the
1861 # reference is safe as long as the *set* collectively dominates it.
1862 capture_map: dict[str, set[str]] = {}
1863 for state_name, state in fsm.states.items():
1864 if state.capture:
1865 capture_map.setdefault(state.capture, set()).add(state_name)
1867 # Step 2: Build reference map (state_name → set of captured var names referenced)
1868 reference_map: dict[str, set[str]] = {}
1869 for state_name, state in fsm.states.items():
1870 # Skip sub-loop delegation states — their action is a loop name,
1871 # and captured vars belong to the child loop's namespace.
1872 if state.loop is not None:
1873 continue
1875 # Only collect references NOT guarded by `:default=` — a guarded
1876 # reference is safe even when the capture is missing on some path.
1877 refs: set[str] = set()
1878 if state.action:
1879 refs.update(_unguarded_captured_refs(state.action))
1880 if state.evaluate is not None and state.evaluate.source:
1881 refs.update(_unguarded_captured_refs(state.evaluate.source))
1882 if refs:
1883 reference_map[state_name] = refs
1885 if not reference_map:
1886 return errors
1888 # Step 3: For each reference, check dominance of capturing state
1889 for ref_state_name, ref_vars in reference_map.items():
1890 for var_name in ref_vars:
1891 if var_name not in capture_map:
1892 # Referenced capture variable has no capturing state in this FSM.
1893 # ENH-1998: downgrade to WARNING (not silent skip) when sub-loops
1894 # are present — the capture may live in a child namespace, but a
1895 # typo'd name should still surface rather than go completely dark.
1896 if _has_sub_loop_state(fsm):
1897 errors.append(
1898 ValidationError(
1899 message=(
1900 f"References ${{captured.{var_name}.*}} but no state in "
1901 f"this loop captures '{var_name}'. "
1902 f"If '{var_name}' is produced by a sub-loop, this may be "
1903 f"intentional; otherwise add 'capture: {var_name}' to the "
1904 f"state that produces this value."
1905 ),
1906 path=f"states.{ref_state_name}.action",
1907 severity=ValidationSeverity.WARNING,
1908 )
1909 )
1910 continue
1911 # No sub-loops: this is genuinely missing.
1912 errors.append(
1913 ValidationError(
1914 message=(
1915 f"References ${{captured.{var_name}.*}} but no state in "
1916 f"this loop captures '{var_name}'. Add 'capture: {var_name}' "
1917 f"to the state that produces this value."
1918 ),
1919 path=f"states.{ref_state_name}.action",
1920 severity=ValidationSeverity.ERROR,
1921 )
1922 )
1923 continue
1925 # Capturing states present in this FSM (shouldn't drop any normally)
1926 cap_states = {s for s in capture_map[var_name] if s in fsm.states}
1927 if not cap_states:
1928 continue
1930 # Group dominance check: do the capturing states collectively
1931 # dominate ref_state_name (does at least one run on every path)?
1932 if not _dominated_by_any(fsm, cap_states, ref_state_name):
1933 bypass_path = _find_bypass_path_any(fsm, cap_states, ref_state_name)
1934 path_str = " → ".join(bypass_path) if bypass_path else "unknown path"
1936 if len(cap_states) == 1:
1937 captured_by = f"state '{next(iter(cap_states))}' which may not"
1938 else:
1939 names = ", ".join(f"'{s}'" for s in sorted(cap_states))
1940 captured_by = f"states {names}, none of which"
1942 errors.append(
1943 ValidationError(
1944 message=(
1945 f"References ${{captured.{var_name}.*}} but '{var_name}' "
1946 f"is captured by {captured_by} "
1947 f"execute on all paths to '{ref_state_name}'. "
1948 f"Path(s) bypassing capture: {path_str}"
1949 ),
1950 path=f"states.{ref_state_name}.action",
1951 severity=ValidationSeverity.WARNING,
1952 )
1953 )
1955 return errors
1958def _find_reachable_states(fsm: FSMLoop) -> set[str]:
1959 """Find all states reachable from the initial state.
1961 Uses breadth-first search to find all reachable states. Seeds the BFS
1962 with the initial state plus top-level transition targets that act as
1963 alternate entry points: ``on_max_iterations`` (fires when the iteration
1964 cap is hit) and ``circuit.repeated_failure.on_repeated_failure`` (fires
1965 when the circuit breaker trips). These are real edges the runtime can
1966 take, so states reached only through them are not orphans.
1968 Args:
1969 fsm: The FSM loop to analyze
1971 Returns:
1972 Set of reachable state names
1973 """
1974 reachable: set[str] = set()
1975 to_visit: deque[str] = deque([fsm.initial])
1976 if fsm.on_max_iterations is not None:
1977 to_visit.append(fsm.on_max_iterations)
1978 if fsm.circuit is not None and fsm.circuit.repeated_failure is not None:
1979 target = fsm.circuit.repeated_failure.on_repeated_failure
1980 if target not in STALL_SPECIAL_TOKENS:
1981 to_visit.append(target)
1983 while to_visit:
1984 current = to_visit.popleft()
1985 if current in reachable or current not in fsm.states:
1986 continue
1988 reachable.add(current)
1989 state = fsm.states[current]
1990 refs = state.get_referenced_states()
1992 for ref in refs:
1993 if ref != "$current" and ref not in reachable:
1994 to_visit.append(ref)
1996 return reachable
1999def is_runnable_loop(path: Path) -> bool:
2000 """Cheap check for whether a YAML file is a runnable FSM loop definition.
2002 Returns True iff the file parses as a YAML mapping with the required
2003 top-level keys ``name``, ``initial``, and either ``states`` or ``flow``
2004 (the shorthand resolved by :func:`resolve_flow`). This matches the
2005 required-fields gate in :func:`load_and_validate` so "counted by the
2006 verifier" stays in sync with "runnable by ll-loop validate".
2008 When the raw YAML contains a ``from:`` key, inheritance is resolved first
2009 (mirroring :func:`load_and_validate`) so pure context-override stubs whose
2010 parent provides ``initial``/``states`` return True. Library fragments under
2011 ``loops/lib/`` still return False — their parent chain also lacks ``initial``.
2012 """
2013 try:
2014 data = yaml.safe_load(path.read_text())
2015 except (OSError, yaml.YAMLError):
2016 return False
2017 if not isinstance(data, dict):
2018 return False
2019 if "from" in data:
2020 try:
2021 data = resolve_inheritance(data, path.parent)
2022 except Exception:
2023 return False
2024 has_flow = "states" in data or "flow" in data
2025 return "name" in data and "initial" in data and has_flow
2028def load_and_validate(
2029 path: Path,
2030 raise_on_error: bool = True,
2031) -> tuple[FSMLoop, list[ValidationError]]:
2032 """Load YAML file and validate FSM structure.
2034 Args:
2035 path: Path to the YAML file to load
2036 raise_on_error: When True (default), raise ValueError on ERROR violations.
2037 When False, return all violations (errors + warnings) without raising.
2039 Returns:
2040 When raise_on_error=True: (FSMLoop, list of WARNING-severity ValidationErrors)
2041 When raise_on_error=False: (FSMLoop, list of all ValidationErrors sorted errors-first)
2043 Raises:
2044 FileNotFoundError: If the file doesn't exist
2045 yaml.YAMLError: If the file is not valid YAML
2046 ValueError: If raise_on_error=True and validation fails (contains error details)
2047 """
2048 if not path.exists():
2049 raise FileNotFoundError(f"FSM file not found: {path}")
2051 with open(path) as f:
2052 data: dict[str, Any] = yaml.safe_load(f)
2054 if not isinstance(data, dict):
2055 raise ValueError(f"FSM file must contain a YAML mapping, got {type(data)}")
2057 # Resolve `from:` inheritance before any further checks, so a child loop
2058 # can omit fields its parent provides (including `initial`/`states`) and
2059 # so a parent's `import:`/`fragments:` blocks survive into the merged
2060 # result for the subsequent `resolve_fragments` pass.
2061 data = resolve_inheritance(data, path.parent)
2063 # Expand flow: linear shorthand into states: before required-fields check
2064 data = resolve_flow(data)
2066 # Check required fields before parsing
2067 missing = []
2068 for field in ["name", "initial"]:
2069 if field not in data:
2070 missing.append(field)
2071 if "states" not in data:
2072 missing.append("states (or flow)")
2074 if missing:
2075 raise ValueError(f"FSM file missing required fields: {', '.join(missing)}")
2077 # Check for unknown top-level keys before parsing
2078 unknown_key_warnings: list[ValidationError] = []
2079 unknown = set(data.keys()) - KNOWN_TOP_LEVEL_KEYS
2080 if unknown:
2081 unknown_key_warnings.append(
2082 ValidationError(
2083 path="<root>",
2084 message=f"Unknown top-level keys: {', '.join(sorted(unknown))}",
2085 severity=ValidationSeverity.WARNING,
2086 )
2087 )
2089 visibility_val = data.get("visibility")
2090 if visibility_val is not None and visibility_val not in VALID_VISIBILITY:
2091 unknown_key_warnings.append(
2092 ValidationError(
2093 path="visibility",
2094 message=(
2095 f"Invalid visibility: {visibility_val!r}. "
2096 f"Must be one of: {', '.join(sorted(VALID_VISIBILITY))}. "
2097 "Loop will be treated as 'public'."
2098 ),
2099 severity=ValidationSeverity.WARNING,
2100 )
2101 )
2103 # Resolve fragment libraries before parsing into dataclass
2104 data = resolve_fragments(data, path.parent)
2106 # Parse into dataclass
2107 fsm = FSMLoop.from_dict(data)
2109 # Validate
2110 errors = validate_fsm(fsm)
2112 # Validate with: bindings against child loop parameters (requires file-system access)
2113 errors.extend(_validate_with_bindings(fsm, path.parent))
2114 errors.extend(_validate_fragment_bindings(fsm, path.parent))
2116 # Filter to errors only (not warnings) for raising
2117 error_list = [e for e in errors if e.severity == ValidationSeverity.ERROR]
2118 struct_warnings = [e for e in errors if e.severity == ValidationSeverity.WARNING]
2119 all_warnings = unknown_key_warnings + struct_warnings
2121 if not raise_on_error:
2122 return fsm, error_list + all_warnings
2124 if error_list:
2125 error_messages = "\n ".join(str(e) for e in error_list)
2126 raise ValueError(f"FSM validation failed:\n {error_messages}")
2128 for warning in all_warnings:
2129 logger.warning(str(warning))
2131 return fsm, all_warnings