Coverage for little_loops / fsm / validation.py: 11%
720 statements
« prev ^ index » next coverage.py v7.12.0, created at 2026-06-15 17:27 -0500
1"""FSM loop validation logic.
3This module provides validation for FSM loop definitions, ensuring
4structural correctness and catching common configuration errors.
6Validation checks:
7- Initial state exists in states dict
8- All referenced states exist
9- At least one terminal state
10- Evaluator configs have required fields for their type
11- No conflicting routing (shorthand vs full route)
12- Numeric fields in valid ranges (max_iterations > 0, backoff >= 0, timeout > 0)
13"""
15from __future__ import annotations
17import logging
18import re
19from collections import deque
20from dataclasses import dataclass
21from enum import Enum
22from pathlib import Path
23from typing import Any
25import yaml
27from little_loops.fsm.evaluators import _NUMERIC_OPERATORS
28from little_loops.fsm.fragments import resolve_flow, resolve_fragments, resolve_inheritance
29from little_loops.fsm.schema import EvaluateConfig, FSMLoop, ParameterSpec, StateConfig
31logger = logging.getLogger(__name__)
34class ValidationSeverity(Enum):
35 """Severity level for validation issues."""
37 ERROR = "error"
38 WARNING = "warning"
41@dataclass
42class ValidationError:
43 """Structured validation error.
45 Attributes:
46 message: Human-readable error description
47 path: Path to the problematic element (e.g., "states.check.route")
48 severity: Error severity (error or warning)
49 """
51 message: str
52 path: str | None = None
53 severity: ValidationSeverity = ValidationSeverity.ERROR
55 def __str__(self) -> str:
56 """Format error for display."""
57 prefix = f"[{self.severity.value.upper()}]"
58 if self.path:
59 return f"{prefix} {self.path}: {self.message}"
60 return f"{prefix} {self.message}"
63# Evaluator type to required fields mapping
64EVALUATOR_REQUIRED_FIELDS: dict[str, list[str]] = {
65 "exit_code": [],
66 "output_numeric": ["operator", "target"],
67 "output_json": ["path", "operator", "target"],
68 "output_contains": ["pattern"],
69 "convergence": ["target"],
70 "diff_stall": [],
71 "action_stall": [],
72 "llm_structured": [],
73 "mcp_result": [],
74 "harbor_scorer": [],
75 "comparator": ["baseline_path"],
76 "contract": ["pairs"],
77}
79# Non-LLM evaluator types: all evaluator types except llm_structured
80# Derived from EVALUATOR_REQUIRED_FIELDS so new types are automatically included
81NON_LLM_EVALUATOR_TYPES: frozenset[str] = frozenset(EVALUATOR_REQUIRED_FIELDS.keys()) - {
82 "llm_structured",
83 "comparator",
84 "contract",
85}
87# Meta-loop detector: action string patterns that indicate harness artifact writes
88_META_LOOP_ACTION_PATTERNS: tuple[re.Pattern[str], ...] = (
89 re.compile(r"loops/[\w-]+\.yaml"),
90 re.compile(r"skills/[\w-]+/SKILL\.md"),
91 re.compile(r"agents/[\w-]+\.md"),
92 re.compile(r"commands/[\w-]+\.md"),
93 re.compile(r"\.claude/(CLAUDE\.md|settings)"),
94)
96# Action string tokens that indicate meta-loop behavior
97_META_LOOP_ACTION_TOKENS: frozenset[str] = frozenset({"yaml_state_editor", "replace_action"})
99# Import paths that identify a loop as a meta-loop (harness optimization framework)
100_META_LOOP_IMPORT_TRIGGERS: frozenset[str] = frozenset({"lib/benchmark.yaml"})
102# MR-3: shared-tmp path detector. The runner injects ${context.run_dir} resolving
103# to .loops/runs/<loop>-<timestamp>/; loops that hardcode .loops/tmp/ instead
104# cause state corruption under concurrent runs (ll-parallel, retries, etc.).
105_SHARED_TMP_PATH_RE = re.compile(r"\.loops/tmp/[\w./-]+")
107# ENH-1961: Regex for extracting captured variable names from ${captured.<var>.*} references
108_CAPTURED_REF_RE = re.compile(r"\$\{captured\.(\w+)")
110# Full-reference form, capturing the var name and the remainder up to the closing
111# brace so we can detect a `:default=` guard. A reference written as
112# `${captured.x.output:default=...}` is provably safe even on paths that bypass
113# the capturing state — the interpolation engine (interpolation.py) substitutes
114# the default when the path is missing — so it must NOT be flagged by the
115# capture-reachability check. _CAPTURED_REF_RE alone can't see the guard.
116_CAPTURED_REF_FULL_RE = re.compile(r"\$\{captured\.(\w+)([^}]*)\}")
119def _unguarded_captured_refs(text: str) -> set[str]:
120 """Return captured var names that have at least one reference WITHOUT a
121 `:default=` guard. Vars referenced only via `${captured.x...:default=...}`
122 are omitted: the default makes a missing value safe, so they should not
123 trigger missing-capture or bypass-path diagnostics.
124 """
125 refs: set[str] = set()
126 for var_name, remainder in _CAPTURED_REF_FULL_RE.findall(text):
127 if ":default=" not in remainder:
128 refs.add(var_name)
129 return refs
132# ENH-1819: Regex patterns for detecting multimodal evaluation in prompt actions
133_MULTIMODAL_EVAL_PATTERNS: tuple[re.Pattern[str], ...] = (
134 re.compile(r"Read the screenshot", re.IGNORECASE),
135 re.compile(r"view the (generated )?(website|page|image)", re.IGNORECASE),
136 re.compile(r"screenshot\.(png|jpg|jpeg|webp)"),
137 re.compile(r"\.(png|jpg|jpeg|webp)\b.*\b(read|view|evaluate|score|judge)", re.IGNORECASE),
138)
140# Valid comparison operators
141VALID_OPERATORS = {"eq", "ne", "lt", "le", "gt", "ge"}
143# Valid values for the top-level `visibility:` field (audience axis for
144# `ll-loop list` filtering). "public" is the default when the field is absent.
145VALID_VISIBILITY: frozenset[str] = frozenset({"public", "internal", "example"})
147# All top-level keys recognized by FSMLoop.from_dict()
148KNOWN_TOP_LEVEL_KEYS: frozenset[str] = frozenset(
149 {
150 "name",
151 "description",
152 "initial",
153 "states",
154 "context",
155 "parameters",
156 "scope",
157 "max_iterations",
158 "on_max_iterations",
159 "max_edge_revisits",
160 "backoff",
161 "timeout",
162 "default_timeout",
163 "maintain",
164 "llm",
165 "on_handoff",
166 "input_key",
167 "required_inputs",
168 "config",
169 "category",
170 "labels",
171 "visibility",
172 "commands",
173 "targets",
174 "circuit",
175 "meta_self_eval_ok",
176 "shared_state_ok",
177 "partial_route_ok",
178 "artifact_versioning",
179 "artifact_versioning_ok",
180 "generator_fix_ok",
181 "import",
182 "fragments",
183 "from",
184 "flow",
185 "state_defs",
186 }
187)
189# Special tokens accepted as the `on_repeated_failure` target on the
190# stall detector — `"abort"` means terminate via _finish("stall_detected").
191STALL_SPECIAL_TOKENS: frozenset[str] = frozenset({"abort"})
193# Valid parameter types for the 'parameters:' block
194VALID_PARAMETER_TYPES: frozenset[str] = frozenset(
195 {"string", "integer", "number", "boolean", "enum", "path"}
196)
199def _validate_evaluator(state_name: str, evaluate: EvaluateConfig) -> list[ValidationError]:
200 """Validate evaluator configuration for type-specific requirements.
202 Args:
203 state_name: Name of the state containing this evaluator
204 evaluate: The evaluator configuration to validate
206 Returns:
207 List of validation errors found
208 """
209 errors: list[ValidationError] = []
210 path = f"states.{state_name}.evaluate"
212 # Check that evaluator type is recognized
213 valid_types = set(EVALUATOR_REQUIRED_FIELDS.keys())
214 if evaluate.type not in valid_types:
215 errors.append(
216 ValidationError(
217 message=f"Unknown evaluator type '{evaluate.type}'. "
218 f"Must be one of: {', '.join(sorted(valid_types))}",
219 path=path,
220 )
221 )
222 return errors # Can't check required fields for unknown type
224 # Check required fields for evaluator type
225 required = EVALUATOR_REQUIRED_FIELDS.get(evaluate.type, [])
226 for field_name in required:
227 value = getattr(evaluate, field_name, None)
228 if value is None:
229 errors.append(
230 ValidationError(
231 message=f"Evaluator type '{evaluate.type}' requires '{field_name}' field",
232 path=path,
233 )
234 )
236 # Validate operator if present
237 if evaluate.operator is not None and evaluate.operator not in VALID_OPERATORS:
238 errors.append(
239 ValidationError(
240 message=f"Invalid operator '{evaluate.operator}'. "
241 f"Must be one of: {', '.join(sorted(VALID_OPERATORS))}",
242 path=f"{path}.operator",
243 )
244 )
246 # Validate convergence-specific fields
247 if evaluate.type == "convergence":
248 if evaluate.direction not in ("minimize", "maximize"):
249 errors.append(
250 ValidationError(
251 message=f"Invalid direction '{evaluate.direction}'. "
252 "Must be 'minimize' or 'maximize'",
253 path=f"{path}.direction",
254 )
255 )
256 # Only validate tolerance if it's a numeric value (not an interpolation string)
257 if (
258 evaluate.tolerance is not None
259 and isinstance(evaluate.tolerance, (int, float))
260 and evaluate.tolerance < 0
261 ):
262 errors.append(
263 ValidationError(
264 message="Tolerance cannot be negative",
265 path=f"{path}.tolerance",
266 )
267 )
269 # Validate llm_structured-specific fields
270 if evaluate.type == "llm_structured":
271 if evaluate.min_confidence < 0 or evaluate.min_confidence > 1:
272 errors.append(
273 ValidationError(
274 message="min_confidence must be between 0 and 1",
275 path=f"{path}.min_confidence",
276 )
277 )
279 # Validate diff_stall-specific fields
280 if evaluate.type == "diff_stall":
281 if evaluate.max_stall < 1:
282 errors.append(
283 ValidationError(
284 message="max_stall must be >= 1",
285 path=f"{path}.max_stall",
286 )
287 )
289 # Validate action_stall-specific fields
290 if evaluate.type == "action_stall":
291 if evaluate.max_repeat < 1:
292 errors.append(
293 ValidationError(
294 message="max_repeat must be >= 1",
295 path=f"{path}.max_repeat",
296 )
297 )
299 return errors
302def _validate_parameters(fsm: FSMLoop) -> list[ValidationError]:
303 """Validate the loop's top-level parameters: block.
305 Args:
306 fsm: The FSM loop to validate
308 Returns:
309 List of validation errors found
310 """
311 errors: list[ValidationError] = []
313 for param_name, param_spec in fsm.parameters.items():
314 path = f"parameters.{param_name}"
316 if param_spec.type not in VALID_PARAMETER_TYPES:
317 errors.append(
318 ValidationError(
319 message=(
320 f"Unknown parameter type '{param_spec.type}'. "
321 f"Must be one of: {', '.join(sorted(VALID_PARAMETER_TYPES))}"
322 ),
323 path=path,
324 )
325 )
327 if param_spec.type == "enum" and not param_spec.values:
328 errors.append(
329 ValidationError(
330 message="Parameter type 'enum' requires a 'values' list",
331 path=path,
332 )
333 )
335 if param_spec.required and param_spec.default is not None:
336 errors.append(
337 ValidationError(
338 message="Parameter cannot be both 'required: true' and have a 'default' value",
339 path=path,
340 )
341 )
343 return errors
346def _check_param_type(value: Any, spec: ParameterSpec) -> str | None:
347 """Return an error message if value does not match spec.type, else None."""
348 if spec.type == "string" and not isinstance(value, str):
349 return f"expected string, got {type(value).__name__}"
350 if spec.type == "integer" and not isinstance(value, int):
351 return f"expected integer, got {type(value).__name__}"
352 if spec.type == "number" and not isinstance(value, (int, float)):
353 return f"expected number, got {type(value).__name__}"
354 if spec.type == "boolean" and not isinstance(value, bool):
355 return f"expected boolean, got {type(value).__name__}"
356 if spec.type == "enum" and spec.values and value not in spec.values:
357 return f"expected one of {spec.values!r}, got {value!r}"
358 return None
361def _validate_with_bindings(fsm: FSMLoop, loop_dir: Path) -> list[ValidationError]:
362 """Validate with: bindings against child loop parameter contracts.
364 Called from load_and_validate (not validate_fsm) because resolving child loops
365 requires file-system access via the loop directory path.
367 Args:
368 fsm: The parent FSM loop
369 loop_dir: Directory to resolve child loop paths from
371 Returns:
372 List of validation errors found
373 """
374 errors: list[ValidationError] = []
376 for state_name, state in fsm.states.items():
377 if state.loop is None or not state.with_:
378 continue
380 # Try to resolve and load the child loop; skip if unavailable
381 try:
382 from little_loops.cli.loop._helpers import resolve_loop_path
384 loop_path = resolve_loop_path(state.loop, loop_dir)
385 child_fsm, _ = load_and_validate(loop_path)
386 except Exception:
387 continue
389 if not child_fsm.parameters:
390 continue # Child has no declared contract — nothing to cross-validate
392 path = f"states.{state_name}"
394 # Unknown with: keys (not declared by child)
395 for key in state.with_:
396 if key not in child_fsm.parameters:
397 errors.append(
398 ValidationError(
399 message=(
400 f"'with.{key}' is not a declared parameter of loop '{state.loop}'. "
401 f"Declared: {', '.join(sorted(child_fsm.parameters))}"
402 ),
403 path=f"{path}.with.{key}",
404 )
405 )
407 # Required parameters not bound
408 for param_name, param_spec in child_fsm.parameters.items():
409 if param_spec.required and param_name not in state.with_:
410 errors.append(
411 ValidationError(
412 message=(
413 f"Required parameter '{param_name}' of loop '{state.loop}' "
414 f"is not bound in 'with'"
415 ),
416 path=f"{path}.with",
417 )
418 )
420 # Statically-detectable type mismatches (skip interpolation strings)
421 for param_name, value in state.with_.items():
422 if param_name not in child_fsm.parameters:
423 continue
424 if isinstance(value, str) and "${" in value:
425 continue
426 type_error = _check_param_type(value, child_fsm.parameters[param_name])
427 if type_error:
428 errors.append(
429 ValidationError(
430 message=f"Parameter '{param_name}': {type_error}",
431 path=f"{path}.with.{param_name}",
432 )
433 )
435 return errors
438def _validate_fragment_bindings(fsm: FSMLoop, loop_dir: Path) -> list[ValidationError]:
439 """Validate fragment with: bindings against fragment parameter contracts.
441 Called from load_and_validate (not validate_fsm) because fragment parameters
442 are populated by resolve_fragments which runs before dataclass parsing.
444 Args:
445 fsm: The FSM loop to validate
446 loop_dir: Directory containing the loop file (unused; kept for API symmetry with
447 _validate_with_bindings)
449 Returns:
450 List of validation errors found
451 """
452 # Runner-injected vars available at runtime but not at static analysis time
453 RUNNER_INJECTED = {"run_dir", "loop_name", "started_at", "input_hash"}
455 errors: list[ValidationError] = []
457 for state_name, state in fsm.states.items():
458 if not state.fragment_parameters:
459 continue # No declared contract — nothing to cross-validate
461 path = f"states.{state_name}"
463 # Unknown with: keys (not declared by fragment)
464 for key in state.fragment_bindings:
465 if key not in state.fragment_parameters:
466 errors.append(
467 ValidationError(
468 message=(
469 f"'with.{key}' is not a declared parameter of fragment "
470 f"'{state.fragment_name}'. "
471 f"Declared: {', '.join(sorted(state.fragment_parameters))}"
472 ),
473 path=f"{path}.with.{key}",
474 )
475 )
477 # Required parameters not bound (whitelist runner-injected vars)
478 for param_name, param_spec in state.fragment_parameters.items():
479 if param_spec.required and param_name not in state.fragment_bindings:
480 if param_name in RUNNER_INJECTED:
481 continue # Available at runtime; not a static error
482 errors.append(
483 ValidationError(
484 message=(
485 f"Required parameter '{param_name}' of fragment "
486 f"'{state.fragment_name}' is not bound in 'with'"
487 ),
488 path=f"{path}.with",
489 )
490 )
492 # Statically-detectable type mismatches (skip interpolation strings)
493 for param_name, value in state.fragment_bindings.items():
494 if param_name not in state.fragment_parameters:
495 continue
496 if isinstance(value, str) and "${" in value:
497 continue
498 type_error = _check_param_type(value, state.fragment_parameters[param_name])
499 if type_error:
500 errors.append(
501 ValidationError(
502 message=f"Parameter '{param_name}': {type_error}",
503 path=f"{path}.with.{param_name}",
504 )
505 )
507 return errors
510def _validate_state_action(state_name: str, state: StateConfig) -> list[ValidationError]:
511 """Validate state action configuration.
513 Args:
514 state_name: Name of the state to validate
515 state: The state configuration to validate
517 Returns:
518 List of validation errors found
519 """
520 errors: list[ValidationError] = []
521 path = f"states.{state_name}"
523 # append_to_messages must contain at least one ${...} interpolation expression
524 if state.append_to_messages is not None:
525 if "${" not in state.append_to_messages:
526 errors.append(
527 ValidationError(
528 message=(
529 "'append_to_messages' must contain a ${...} interpolation expression "
530 f"(e.g. '${{captured.{state_name}.output}}')"
531 ),
532 path=f"{path}.append_to_messages",
533 )
534 )
536 # model: override is silently ignored for non-prompt states (host CLI is not invoked)
537 if state.model is not None and state.action_type not in ("prompt", "slash_command", None):
538 errors.append(
539 ValidationError(
540 message="model: override is ignored for shell/mcp_tool/contract states",
541 path=f"{path}.model",
542 severity=ValidationSeverity.WARNING,
543 )
544 )
546 # params field is only valid for mcp_tool states
547 if state.params and state.action_type != "mcp_tool":
548 errors.append(
549 ValidationError(
550 message="'params' field is only valid when action_type is 'mcp_tool'",
551 path=f"{path}.params",
552 )
553 )
555 # loop and action are mutually exclusive
556 if state.loop is not None and state.action is not None:
557 errors.append(
558 ValidationError(
559 message="'loop' and 'action' are mutually exclusive — "
560 "a sub-loop state cannot also have an action",
561 path=f"{path}",
562 )
563 )
565 # with: requires loop: to be set
566 if state.with_ and state.loop is None:
567 errors.append(
568 ValidationError(
569 message="'with' is only valid when 'loop' is set",
570 path=f"{path}.with",
571 )
572 )
574 # FEAT-1283: type=learning requires a populated LearningConfig
575 if state.type == "learning" and state.learning is not None:
576 if not state.learning.targets and not state.learning.targets_csv:
577 errors.append(
578 ValidationError(
579 message="type=learning requires non-empty 'learning.targets' or 'learning.targets_csv'",
580 path=f"{path}.learning.targets",
581 )
582 )
583 if state.learning.max_retries < 0:
584 errors.append(
585 ValidationError(
586 message=(
587 f"learning.max_retries must be >= 0, got {state.learning.max_retries}"
588 ),
589 path=f"{path}.learning.max_retries",
590 )
591 )
592 if state.on_yes is None:
593 errors.append(
594 ValidationError(
595 message="type=learning requires 'on_yes' (target for all-proven)",
596 path=f"{path}.on_yes",
597 )
598 )
599 if state.on_blocked is None and state.on_no is None:
600 errors.append(
601 ValidationError(
602 message=(
603 "type=learning requires 'on_blocked' or 'on_no' "
604 "(target for refuted / retries_exhausted)"
605 ),
606 path=f"{path}",
607 )
608 )
610 # with: and context_passthrough are mutually exclusive
611 if state.with_ and state.context_passthrough:
612 errors.append(
613 ValidationError(
614 message=(
615 "'with' and 'context_passthrough' are mutually exclusive — "
616 "use 'with' for explicit parameter bindings or 'context_passthrough' "
617 "for legacy bulk passthrough, not both"
618 ),
619 path=f"{path}",
620 )
621 )
623 return errors
626def _validate_state_routing(state_name: str, state: StateConfig) -> list[ValidationError]:
627 """Validate state routing configuration.
629 Checks for conflicting routing definitions (shorthand vs full route).
631 Args:
632 state_name: Name of the state to validate
633 state: The state configuration to validate
635 Returns:
636 List of validation errors/warnings found
637 """
638 errors: list[ValidationError] = []
639 path = f"states.{state_name}"
641 has_shorthand = (
642 state.on_yes is not None
643 or state.on_no is not None
644 or state.on_error is not None
645 or state.on_partial is not None
646 or state.on_blocked is not None
647 or bool(state.extra_routes)
648 )
649 has_route = state.route is not None
651 # Warn about conflicting definitions
652 if has_shorthand and has_route:
653 errors.append(
654 ValidationError(
655 message="Both shorthand routing (on_yes/on_no/on_error) "
656 "and full route table defined. Route table will take precedence.",
657 path=path,
658 severity=ValidationSeverity.WARNING,
659 )
660 )
662 # Check for no valid transition definition
663 has_next = state.next is not None
664 has_terminal = state.terminal
665 has_loop = state.loop is not None
667 if not has_shorthand and not has_route and not has_next and not has_terminal and not has_loop:
668 errors.append(
669 ValidationError(
670 message="State has no transition defined. Add routing, 'next', "
671 "or mark as 'terminal: true'",
672 path=path,
673 )
674 )
676 # Validate retry field pairing: max_retries requires on_retry_exhausted and vice versa
677 if state.max_retries is not None and state.on_retry_exhausted is None:
678 errors.append(
679 ValidationError(
680 message="'max_retries' requires 'on_retry_exhausted' to also be set",
681 path=path,
682 )
683 )
684 if state.on_retry_exhausted is not None and state.max_retries is None:
685 errors.append(
686 ValidationError(
687 message="'on_retry_exhausted' requires 'max_retries' to also be set",
688 path=path,
689 )
690 )
691 if state.max_retries is not None and state.max_retries < 1:
692 errors.append(
693 ValidationError(
694 message=f"'max_retries' must be >= 1, got {state.max_retries}",
695 path=path,
696 )
697 )
699 # Validate retryable_exit_codes: requires on_error; all codes must be positive ints
700 if state.retryable_exit_codes is not None:
701 if state.on_error is None:
702 errors.append(
703 ValidationError(
704 message="'retryable_exit_codes' requires 'on_error' to also be set",
705 path=path,
706 )
707 )
708 for code in state.retryable_exit_codes:
709 if not isinstance(code, int) or code < 1:
710 errors.append(
711 ValidationError(
712 message=(
713 f"'retryable_exit_codes' entries must be positive "
714 f"integers, got {code!r}"
715 ),
716 path=f"{path}.retryable_exit_codes",
717 )
718 )
719 break
721 # Validate rate-limit retry field pairing (mirrors max_retries/on_retry_exhausted)
722 if state.max_rate_limit_retries is not None and state.on_rate_limit_exhausted is None:
723 errors.append(
724 ValidationError(
725 message="'max_rate_limit_retries' requires 'on_rate_limit_exhausted' to also be set",
726 path=path,
727 )
728 )
729 if state.on_rate_limit_exhausted is not None and state.max_rate_limit_retries is None:
730 errors.append(
731 ValidationError(
732 message="'on_rate_limit_exhausted' requires 'max_rate_limit_retries' to also be set",
733 path=path,
734 )
735 )
736 if state.max_rate_limit_retries is not None and state.max_rate_limit_retries < 1:
737 errors.append(
738 ValidationError(
739 message=f"'max_rate_limit_retries' must be >= 1, got {state.max_rate_limit_retries}",
740 path=path,
741 )
742 )
743 if (
744 state.rate_limit_backoff_base_seconds is not None
745 and state.rate_limit_backoff_base_seconds < 1
746 ):
747 errors.append(
748 ValidationError(
749 message=(
750 f"'rate_limit_backoff_base_seconds' must be >= 1, "
751 f"got {state.rate_limit_backoff_base_seconds}"
752 ),
753 path=path,
754 )
755 )
756 if state.rate_limit_max_wait_seconds is not None and state.rate_limit_max_wait_seconds < 1:
757 errors.append(
758 ValidationError(
759 message=(
760 f"'rate_limit_max_wait_seconds' must be >= 1, "
761 f"got {state.rate_limit_max_wait_seconds}"
762 ),
763 path=path,
764 )
765 )
766 if state.rate_limit_long_wait_ladder is not None:
767 if len(state.rate_limit_long_wait_ladder) == 0:
768 errors.append(
769 ValidationError(
770 message="'rate_limit_long_wait_ladder' must be non-empty if specified",
771 path=path,
772 )
773 )
774 else:
775 for idx, value in enumerate(state.rate_limit_long_wait_ladder):
776 if not isinstance(value, int) or value < 1:
777 errors.append(
778 ValidationError(
779 message=(
780 f"'rate_limit_long_wait_ladder[{idx}]' must be a "
781 f"positive integer, got {value!r}"
782 ),
783 path=path,
784 )
785 )
787 # Validate throttle config when present
788 if state.throttle is not None:
789 t = state.throttle
790 fields = {
791 "normal_max": t.normal_max,
792 "warn_max": t.warn_max,
793 "hard_max": t.hard_max,
794 }
795 for field_name, val in fields.items():
796 if val is not None and (not isinstance(val, int) or val < 1):
797 errors.append(
798 ValidationError(
799 message=f"'throttle.{field_name}' must be a positive integer, got {val!r}",
800 path=path,
801 )
802 )
803 # Enforce ordering when all three are set
804 if t.normal_max is not None and t.warn_max is not None and t.normal_max >= t.warn_max:
805 errors.append(
806 ValidationError(
807 message=(
808 f"'throttle.normal_max' ({t.normal_max}) must be less than "
809 f"'throttle.warn_max' ({t.warn_max})"
810 ),
811 path=path,
812 )
813 )
814 if t.warn_max is not None and t.hard_max is not None and t.warn_max >= t.hard_max:
815 errors.append(
816 ValidationError(
817 message=(
818 f"'throttle.warn_max' ({t.warn_max}) must be less than "
819 f"'throttle.hard_max' ({t.hard_max})"
820 ),
821 path=path,
822 )
823 )
825 return errors
828def _validate_targets(fsm: FSMLoop) -> list[ValidationError]:
829 """Validate top-level targets[] entries (ENH-1552).
831 Rejects any targets[].states[] entry whose sibling file: value does not
832 end with a .yaml extension.
833 """
834 errors: list[ValidationError] = []
835 for i, target in enumerate(fsm.targets):
836 if target.file is not None and not target.file.endswith(".yaml"):
837 errors.append(
838 ValidationError(
839 message=(f"targets[{i}].file must be a .yaml file, got '{target.file}'"),
840 path=f"targets[{i}].file",
841 )
842 )
843 return errors
846def _validate_failure_terminal_action(fsm: FSMLoop) -> list[ValidationError]:
847 """Warn when a failure-named terminal state has no diagnostic predecessor.
849 Failure terminals (failed, error, aborted) should have at least one
850 predecessor state with an action or sub-loop that provides diagnostic
851 output before termination. Otherwise the failure is silent — the
852 executor calls _finish("terminal") before any action on the terminal
853 itself can execute.
855 Severity is WARNING (not ERROR) so that existing loops with bare
856 failure terminals continue to load, and test_terminal_only_state_valid
857 (which filters by ERROR) passes without modification.
858 """
859 FAILURE_TERMINAL_NAMES: frozenset[str] = frozenset({"failed", "error", "aborted"})
860 errors: list[ValidationError] = []
862 terminal_states = fsm.get_terminal_states()
863 failure_terminals = terminal_states & FAILURE_TERMINAL_NAMES
865 for ft_name in failure_terminals:
866 has_diagnostic_predecessor = False
867 for state_name, state in fsm.states.items():
868 if state_name == ft_name:
869 continue
870 if ft_name in state.get_referenced_states():
871 if state.action is not None or state.loop is not None:
872 has_diagnostic_predecessor = True
873 break
875 if not has_diagnostic_predecessor:
876 errors.append(
877 ValidationError(
878 message=(
879 f"Failure-named terminal state '{ft_name}' has no predecessor "
880 "state with a diagnostic action. Add a non-terminal diagnostic "
881 "state (e.g. 'diagnose') with an action or sub-loop that routes "
882 f"to '{ft_name}'."
883 ),
884 path=f"states.{ft_name}",
885 severity=ValidationSeverity.WARNING,
886 )
887 )
889 return errors
892def validate_fsm(fsm: FSMLoop) -> list[ValidationError]:
893 """Validate FSM structure and return list of errors.
895 Performs comprehensive validation:
896 - Initial state exists
897 - All referenced states exist
898 - At least one terminal state
899 - Evaluator configurations are valid
900 - Routing configurations are valid
901 - Numeric fields are in valid ranges (max_iterations > 0, backoff >= 0, timeout > 0)
903 Args:
904 fsm: The FSM loop to validate
906 Returns:
907 List of validation errors (empty if valid)
908 """
909 errors: list[ValidationError] = []
910 defined_states = fsm.get_all_state_names()
912 # Warn when no top-level description: field is set. The field is optional
913 # for FSM execution but required for goal-alignment skills (debug-loop-run,
914 # audit-loop-run) and for ll-loop show --json to surface intent text.
915 if not fsm.description:
916 errors.append(
917 ValidationError(
918 path="<root>",
919 message=("No 'description' field defined. Add a top-level description: key."),
920 severity=ValidationSeverity.WARNING,
921 )
922 )
924 # Validate parameters block
925 errors.extend(_validate_parameters(fsm))
927 # Validate targets block (ENH-1552)
928 errors.extend(_validate_targets(fsm))
930 # Check initial state exists
931 if fsm.initial not in defined_states:
932 errors.append(
933 ValidationError(
934 message=f"Initial state '{fsm.initial}' not found in states",
935 path="initial",
936 )
937 )
939 # Check at least one terminal state
940 terminal_states = fsm.get_terminal_states()
941 if not terminal_states:
942 errors.append(
943 ValidationError(
944 message="No terminal state defined. At least one state must have 'terminal: true'",
945 path="states",
946 )
947 )
949 # Validate each state
950 for state_name, state in fsm.states.items():
951 # Check all referenced states exist
952 refs = state.get_referenced_states()
953 for ref in refs:
954 # $current is a special token for retry
955 if ref != "$current" and ref not in defined_states:
956 errors.append(
957 ValidationError(
958 message=f"References unknown state '{ref}'",
959 path=f"states.{state_name}",
960 )
961 )
963 # Validate action configuration
964 errors.extend(_validate_state_action(state_name, state))
966 # Validate evaluator if present
967 if state.evaluate is not None:
968 errors.extend(_validate_evaluator(state_name, state.evaluate))
970 # Validate routing configuration
971 errors.extend(_validate_state_routing(state_name, state))
973 # Check numeric field ranges
974 if fsm.max_iterations <= 0:
975 errors.append(
976 ValidationError(
977 message=f"max_iterations must be > 0, got {fsm.max_iterations}",
978 path="max_iterations",
979 )
980 )
981 if fsm.max_edge_revisits <= 0:
982 errors.append(
983 ValidationError(
984 message=f"max_edge_revisits must be > 0, got {fsm.max_edge_revisits}",
985 path="max_edge_revisits",
986 )
987 )
988 if fsm.backoff is not None and fsm.backoff < 0:
989 errors.append(
990 ValidationError(
991 message=f"backoff must be >= 0, got {fsm.backoff}",
992 path="backoff",
993 )
994 )
995 if fsm.timeout is not None and fsm.timeout <= 0:
996 errors.append(
997 ValidationError(
998 message=f"timeout must be > 0, got {fsm.timeout}",
999 path="timeout",
1000 )
1001 )
1002 if fsm.llm.max_tokens <= 0:
1003 errors.append(
1004 ValidationError(
1005 message=f"llm.max_tokens must be > 0, got {fsm.llm.max_tokens}",
1006 path="llm.max_tokens",
1007 )
1008 )
1009 if fsm.llm.timeout <= 0:
1010 errors.append(
1011 ValidationError(
1012 message=f"llm.timeout must be > 0, got {fsm.llm.timeout}",
1013 path="llm.timeout",
1014 )
1015 )
1017 # Check for unreachable states (warning only)
1018 reachable = _find_reachable_states(fsm)
1019 unreachable = defined_states - reachable
1020 for state_name in unreachable:
1021 errors.append(
1022 ValidationError(
1023 message="State is not reachable from initial state",
1024 path=f"states.{state_name}",
1025 severity=ValidationSeverity.WARNING,
1026 )
1027 )
1029 errors.extend(_validate_failure_terminal_action(fsm))
1031 errors.extend(_validate_meta_loop_evaluation(fsm))
1033 errors.extend(_validate_input_key_without_guard(fsm))
1035 errors.extend(_validate_artifact_isolation(fsm))
1037 errors.extend(_validate_harness_multimodal_evaluator_blind_spot(fsm))
1039 errors.extend(_validate_partial_route_dead_end(fsm))
1041 errors.extend(_validate_artifact_overwrite(fsm))
1043 errors.extend(_validate_generator_fix_discipline(fsm))
1045 errors.extend(_validate_zero_retry_counter(fsm))
1047 errors.extend(_validate_on_max_iterations(fsm, defined_states))
1049 errors.extend(_validate_circuit(fsm, defined_states))
1051 errors.extend(_validate_progress_paths_isolation(fsm))
1053 errors.extend(_validate_capture_reachability(fsm))
1055 return errors
1058def _is_meta_loop(fsm: FSMLoop) -> bool:
1059 """Return True if fsm is classified as a meta-loop.
1061 A loop is meta if ANY of the following match:
1062 1. Any state action string matches a harness-artifact path regex
1063 (writes another loop YAML, skill, agent, command, or project config)
1064 2. The loop's import list contains lib/benchmark.yaml
1065 3. Any state action references yaml_state_editor or replace_action
1066 """
1067 # Condition 2: imports lib/benchmark.yaml
1068 if any(imp in _META_LOOP_IMPORT_TRIGGERS for imp in fsm.imports):
1069 return True
1070 # Conditions 1 and 3: scan action strings
1071 for state in fsm.states.values():
1072 if state.action is None:
1073 continue
1074 for pattern in _META_LOOP_ACTION_PATTERNS:
1075 if pattern.search(state.action):
1076 return True
1077 for token in _META_LOOP_ACTION_TOKENS:
1078 if token in state.action:
1079 return True
1080 return False
1083def _validate_meta_loop_evaluation(fsm: FSMLoop) -> list[ValidationError]:
1084 """Validate meta-loop evaluation rules MR-1 and MR-2.
1086 MR-1 (ERROR): meta-loop must have at least one non-LLM evaluator.
1087 MR-2 (WARNING): meta-loop should reference a captured baseline in an evaluator.
1089 Both rules are suppressed by ``meta_self_eval_ok: true`` at the loop top-level.
1090 """
1091 errors: list[ValidationError] = []
1092 if fsm.meta_self_eval_ok or not _is_meta_loop(fsm):
1093 return errors
1095 # Collect all evaluator types used across all states
1096 evaluator_types: set[str] = set()
1097 for state in fsm.states.values():
1098 if state.evaluate is not None:
1099 evaluator_types.add(state.evaluate.type)
1101 # MR-1: must have at least one non-LLM evaluator
1102 if not evaluator_types & NON_LLM_EVALUATOR_TYPES:
1103 errors.append(
1104 ValidationError(
1105 message=(
1106 "Loop modifies harness artifacts but has no non-LLM evaluator. "
1107 "LLM self-grades on harness updates are unreliable (SHOR Table 1: "
1108 "33-55% accuracy). Pair every check_semantic state with at least one "
1109 "of: exit_code, output_numeric, convergence, diff_stall, action_stall, mcp_result. "
1110 "Note: llm_structured and comparator both use the LLM and do not satisfy MR-1. "
1111 "To suppress with justification, set `meta_self_eval_ok: true` at the "
1112 "loop top-level."
1113 ),
1114 path="<root>",
1115 severity=ValidationSeverity.ERROR,
1116 )
1117 )
1119 # MR-2: should reference a captured baseline in a later evaluator
1120 capture_names: set[str] = {state.capture for state in fsm.states.values() if state.capture}
1121 if capture_names and not _has_baseline_reference(fsm, capture_names):
1122 errors.append(
1123 ValidationError(
1124 message=(
1125 "Meta-loop appears to lack a measure→propose→apply→re-measure "
1126 "spine: no captured baseline value is referenced by a later evaluator. "
1127 "Meta-loops should compare a post-change score against a pre-change "
1128 "baseline (see loops/harness-optimize.yaml as reference template). "
1129 "To suppress, set `meta_self_eval_ok: true`."
1130 ),
1131 path="<root>",
1132 severity=ValidationSeverity.WARNING,
1133 )
1134 )
1136 return errors
1139# Regex patterns for detecting counter-increment actions.
1140# Must contain a printf/echo writing to a file AND an arithmetic increment.
1141_COUNTER_FILE_WRITE_RE = re.compile(r"(?:printf|echo)\s+.*>")
1142_COUNTER_INCREMENT_RE = re.compile(
1143 r"\$\(\(.*\+\s*1\s*\)\)" # $((N + 1)) or $((N+1))
1144 r"|\+\+" # C-style increment
1145 r"|\+=1" # compound assignment
1146 r"|awk\s+.*\+\+" # awk with increment
1147)
1150def _validate_zero_retry_counter(fsm: FSMLoop) -> list[ValidationError]:
1151 """Detect counter + output_numeric combos that yield zero effective retries.
1153 A common loop-authoring footgun: a state increments a counter file and then
1154 evaluates ``output_numeric`` with ``operator: lt, target: 1`` against it.
1155 After the first increment the counter is 1, ``1 < 1 == false``, so the
1156 retry budget is 0 by construction. Author almost always intended target=2.
1157 """
1158 errors: list[ValidationError] = []
1160 for state_name, state in fsm.states.items():
1161 if not state.action or not state.evaluate:
1162 continue
1164 ev = state.evaluate
1165 if ev.type != "output_numeric":
1166 continue
1167 if ev.operator is None or ev.target is None:
1168 continue
1170 # Must be a number-like target for numeric comparison
1171 try:
1172 target = float(ev.target)
1173 except (ValueError, TypeError):
1174 continue
1176 if not _is_counter_action(state.action):
1177 continue
1179 # Check: after first increment (0→1), does operator(1, target) already fail?
1180 op_fn = _NUMERIC_OPERATORS.get(ev.operator)
1181 if op_fn is None:
1182 continue
1184 if not op_fn(1.0, target):
1185 suggested_target = _suggested_target(ev.operator, target)
1186 errors.append(
1187 ValidationError(
1188 message=(
1189 f"Zero retry budget: operator={ev.operator} target={target} "
1190 f"means the first post-increment value (1) already fails "
1191 f"({ev.operator}(1, {target}) == False). "
1192 f"Did you mean target={suggested_target}?"
1193 ),
1194 path=f"states.{state_name}.evaluate",
1195 severity=ValidationSeverity.WARNING,
1196 )
1197 )
1199 return errors
1202def _is_counter_action(action: str) -> bool:
1203 """Return True if the action string contains a counter-increment pattern."""
1204 return bool(_COUNTER_FILE_WRITE_RE.search(action) and _COUNTER_INCREMENT_RE.search(action))
1207def _suggested_target(operator: str, target: float) -> str:
1208 """Suggest a target value that allows at least one retry."""
1209 # For lt/le with a too-low target, suggest target+1 so first post-increment passes
1210 if operator in ("lt", "le"):
1211 return str(int(target) + 1)
1212 # For eq with target=0, suggest 1 so the counter can eventually match
1213 if operator == "eq" and target == 0:
1214 return "1"
1215 # For other cases, suggest target+1 as a default nudge
1216 return str(int(target) + 1)
1219def _validate_harness_multimodal_evaluator_blind_spot(fsm: FSMLoop) -> list[ValidationError]:
1220 """Warn when harness loops use LLM multimodal eval as sole gate to terminal.
1222 LLMs can silently fall back to text-only analysis when reading images,
1223 producing verdicts based on incomplete information. The output_contains
1224 evaluator can verify the LLM wrote the pass string but not that it
1225 actually processed the image. This is the same class of failure as MR-1
1226 (LLM self-evaluation bias) applied to artifact evaluation rather than
1227 harness modification.
1229 Suppressed by ``meta_self_eval_ok: true`` at the loop top-level.
1230 """
1231 errors: list[ValidationError] = []
1232 if fsm.meta_self_eval_ok or fsm.category != "harness":
1233 return errors
1235 terminal_states = fsm.get_terminal_states()
1237 for state_name, state in fsm.states.items():
1238 if state.action_type != "prompt" or not state.action:
1239 continue
1240 if state.evaluate is None or state.evaluate.type != "output_contains":
1241 continue
1242 if not any(p.search(state.action) for p in _MULTIMODAL_EVAL_PATTERNS):
1243 continue
1244 if state.on_yes not in terminal_states:
1245 continue
1246 errors.append(
1247 ValidationError(
1248 message=(
1249 f"State '{state_name}' evaluates a screenshot/image via LLM "
1250 "prompt and routes directly to a terminal on success. The "
1251 "output_contains evaluator can verify the LLM wrote the pass "
1252 "string but not that the LLM actually processed the image. "
1253 "Consider adding a shell-action verification state (e.g., "
1254 "functional smoke test) between scoring and the terminal."
1255 ),
1256 path=f"states.{state_name}",
1257 severity=ValidationSeverity.WARNING,
1258 )
1259 )
1261 return errors
1264def _find_shared_tmp_writes(fsm: FSMLoop) -> list[tuple[str, str]]:
1265 """Return (state_name, matched_path) for every action referencing shared .loops/tmp/.
1267 Scans `state.action` only. Prompts and sub-loop bindings can also reference
1268 paths, but those are out of static-scan reach: action strings are the
1269 place where loop YAMLs directly encode artifact paths.
1270 """
1271 findings: list[tuple[str, str]] = []
1272 for state_name, state in fsm.states.items():
1273 if not state.action:
1274 continue
1275 for match in _SHARED_TMP_PATH_RE.finditer(state.action):
1276 findings.append((state_name, match.group(0)))
1277 return findings
1280def _validate_input_key_without_guard(fsm: FSMLoop) -> list[ValidationError]:
1281 """Warn when a loop sets a custom input_key but omits required_inputs.
1283 A loop that accepts a runtime input via a named key (e.g. input_key: description)
1284 but doesn't declare required_inputs will silently proceed with an empty value if the
1285 user forgets to pass one. Declaring required_inputs shifts that failure to start-time.
1286 """
1287 if fsm.input_key == "input":
1288 return []
1289 if fsm.required_inputs:
1290 return []
1291 return [
1292 ValidationError(
1293 message=(
1294 f"Loop sets input_key: '{fsm.input_key}' but does not declare "
1295 f"required_inputs. If this input is mandatory, add "
1296 f"'required_inputs: [\"{fsm.input_key}\"]' to make the runner "
1297 f"abort when no input is provided."
1298 ),
1299 path="required_inputs",
1300 severity=ValidationSeverity.WARNING,
1301 )
1302 ]
1305def _validate_artifact_isolation(fsm: FSMLoop) -> list[ValidationError]:
1306 """Validate rule MR-3: loops must isolate artifacts to ${context.run_dir}.
1308 The runner injects ${context.run_dir} pointing at .loops/runs/<loop>-<ts>/
1309 and creates the folder before execution. Loops that write intermediate
1310 state (queues, checkpoints, generated files) to shared .loops/tmp/ paths
1311 will corrupt each other under concurrent runs (ll-parallel workers, retries,
1312 repeated invocations).
1314 Suppressed by `shared_state_ok: true` at the loop top-level for loops that
1315 intentionally share state across runs.
1316 """
1317 if fsm.shared_state_ok:
1318 return []
1319 errors: list[ValidationError] = []
1320 for state_name, path in _find_shared_tmp_writes(fsm):
1321 errors.append(
1322 ValidationError(
1323 message=(
1324 f"State writes to shared '{path}' instead of "
1325 "'${context.run_dir}/...'. Concurrent runs of this loop "
1326 "(e.g., under ll-parallel) will corrupt each other's state. "
1327 "Use the runner-injected `${context.run_dir}` for per-run "
1328 "artifact paths, or set `shared_state_ok: true` at the loop "
1329 "top-level if cross-run sharing is intentional."
1330 ),
1331 path=f"states.{state_name}.action",
1332 severity=ValidationSeverity.WARNING,
1333 )
1334 )
1335 return errors
1338def _is_llm_judged(state: StateConfig) -> bool:
1339 """Return True if this state will be graded by the default LLM judge.
1341 Mirrors the action-mode detection in executor._action_mode() (not imported
1342 here because that is runtime code). A state is LLM-judged when:
1343 - it has no explicit evaluate block AND its action is a prompt or slash_command, OR
1344 - it has an explicit evaluate block of type llm_structured or check_semantic.
1345 """
1346 if state.evaluate is None:
1347 # Heuristic: explicit action_type wins; fall back to leading "/" on action string.
1348 action_type = state.action_type
1349 if action_type in ("prompt", "slash_command"):
1350 return True
1351 if action_type is None and state.action and state.action.lstrip().startswith("/"):
1352 return True
1353 return False
1354 return state.evaluate.type in ("llm_structured", "check_semantic")
1357def _validate_partial_route_dead_end(fsm: FSMLoop) -> list[ValidationError]:
1358 """Validate rule MR-4: LLM-judged states with only on_yes have a partial/no dead-end.
1360 A state gated by the default LLM judge can receive yes/no/partial verdicts.
1361 If only on_yes is mapped (and no on_no, on_partial, next, or route table with
1362 a default exist), a partial or no verdict returns None from _route and silently
1363 terminates the loop — the parent treats this as failed.
1365 Suppressed by `partial_route_ok: true` at the loop top-level for the rare
1366 case where dead-ending on a non-yes verdict is intentional.
1367 """
1368 if fsm.partial_route_ok:
1369 return []
1370 errors: list[ValidationError] = []
1371 for state_name, state in fsm.states.items():
1372 if not _is_llm_judged(state):
1373 continue
1374 # States with an unconditional next: or a full route: table are safe.
1375 if state.next is not None or state.route is not None:
1376 continue
1377 # Only flag when on_yes is set but at least one of on_no/on_partial is missing.
1378 if state.on_yes is None:
1379 continue
1380 missing = [v for v in ("no", "partial") if getattr(state, f"on_{v}") is None]
1381 if not missing:
1382 continue
1383 unrouted = " or ".join(f"`{v}`" for v in missing)
1384 errors.append(
1385 ValidationError(
1386 message=(
1387 f"[state: {state_name}] LLM-judged prompt routes only on_yes; "
1388 f"a {unrouted} verdict has no route and will dead-end the loop "
1389 "(parent reads this as failed). Add on_no/on_partial, use `next:` "
1390 "for an unconditional handoff, or a `route:` table with a default. "
1391 "Set `partial_route_ok: true` at the loop top-level to suppress "
1392 "if intentional. (ENH-1917)"
1393 ),
1394 path=f"states.{state_name}",
1395 severity=ValidationSeverity.WARNING,
1396 )
1397 )
1398 return errors
1401def _validate_artifact_overwrite(fsm: FSMLoop) -> list[ValidationError]:
1402 """Validate rule MR-5 (ENH-1957): harness loops should version artifacts per iteration.
1404 A harness-category loop that iteratively generates and overwrites a flat artifact
1405 path (e.g. ``${context.run_dir}/image.svg``) loses all intermediate versions.
1406 Only the final iteration survives. This rule flags iterative generate→evaluate→generate
1407 cycles that write to artifact paths without declaring versioning intent.
1409 Suppressed by ``artifact_versioning: true`` (loop snapshots per-iteration artifacts)
1410 or ``artifact_versioning_ok: true`` (intentional overwrite, e.g. artifact varies
1411 by task).
1412 """
1413 if fsm.artifact_versioning or fsm.artifact_versioning_ok:
1414 return []
1415 if fsm.category not in ("harness",):
1416 return []
1418 errors: list[ValidationError] = []
1420 # Find states that write to artifact paths (shell actions with file output)
1421 writers: dict[str, set[str]] = {} # state_name -> set of artifact paths
1422 for state_name, state in fsm.states.items():
1423 if not state.action or state.action_type not in ("shell", None):
1424 continue
1425 # Skip sub-loop delegation states
1426 if state.action_type == "loop":
1427 continue
1428 action = state.action
1429 # Find run_dir-based artifact writes: ${context.run_dir}/<path> or $RUNDIR/<path>
1430 import re
1432 # Match patterns like: ${context.run_dir}/output.svg, $RUNDIR/image.png, > path
1433 # We look for output redirections or cp/mv commands writing to run_dir
1434 artifact_refs = set()
1435 for pattern in (
1436 r'\$\{context\.run_dir\}/([^\s"\';&]+)',
1437 r'\$\{captured\.[^}]+\}/([^\s"\';&]+)',
1438 ):
1439 for m in re.finditer(pattern, action):
1440 artifact_refs.add(m.group(1))
1441 # Also detect explicit cp/mv writing to run_dir paths
1442 for m in re.finditer(r'(?:cp|mv)\s+.*\s+\$\{context\.run_dir\}/([^\s"\';&]+)', action):
1443 artifact_refs.add(m.group(1))
1444 if artifact_refs:
1445 writers[state_name] = artifact_refs
1447 if not writers:
1448 return []
1450 # Detect iterative cycles: a writer state that is reachable from itself
1451 # via a non-trivial path through other states (generate → evaluate → generate)
1452 for state_name in writers:
1453 refs = fsm.states[state_name].get_referenced_states()
1454 # Check if this writer or its downstream states loop back to this writer
1455 visited: set[str] = set()
1456 to_visit = list(refs - {"$current"})
1457 while to_visit:
1458 target = to_visit.pop()
1459 if target in visited or target not in fsm.states:
1460 continue
1461 visited.add(target)
1462 if target == state_name:
1463 # Found a cycle: this writer state is reachable from itself
1464 artifact_list = ", ".join(sorted(writers[state_name]))
1465 errors.append(
1466 ValidationError(
1467 message=(
1468 f"[state: {state_name}] Harness loop writes artifact(s) "
1469 f"({artifact_list}) to a flat path in an iterative cycle "
1470 f"({state_name} → ... → {state_name}). Per-iteration versions "
1471 "are lost; only the final output survives. Add per-iteration "
1472 "snapshots (see oracle generator-evaluator for pattern) and "
1473 "declare `artifact_versioning: true`, or set "
1474 "`artifact_versioning_ok: true` if intentional. (ENH-1957)"
1475 ),
1476 path=f"states.{state_name}",
1477 severity=ValidationSeverity.WARNING,
1478 )
1479 )
1480 break
1481 target_state = fsm.states.get(target)
1482 if target_state is not None:
1483 target_refs = target_state.get_referenced_states()
1484 for r in target_refs:
1485 if r != "$current" and r not in visited:
1486 to_visit.append(r)
1488 return errors
1491def _validate_generator_fix_discipline(fsm: FSMLoop) -> list[ValidationError]:
1492 """Validate rule MR-6 (ENH-2079): meta-loops should not hand-patch generator artifacts.
1494 Detects the hand-patching anti-pattern: a ``shell``-type state that writes to the
1495 same file path as a non-shell (LLM-type) generator state in the same loop.
1496 Hand-patching creates fragile output that diverges from the generator on the next
1497 run; the stable fix is to update the generator action instead.
1499 Suppressed by ``generator_fix_ok: true`` at the loop top-level for intentional
1500 post-processing cases.
1501 """
1502 if fsm.generator_fix_ok or not _is_meta_loop(fsm):
1503 return []
1505 # Markers that indicate a prompt/slash_command state is generating file artifacts
1506 _GENERATOR_MARKERS = ("yaml_state_editor", "replace_action", "to_file:")
1508 _PATH_PATTERNS = (
1509 re.compile(r'\$\{context\.run_dir\}/([^\s"\';&|]+)'),
1510 re.compile(r'\$\{captured\.[^}]+\}/([^\s"\';&|]+)'),
1511 )
1513 def _extract_paths(action: str) -> set[str]:
1514 paths: set[str] = set()
1515 for pat in _PATH_PATTERNS:
1516 for m in pat.finditer(action):
1517 paths.add(m.group(1).rstrip("/"))
1518 return paths
1520 shell_targets: dict[str, set[str]] = {} # state_name -> set of file paths
1521 generator_targets: dict[str, set[str]] = {} # state_name -> set of file paths
1523 for state_name, state in fsm.states.items():
1524 if not state.action:
1525 continue
1526 action = state.action
1527 paths = _extract_paths(action)
1528 if not paths:
1529 continue
1530 action_type = state.action_type
1531 if action_type in ("shell", None):
1532 shell_targets[state_name] = paths
1533 elif action_type in ("prompt", "slash_command"):
1534 if any(marker in action for marker in _GENERATOR_MARKERS):
1535 generator_targets[state_name] = paths
1537 errors: list[ValidationError] = []
1538 for gen_name, gen_paths in generator_targets.items():
1539 for shell_name, shell_paths in shell_targets.items():
1540 overlap = gen_paths & shell_paths
1541 if overlap:
1542 artifact_list = ", ".join(sorted(overlap))
1543 errors.append(
1544 ValidationError(
1545 message=(
1546 f"[states: {gen_name}, {shell_name}] Hand-patching anti-pattern: "
1547 f"LLM-generator state '{gen_name}' and shell state '{shell_name}' "
1548 f"both write to ({artifact_list}). Move the fix into the generator "
1549 "action so every run produces correct output automatically. "
1550 "Set `generator_fix_ok: true` to suppress for intentional "
1551 "post-processing. (ENH-2079)"
1552 ),
1553 path=f"states.{shell_name}",
1554 severity=ValidationSeverity.WARNING,
1555 )
1556 )
1558 return errors
1561def _has_baseline_reference(fsm: FSMLoop, capture_names: set[str]) -> bool:
1562 """Return True if any evaluate block references a captured variable."""
1563 for state in fsm.states.values():
1564 ev = state.evaluate
1565 if ev is None:
1566 continue
1567 # Check string fields that may interpolate captured values
1568 candidates = [ev.previous, ev.source]
1569 if isinstance(ev.target, str):
1570 candidates.append(ev.target)
1571 for field_val in candidates:
1572 if not field_val:
1573 continue
1574 for name in capture_names:
1575 if f"captured.{name}" in field_val:
1576 return True
1577 return False
1580def _validate_on_max_iterations(fsm: FSMLoop, defined_states: set[str]) -> list[ValidationError]:
1581 """Validate the top-level `on_max_iterations` field (ENH-1631).
1583 Checks that the named state exists when `on_max_iterations` is set.
1584 """
1585 errors: list[ValidationError] = []
1586 if fsm.on_max_iterations is None:
1587 return errors
1588 if fsm.on_max_iterations not in defined_states:
1589 errors.append(
1590 ValidationError(
1591 message=(
1592 f"on_max_iterations references unknown state "
1593 f"'{fsm.on_max_iterations}' (must be a declared state)"
1594 ),
1595 path="on_max_iterations",
1596 )
1597 )
1598 return errors
1601def _validate_circuit(fsm: FSMLoop, defined_states: set[str]) -> list[ValidationError]:
1602 """Validate the top-level `circuit:` block (FEAT-1637).
1604 Checks:
1605 - `circuit.repeated_failure.window` is a positive integer.
1606 - `circuit.repeated_failure.on_repeated_failure` is either the special
1607 token ``"abort"`` or the name of a declared state.
1608 """
1609 errors: list[ValidationError] = []
1610 if fsm.circuit is None or fsm.circuit.repeated_failure is None:
1611 return errors
1613 rf = fsm.circuit.repeated_failure
1614 if rf.window < 1:
1615 errors.append(
1616 ValidationError(
1617 message=f"circuit.repeated_failure.window must be >= 1, got {rf.window}",
1618 path="circuit.repeated_failure.window",
1619 )
1620 )
1622 target = rf.on_repeated_failure
1623 if target not in STALL_SPECIAL_TOKENS and target not in defined_states:
1624 errors.append(
1625 ValidationError(
1626 message=(
1627 f"circuit.repeated_failure.on_repeated_failure references "
1628 f"unknown state '{target}' (must be a declared state or "
1629 f'the literal "abort")'
1630 ),
1631 path="circuit.repeated_failure.on_repeated_failure",
1632 )
1633 )
1635 return errors
1638# Matches common interpolation prefixes used in loop YAML paths so we can
1639# extract the portable relative component for action-string scanning.
1640_INTERPOLATION_PREFIX_RE = re.compile(r"^\$\{[^}]+\}/")
1643def _strip_interpolation_prefix(path: str) -> str:
1644 """Return the path with any leading ${...}/ prefix removed."""
1645 return _INTERPOLATION_PREFIX_RE.sub("", path)
1648def _validate_progress_paths_isolation(fsm: FSMLoop) -> list[ValidationError]:
1649 """Warn when a state's action writes to a file listed in progress_paths (BUG-1767).
1651 When a loop's own bookkeeping files appear in both progress_paths and the
1652 state action strings, every append to those files resets the stall window,
1653 silently disabling the BUG-1674 stall guard for that loop. Authors should
1654 move such files to exclude_paths so the stall detector can still fire.
1655 """
1656 if fsm.circuit is None or fsm.circuit.repeated_failure is None:
1657 return []
1658 rf = fsm.circuit.repeated_failure
1659 if not rf.progress_paths:
1660 return []
1662 # Build a set of the relative path components we need to look for.
1663 watched = {_strip_interpolation_prefix(p) for p in rf.progress_paths}
1664 # Exclude paths that are already in exclude_paths — author acknowledged.
1665 excluded = {_strip_interpolation_prefix(p) for p in rf.exclude_paths}
1666 active_watched = watched - excluded
1667 if not active_watched:
1668 return []
1670 errors: list[ValidationError] = []
1671 for state_name, state in fsm.states.items():
1672 if not state.action:
1673 continue
1674 for path_fragment in active_watched:
1675 if path_fragment in state.action:
1676 errors.append(
1677 ValidationError(
1678 message=(
1679 f"State action references '{path_fragment}', which is also "
1680 "listed in circuit.repeated_failure.progress_paths. Writes "
1681 "to this file will reset the stall window every cycle, "
1682 "silently disabling stall detection. Move it to "
1683 "circuit.repeated_failure.exclude_paths to separate "
1684 "bookkeeping files from real progress signals."
1685 ),
1686 path=f"states.{state_name}.action",
1687 severity=ValidationSeverity.WARNING,
1688 )
1689 )
1690 return errors
1693def _dominated_by_any(fsm: FSMLoop, dominators: set[str], dominated: str) -> bool:
1694 """Return True if the set ``dominators`` collectively dominates ``dominated``.
1696 Group domination: every path from the initial state to ``dominated`` must
1697 pass through at least one state in ``dominators``. Checked by removing all
1698 dominator states from the graph and testing whether ``dominated`` is still
1699 reachable from the initial state.
1701 This generalizes single-state domination — used when a capture variable is
1702 produced by more than one state on mutually-exclusive branches, where the
1703 reference is safe as long as *some* capturing state runs on every path.
1705 Args:
1706 fsm: The FSM loop to analyze
1707 dominators: Names of the states that should collectively dominate
1708 dominated: Name of the state that should be dominated
1710 Returns:
1711 True if the dominators collectively dominate ``dominated``
1712 """
1713 if dominated in dominators:
1714 return True
1715 if dominated not in fsm.states:
1716 return False
1718 visited: set[str] = set()
1719 to_visit: deque[str] = deque([fsm.initial])
1721 while to_visit:
1722 current = to_visit.popleft()
1723 if current in visited or current not in fsm.states:
1724 continue
1725 if current in dominators:
1726 continue # Block this node (simulate removal)
1728 visited.add(current)
1730 if current == dominated:
1731 # Reached dominated without going through any dominator
1732 return False
1734 state = fsm.states[current]
1735 for ref in state.get_referenced_states():
1736 if ref != "$current" and ref not in visited:
1737 to_visit.append(ref)
1739 # Dominated not reachable without the dominators → they dominate
1740 return True
1743def _dominates(fsm: FSMLoop, dominator: str, dominated: str) -> bool:
1744 """Return True if dominator dominates dominated in the FSM graph.
1746 A state D dominates S if every path from the initial state to S must pass
1747 through D. Thin single-state wrapper around :func:`_dominated_by_any`.
1748 """
1749 if dominator not in fsm.states:
1750 return False
1751 return _dominated_by_any(fsm, {dominator}, dominated)
1754def _find_bypass_path_any(fsm: FSMLoop, dominators: set[str], dominated: str) -> list[str]:
1755 """Find an example path from initial to dominated that bypasses all dominators.
1757 Uses BFS to find the shortest path that avoids every state in ``dominators``.
1758 Returns empty list if no bypass exists (should not happen when called after
1759 :func:`_dominated_by_any` returns False).
1760 """
1761 parent: dict[str, str] = {}
1762 to_visit: deque[str] = deque([fsm.initial])
1763 visited: set[str] = set()
1765 while to_visit:
1766 current = to_visit.popleft()
1767 if current in visited or current not in fsm.states:
1768 continue
1769 if current in dominators:
1770 continue
1772 visited.add(current)
1774 if current == dominated:
1775 # Reconstruct path
1776 path = [dominated]
1777 while path[-1] in parent:
1778 path.append(parent[path[-1]])
1779 path.reverse()
1780 return path
1782 state = fsm.states[current]
1783 for ref in state.get_referenced_states():
1784 if ref != "$current" and ref not in visited:
1785 if ref not in parent:
1786 parent[ref] = current
1787 to_visit.append(ref)
1789 return []
1792def _find_bypass_path(fsm: FSMLoop, dominator: str, dominated: str) -> list[str]:
1793 """Find an example path from initial to dominated that bypasses dominator.
1795 Thin single-state wrapper around :func:`_find_bypass_path_any`.
1796 """
1797 return _find_bypass_path_any(fsm, {dominator}, dominated)
1800def _has_sub_loop_state(fsm: FSMLoop) -> bool:
1801 """Return True if any state in the FSM has ``loop:`` set (delegates to a child loop).
1803 Used by ENH-1961 to distinguish "capture lives in a sub-loop" from "capture is missing".
1804 """
1805 return any(state.loop is not None for state in fsm.states.values())
1808def _validate_capture_reachability(fsm: FSMLoop) -> list[ValidationError]:
1809 """Validate that ``${captured.*}`` references are dominated by their capturing states.
1811 ENH-1961: For each state that references ``${captured.<var>.*}`` in its action
1812 or evaluate source, checks that the capturing state dominates the referencing
1813 state (i.e., all paths from the initial state pass through the capture state).
1815 Emits:
1816 - WARNING when a capture state does not dominate a referencing state
1817 (the reference may crash at runtime on paths that bypass the capture).
1818 - ERROR when a referenced capture variable has no capturing state at all
1819 in this FSM (excluding sub-loop captures which live in child namespaces).
1820 """
1821 errors: list[ValidationError] = []
1823 # Step 1: Build capture map (var_name → set of capturing state names).
1824 # A variable may be captured by more than one state on mutually-exclusive
1825 # branches (e.g. fifo_pop vs select_next dispatched by schedule_mode); the
1826 # reference is safe as long as the *set* collectively dominates it.
1827 capture_map: dict[str, set[str]] = {}
1828 for state_name, state in fsm.states.items():
1829 if state.capture:
1830 capture_map.setdefault(state.capture, set()).add(state_name)
1832 # Step 2: Build reference map (state_name → set of captured var names referenced)
1833 reference_map: dict[str, set[str]] = {}
1834 for state_name, state in fsm.states.items():
1835 # Skip sub-loop delegation states — their action is a loop name,
1836 # and captured vars belong to the child loop's namespace.
1837 if state.loop is not None:
1838 continue
1840 # Only collect references NOT guarded by `:default=` — a guarded
1841 # reference is safe even when the capture is missing on some path.
1842 refs: set[str] = set()
1843 if state.action:
1844 refs.update(_unguarded_captured_refs(state.action))
1845 if state.evaluate is not None and state.evaluate.source:
1846 refs.update(_unguarded_captured_refs(state.evaluate.source))
1847 if refs:
1848 reference_map[state_name] = refs
1850 if not reference_map:
1851 return errors
1853 # Step 3: For each reference, check dominance of capturing state
1854 for ref_state_name, ref_vars in reference_map.items():
1855 for var_name in ref_vars:
1856 if var_name not in capture_map:
1857 # Referenced capture variable has no capturing state in this FSM.
1858 # ENH-1998: downgrade to WARNING (not silent skip) when sub-loops
1859 # are present — the capture may live in a child namespace, but a
1860 # typo'd name should still surface rather than go completely dark.
1861 if _has_sub_loop_state(fsm):
1862 errors.append(
1863 ValidationError(
1864 message=(
1865 f"References ${{captured.{var_name}.*}} but no state in "
1866 f"this loop captures '{var_name}'. "
1867 f"If '{var_name}' is produced by a sub-loop, this may be "
1868 f"intentional; otherwise add 'capture: {var_name}' to the "
1869 f"state that produces this value."
1870 ),
1871 path=f"states.{ref_state_name}.action",
1872 severity=ValidationSeverity.WARNING,
1873 )
1874 )
1875 continue
1876 # No sub-loops: this is genuinely missing.
1877 errors.append(
1878 ValidationError(
1879 message=(
1880 f"References ${{captured.{var_name}.*}} but no state in "
1881 f"this loop captures '{var_name}'. Add 'capture: {var_name}' "
1882 f"to the state that produces this value."
1883 ),
1884 path=f"states.{ref_state_name}.action",
1885 severity=ValidationSeverity.ERROR,
1886 )
1887 )
1888 continue
1890 # Capturing states present in this FSM (shouldn't drop any normally)
1891 cap_states = {s for s in capture_map[var_name] if s in fsm.states}
1892 if not cap_states:
1893 continue
1895 # Group dominance check: do the capturing states collectively
1896 # dominate ref_state_name (does at least one run on every path)?
1897 if not _dominated_by_any(fsm, cap_states, ref_state_name):
1898 bypass_path = _find_bypass_path_any(fsm, cap_states, ref_state_name)
1899 path_str = " → ".join(bypass_path) if bypass_path else "unknown path"
1901 if len(cap_states) == 1:
1902 captured_by = f"state '{next(iter(cap_states))}' which may not"
1903 else:
1904 names = ", ".join(f"'{s}'" for s in sorted(cap_states))
1905 captured_by = f"states {names}, none of which"
1907 errors.append(
1908 ValidationError(
1909 message=(
1910 f"References ${{captured.{var_name}.*}} but '{var_name}' "
1911 f"is captured by {captured_by} "
1912 f"execute on all paths to '{ref_state_name}'. "
1913 f"Path(s) bypassing capture: {path_str}"
1914 ),
1915 path=f"states.{ref_state_name}.action",
1916 severity=ValidationSeverity.WARNING,
1917 )
1918 )
1920 return errors
1923def _find_reachable_states(fsm: FSMLoop) -> set[str]:
1924 """Find all states reachable from the initial state.
1926 Uses breadth-first search to find all reachable states. Seeds the BFS
1927 with the initial state plus top-level transition targets that act as
1928 alternate entry points: ``on_max_iterations`` (fires when the iteration
1929 cap is hit) and ``circuit.repeated_failure.on_repeated_failure`` (fires
1930 when the circuit breaker trips). These are real edges the runtime can
1931 take, so states reached only through them are not orphans.
1933 Args:
1934 fsm: The FSM loop to analyze
1936 Returns:
1937 Set of reachable state names
1938 """
1939 reachable: set[str] = set()
1940 to_visit: deque[str] = deque([fsm.initial])
1941 if fsm.on_max_iterations is not None:
1942 to_visit.append(fsm.on_max_iterations)
1943 if fsm.circuit is not None and fsm.circuit.repeated_failure is not None:
1944 target = fsm.circuit.repeated_failure.on_repeated_failure
1945 if target not in STALL_SPECIAL_TOKENS:
1946 to_visit.append(target)
1948 while to_visit:
1949 current = to_visit.popleft()
1950 if current in reachable or current not in fsm.states:
1951 continue
1953 reachable.add(current)
1954 state = fsm.states[current]
1955 refs = state.get_referenced_states()
1957 for ref in refs:
1958 if ref != "$current" and ref not in reachable:
1959 to_visit.append(ref)
1961 return reachable
1964def is_runnable_loop(path: Path) -> bool:
1965 """Cheap check for whether a YAML file is a runnable FSM loop definition.
1967 Returns True iff the file parses as a YAML mapping with the required
1968 top-level keys ``name``, ``initial``, and either ``states`` or ``flow``
1969 (the shorthand resolved by :func:`resolve_flow`). This matches the
1970 required-fields gate in :func:`load_and_validate` (lines 885-894) so
1971 "counted by the verifier" stays in sync with "runnable by ll-loop validate".
1973 No fragment/inheritance resolution is performed — library fragments under
1974 ``loops/lib/`` that omit ``initial`` or ``states`` correctly return False.
1975 """
1976 try:
1977 data = yaml.safe_load(path.read_text())
1978 except (OSError, yaml.YAMLError):
1979 return False
1980 if not isinstance(data, dict):
1981 return False
1982 has_flow = "states" in data or "flow" in data
1983 return "name" in data and "initial" in data and has_flow
1986def load_and_validate(
1987 path: Path,
1988 raise_on_error: bool = True,
1989) -> tuple[FSMLoop, list[ValidationError]]:
1990 """Load YAML file and validate FSM structure.
1992 Args:
1993 path: Path to the YAML file to load
1994 raise_on_error: When True (default), raise ValueError on ERROR violations.
1995 When False, return all violations (errors + warnings) without raising.
1997 Returns:
1998 When raise_on_error=True: (FSMLoop, list of WARNING-severity ValidationErrors)
1999 When raise_on_error=False: (FSMLoop, list of all ValidationErrors sorted errors-first)
2001 Raises:
2002 FileNotFoundError: If the file doesn't exist
2003 yaml.YAMLError: If the file is not valid YAML
2004 ValueError: If raise_on_error=True and validation fails (contains error details)
2005 """
2006 if not path.exists():
2007 raise FileNotFoundError(f"FSM file not found: {path}")
2009 with open(path) as f:
2010 data: dict[str, Any] = yaml.safe_load(f)
2012 if not isinstance(data, dict):
2013 raise ValueError(f"FSM file must contain a YAML mapping, got {type(data)}")
2015 # Resolve `from:` inheritance before any further checks, so a child loop
2016 # can omit fields its parent provides (including `initial`/`states`) and
2017 # so a parent's `import:`/`fragments:` blocks survive into the merged
2018 # result for the subsequent `resolve_fragments` pass.
2019 data = resolve_inheritance(data, path.parent)
2021 # Expand flow: linear shorthand into states: before required-fields check
2022 data = resolve_flow(data)
2024 # Check required fields before parsing
2025 missing = []
2026 for field in ["name", "initial"]:
2027 if field not in data:
2028 missing.append(field)
2029 if "states" not in data:
2030 missing.append("states (or flow)")
2032 if missing:
2033 raise ValueError(f"FSM file missing required fields: {', '.join(missing)}")
2035 # Check for unknown top-level keys before parsing
2036 unknown_key_warnings: list[ValidationError] = []
2037 unknown = set(data.keys()) - KNOWN_TOP_LEVEL_KEYS
2038 if unknown:
2039 unknown_key_warnings.append(
2040 ValidationError(
2041 path="<root>",
2042 message=f"Unknown top-level keys: {', '.join(sorted(unknown))}",
2043 severity=ValidationSeverity.WARNING,
2044 )
2045 )
2047 visibility_val = data.get("visibility")
2048 if visibility_val is not None and visibility_val not in VALID_VISIBILITY:
2049 unknown_key_warnings.append(
2050 ValidationError(
2051 path="visibility",
2052 message=(
2053 f"Invalid visibility: {visibility_val!r}. "
2054 f"Must be one of: {', '.join(sorted(VALID_VISIBILITY))}. "
2055 "Loop will be treated as 'public'."
2056 ),
2057 severity=ValidationSeverity.WARNING,
2058 )
2059 )
2061 # Resolve fragment libraries before parsing into dataclass
2062 data = resolve_fragments(data, path.parent)
2064 # Parse into dataclass
2065 fsm = FSMLoop.from_dict(data)
2067 # Validate
2068 errors = validate_fsm(fsm)
2070 # Validate with: bindings against child loop parameters (requires file-system access)
2071 errors.extend(_validate_with_bindings(fsm, path.parent))
2072 errors.extend(_validate_fragment_bindings(fsm, path.parent))
2074 # Filter to errors only (not warnings) for raising
2075 error_list = [e for e in errors if e.severity == ValidationSeverity.ERROR]
2076 struct_warnings = [e for e in errors if e.severity == ValidationSeverity.WARNING]
2077 all_warnings = unknown_key_warnings + struct_warnings
2079 if not raise_on_error:
2080 return fsm, error_list + all_warnings
2082 if error_list:
2083 error_messages = "\n ".join(str(e) for e in error_list)
2084 raise ValueError(f"FSM validation failed:\n {error_messages}")
2086 for warning in all_warnings:
2087 logger.warning(str(warning))
2089 return fsm, all_warnings