Coverage for little_loops / fsm / validation.py: 13%
360 statements
« prev ^ index » next coverage.py v7.12.0, created at 2026-05-28 13:07 -0500
1"""FSM loop validation logic.
3This module provides validation for FSM loop definitions, ensuring
4structural correctness and catching common configuration errors.
6Validation checks:
7- Initial state exists in states dict
8- All referenced states exist
9- At least one terminal state
10- Evaluator configs have required fields for their type
11- No conflicting routing (shorthand vs full route)
12- Numeric fields in valid ranges (max_iterations > 0, backoff >= 0, timeout > 0)
13"""
15from __future__ import annotations
17import logging
18import re
19from collections import deque
20from dataclasses import dataclass
21from enum import Enum
22from pathlib import Path
23from typing import Any
25import yaml
27from little_loops.fsm.fragments import resolve_flow, resolve_fragments, resolve_inheritance
28from little_loops.fsm.schema import EvaluateConfig, FSMLoop, ParameterSpec, StateConfig
30logger = logging.getLogger(__name__)
33class ValidationSeverity(Enum):
34 """Severity level for validation issues."""
36 ERROR = "error"
37 WARNING = "warning"
40@dataclass
41class ValidationError:
42 """Structured validation error.
44 Attributes:
45 message: Human-readable error description
46 path: Path to the problematic element (e.g., "states.check.route")
47 severity: Error severity (error or warning)
48 """
50 message: str
51 path: str | None = None
52 severity: ValidationSeverity = ValidationSeverity.ERROR
54 def __str__(self) -> str:
55 """Format error for display."""
56 prefix = f"[{self.severity.value.upper()}]"
57 if self.path:
58 return f"{prefix} {self.path}: {self.message}"
59 return f"{prefix} {self.message}"
62# Evaluator type to required fields mapping
63EVALUATOR_REQUIRED_FIELDS: dict[str, list[str]] = {
64 "exit_code": [],
65 "output_numeric": ["operator", "target"],
66 "output_json": ["path", "operator", "target"],
67 "output_contains": ["pattern"],
68 "convergence": ["target"],
69 "diff_stall": [],
70 "llm_structured": [],
71 "mcp_result": [],
72 "harbor_scorer": [],
73}
75# Non-LLM evaluator types: all evaluator types except llm_structured
76# Derived from EVALUATOR_REQUIRED_FIELDS so new types are automatically included
77NON_LLM_EVALUATOR_TYPES: frozenset[str] = frozenset(EVALUATOR_REQUIRED_FIELDS.keys()) - {
78 "llm_structured"
79}
81# Meta-loop detector: action string patterns that indicate harness artifact writes
82_META_LOOP_ACTION_PATTERNS: tuple[re.Pattern[str], ...] = (
83 re.compile(r"loops/[\w-]+\.yaml"),
84 re.compile(r"skills/[\w-]+/SKILL\.md"),
85 re.compile(r"agents/[\w-]+\.md"),
86 re.compile(r"commands/[\w-]+\.md"),
87 re.compile(r"\.claude/(CLAUDE\.md|settings)"),
88)
90# Action string tokens that indicate meta-loop behavior
91_META_LOOP_ACTION_TOKENS: frozenset[str] = frozenset({"yaml_state_editor", "replace_action"})
93# Import paths that identify a loop as a meta-loop (harness optimization framework)
94_META_LOOP_IMPORT_TRIGGERS: frozenset[str] = frozenset({"lib/benchmark.yaml"})
96# Valid comparison operators
97VALID_OPERATORS = {"eq", "ne", "lt", "le", "gt", "ge"}
99# All top-level keys recognized by FSMLoop.from_dict()
100KNOWN_TOP_LEVEL_KEYS: frozenset[str] = frozenset(
101 {
102 "name",
103 "description",
104 "initial",
105 "states",
106 "context",
107 "parameters",
108 "scope",
109 "max_iterations",
110 "on_max_iterations",
111 "max_edge_revisits",
112 "backoff",
113 "timeout",
114 "default_timeout",
115 "maintain",
116 "llm",
117 "on_handoff",
118 "input_key",
119 "config",
120 "category",
121 "labels",
122 "commands",
123 "targets",
124 "circuit",
125 "meta_self_eval_ok",
126 "import",
127 "fragments",
128 "from",
129 "flow",
130 "state_defs",
131 }
132)
134# Special tokens accepted as the `on_repeated_failure` target on the
135# stall detector — `"abort"` means terminate via _finish("stall_detected").
136STALL_SPECIAL_TOKENS: frozenset[str] = frozenset({"abort"})
138# Valid parameter types for the 'parameters:' block
139VALID_PARAMETER_TYPES: frozenset[str] = frozenset(
140 {"string", "integer", "number", "boolean", "enum", "path"}
141)
144def _validate_evaluator(state_name: str, evaluate: EvaluateConfig) -> list[ValidationError]:
145 """Validate evaluator configuration for type-specific requirements.
147 Args:
148 state_name: Name of the state containing this evaluator
149 evaluate: The evaluator configuration to validate
151 Returns:
152 List of validation errors found
153 """
154 errors: list[ValidationError] = []
155 path = f"states.{state_name}.evaluate"
157 # Check that evaluator type is recognized
158 valid_types = set(EVALUATOR_REQUIRED_FIELDS.keys())
159 if evaluate.type not in valid_types:
160 errors.append(
161 ValidationError(
162 message=f"Unknown evaluator type '{evaluate.type}'. "
163 f"Must be one of: {', '.join(sorted(valid_types))}",
164 path=path,
165 )
166 )
167 return errors # Can't check required fields for unknown type
169 # Check required fields for evaluator type
170 required = EVALUATOR_REQUIRED_FIELDS.get(evaluate.type, [])
171 for field_name in required:
172 value = getattr(evaluate, field_name, None)
173 if value is None:
174 errors.append(
175 ValidationError(
176 message=f"Evaluator type '{evaluate.type}' requires '{field_name}' field",
177 path=path,
178 )
179 )
181 # Validate operator if present
182 if evaluate.operator is not None and evaluate.operator not in VALID_OPERATORS:
183 errors.append(
184 ValidationError(
185 message=f"Invalid operator '{evaluate.operator}'. "
186 f"Must be one of: {', '.join(sorted(VALID_OPERATORS))}",
187 path=f"{path}.operator",
188 )
189 )
191 # Validate convergence-specific fields
192 if evaluate.type == "convergence":
193 if evaluate.direction not in ("minimize", "maximize"):
194 errors.append(
195 ValidationError(
196 message=f"Invalid direction '{evaluate.direction}'. "
197 "Must be 'minimize' or 'maximize'",
198 path=f"{path}.direction",
199 )
200 )
201 # Only validate tolerance if it's a numeric value (not an interpolation string)
202 if (
203 evaluate.tolerance is not None
204 and isinstance(evaluate.tolerance, (int, float))
205 and evaluate.tolerance < 0
206 ):
207 errors.append(
208 ValidationError(
209 message="Tolerance cannot be negative",
210 path=f"{path}.tolerance",
211 )
212 )
214 # Validate llm_structured-specific fields
215 if evaluate.type == "llm_structured":
216 if evaluate.min_confidence < 0 or evaluate.min_confidence > 1:
217 errors.append(
218 ValidationError(
219 message="min_confidence must be between 0 and 1",
220 path=f"{path}.min_confidence",
221 )
222 )
224 # Validate diff_stall-specific fields
225 if evaluate.type == "diff_stall":
226 if evaluate.max_stall < 1:
227 errors.append(
228 ValidationError(
229 message="max_stall must be >= 1",
230 path=f"{path}.max_stall",
231 )
232 )
234 return errors
237def _validate_parameters(fsm: FSMLoop) -> list[ValidationError]:
238 """Validate the loop's top-level parameters: block.
240 Args:
241 fsm: The FSM loop to validate
243 Returns:
244 List of validation errors found
245 """
246 errors: list[ValidationError] = []
248 for param_name, param_spec in fsm.parameters.items():
249 path = f"parameters.{param_name}"
251 if param_spec.type not in VALID_PARAMETER_TYPES:
252 errors.append(
253 ValidationError(
254 message=(
255 f"Unknown parameter type '{param_spec.type}'. "
256 f"Must be one of: {', '.join(sorted(VALID_PARAMETER_TYPES))}"
257 ),
258 path=path,
259 )
260 )
262 if param_spec.type == "enum" and not param_spec.values:
263 errors.append(
264 ValidationError(
265 message="Parameter type 'enum' requires a 'values' list",
266 path=path,
267 )
268 )
270 if param_spec.required and param_spec.default is not None:
271 errors.append(
272 ValidationError(
273 message="Parameter cannot be both 'required: true' and have a 'default' value",
274 path=path,
275 )
276 )
278 return errors
281def _check_param_type(value: Any, spec: ParameterSpec) -> str | None:
282 """Return an error message if value does not match spec.type, else None."""
283 if spec.type == "string" and not isinstance(value, str):
284 return f"expected string, got {type(value).__name__}"
285 if spec.type == "integer" and not isinstance(value, int):
286 return f"expected integer, got {type(value).__name__}"
287 if spec.type == "number" and not isinstance(value, (int, float)):
288 return f"expected number, got {type(value).__name__}"
289 if spec.type == "boolean" and not isinstance(value, bool):
290 return f"expected boolean, got {type(value).__name__}"
291 if spec.type == "enum" and spec.values and value not in spec.values:
292 return f"expected one of {spec.values!r}, got {value!r}"
293 return None
296def _validate_with_bindings(fsm: FSMLoop, loop_dir: Path) -> list[ValidationError]:
297 """Validate with: bindings against child loop parameter contracts.
299 Called from load_and_validate (not validate_fsm) because resolving child loops
300 requires file-system access via the loop directory path.
302 Args:
303 fsm: The parent FSM loop
304 loop_dir: Directory to resolve child loop paths from
306 Returns:
307 List of validation errors found
308 """
309 errors: list[ValidationError] = []
311 for state_name, state in fsm.states.items():
312 if state.loop is None or not state.with_:
313 continue
315 # Try to resolve and load the child loop; skip if unavailable
316 try:
317 from little_loops.cli.loop._helpers import resolve_loop_path
319 loop_path = resolve_loop_path(state.loop, loop_dir)
320 child_fsm, _ = load_and_validate(loop_path)
321 except Exception:
322 continue
324 if not child_fsm.parameters:
325 continue # Child has no declared contract — nothing to cross-validate
327 path = f"states.{state_name}"
329 # Unknown with: keys (not declared by child)
330 for key in state.with_:
331 if key not in child_fsm.parameters:
332 errors.append(
333 ValidationError(
334 message=(
335 f"'with.{key}' is not a declared parameter of loop '{state.loop}'. "
336 f"Declared: {', '.join(sorted(child_fsm.parameters))}"
337 ),
338 path=f"{path}.with.{key}",
339 )
340 )
342 # Required parameters not bound
343 for param_name, param_spec in child_fsm.parameters.items():
344 if param_spec.required and param_name not in state.with_:
345 errors.append(
346 ValidationError(
347 message=(
348 f"Required parameter '{param_name}' of loop '{state.loop}' "
349 f"is not bound in 'with'"
350 ),
351 path=f"{path}.with",
352 )
353 )
355 # Statically-detectable type mismatches (skip interpolation strings)
356 for param_name, value in state.with_.items():
357 if param_name not in child_fsm.parameters:
358 continue
359 if isinstance(value, str) and "${" in value:
360 continue
361 type_error = _check_param_type(value, child_fsm.parameters[param_name])
362 if type_error:
363 errors.append(
364 ValidationError(
365 message=f"Parameter '{param_name}': {type_error}",
366 path=f"{path}.with.{param_name}",
367 )
368 )
370 return errors
373def _validate_state_action(state_name: str, state: StateConfig) -> list[ValidationError]:
374 """Validate state action configuration.
376 Args:
377 state_name: Name of the state to validate
378 state: The state configuration to validate
380 Returns:
381 List of validation errors found
382 """
383 errors: list[ValidationError] = []
384 path = f"states.{state_name}"
386 # params field is only valid for mcp_tool states
387 if state.params and state.action_type != "mcp_tool":
388 errors.append(
389 ValidationError(
390 message="'params' field is only valid when action_type is 'mcp_tool'",
391 path=f"{path}.params",
392 )
393 )
395 # loop and action are mutually exclusive
396 if state.loop is not None and state.action is not None:
397 errors.append(
398 ValidationError(
399 message="'loop' and 'action' are mutually exclusive — "
400 "a sub-loop state cannot also have an action",
401 path=f"{path}",
402 )
403 )
405 # with: requires loop: to be set
406 if state.with_ and state.loop is None:
407 errors.append(
408 ValidationError(
409 message="'with' is only valid when 'loop' is set",
410 path=f"{path}.with",
411 )
412 )
414 # FEAT-1283: type=learning requires a populated LearningConfig
415 if state.type == "learning" and state.learning is not None:
416 if not state.learning.targets:
417 errors.append(
418 ValidationError(
419 message="type=learning requires non-empty 'learning.targets'",
420 path=f"{path}.learning.targets",
421 )
422 )
423 if state.learning.max_retries < 0:
424 errors.append(
425 ValidationError(
426 message=(
427 f"learning.max_retries must be >= 0, got {state.learning.max_retries}"
428 ),
429 path=f"{path}.learning.max_retries",
430 )
431 )
432 if state.on_yes is None:
433 errors.append(
434 ValidationError(
435 message="type=learning requires 'on_yes' (target for all-proven)",
436 path=f"{path}.on_yes",
437 )
438 )
439 if state.on_blocked is None and state.on_no is None:
440 errors.append(
441 ValidationError(
442 message=(
443 "type=learning requires 'on_blocked' or 'on_no' "
444 "(target for refuted / retries_exhausted)"
445 ),
446 path=f"{path}",
447 )
448 )
450 # with: and context_passthrough are mutually exclusive
451 if state.with_ and state.context_passthrough:
452 errors.append(
453 ValidationError(
454 message=(
455 "'with' and 'context_passthrough' are mutually exclusive — "
456 "use 'with' for explicit parameter bindings or 'context_passthrough' "
457 "for legacy bulk passthrough, not both"
458 ),
459 path=f"{path}",
460 )
461 )
463 return errors
466def _validate_state_routing(state_name: str, state: StateConfig) -> list[ValidationError]:
467 """Validate state routing configuration.
469 Checks for conflicting routing definitions (shorthand vs full route).
471 Args:
472 state_name: Name of the state to validate
473 state: The state configuration to validate
475 Returns:
476 List of validation errors/warnings found
477 """
478 errors: list[ValidationError] = []
479 path = f"states.{state_name}"
481 has_shorthand = (
482 state.on_yes is not None
483 or state.on_no is not None
484 or state.on_error is not None
485 or state.on_partial is not None
486 or state.on_blocked is not None
487 or bool(state.extra_routes)
488 )
489 has_route = state.route is not None
491 # Warn about conflicting definitions
492 if has_shorthand and has_route:
493 errors.append(
494 ValidationError(
495 message="Both shorthand routing (on_yes/on_no/on_error) "
496 "and full route table defined. Route table will take precedence.",
497 path=path,
498 severity=ValidationSeverity.WARNING,
499 )
500 )
502 # Check for no valid transition definition
503 has_next = state.next is not None
504 has_terminal = state.terminal
505 has_loop = state.loop is not None
507 if not has_shorthand and not has_route and not has_next and not has_terminal and not has_loop:
508 errors.append(
509 ValidationError(
510 message="State has no transition defined. Add routing, 'next', "
511 "or mark as 'terminal: true'",
512 path=path,
513 )
514 )
516 # Validate retry field pairing: max_retries requires on_retry_exhausted and vice versa
517 if state.max_retries is not None and state.on_retry_exhausted is None:
518 errors.append(
519 ValidationError(
520 message="'max_retries' requires 'on_retry_exhausted' to also be set",
521 path=path,
522 )
523 )
524 if state.on_retry_exhausted is not None and state.max_retries is None:
525 errors.append(
526 ValidationError(
527 message="'on_retry_exhausted' requires 'max_retries' to also be set",
528 path=path,
529 )
530 )
531 if state.max_retries is not None and state.max_retries < 1:
532 errors.append(
533 ValidationError(
534 message=f"'max_retries' must be >= 1, got {state.max_retries}",
535 path=path,
536 )
537 )
539 # Validate rate-limit retry field pairing (mirrors max_retries/on_retry_exhausted)
540 if state.max_rate_limit_retries is not None and state.on_rate_limit_exhausted is None:
541 errors.append(
542 ValidationError(
543 message="'max_rate_limit_retries' requires 'on_rate_limit_exhausted' to also be set",
544 path=path,
545 )
546 )
547 if state.on_rate_limit_exhausted is not None and state.max_rate_limit_retries is None:
548 errors.append(
549 ValidationError(
550 message="'on_rate_limit_exhausted' requires 'max_rate_limit_retries' to also be set",
551 path=path,
552 )
553 )
554 if state.max_rate_limit_retries is not None and state.max_rate_limit_retries < 1:
555 errors.append(
556 ValidationError(
557 message=f"'max_rate_limit_retries' must be >= 1, got {state.max_rate_limit_retries}",
558 path=path,
559 )
560 )
561 if (
562 state.rate_limit_backoff_base_seconds is not None
563 and state.rate_limit_backoff_base_seconds < 1
564 ):
565 errors.append(
566 ValidationError(
567 message=(
568 f"'rate_limit_backoff_base_seconds' must be >= 1, "
569 f"got {state.rate_limit_backoff_base_seconds}"
570 ),
571 path=path,
572 )
573 )
574 if state.rate_limit_max_wait_seconds is not None and state.rate_limit_max_wait_seconds < 1:
575 errors.append(
576 ValidationError(
577 message=(
578 f"'rate_limit_max_wait_seconds' must be >= 1, "
579 f"got {state.rate_limit_max_wait_seconds}"
580 ),
581 path=path,
582 )
583 )
584 if state.rate_limit_long_wait_ladder is not None:
585 if len(state.rate_limit_long_wait_ladder) == 0:
586 errors.append(
587 ValidationError(
588 message="'rate_limit_long_wait_ladder' must be non-empty if specified",
589 path=path,
590 )
591 )
592 else:
593 for idx, value in enumerate(state.rate_limit_long_wait_ladder):
594 if not isinstance(value, int) or value < 1:
595 errors.append(
596 ValidationError(
597 message=(
598 f"'rate_limit_long_wait_ladder[{idx}]' must be a "
599 f"positive integer, got {value!r}"
600 ),
601 path=path,
602 )
603 )
605 # Validate throttle config when present
606 if state.throttle is not None:
607 t = state.throttle
608 fields = {
609 "normal_max": t.normal_max,
610 "warn_max": t.warn_max,
611 "hard_max": t.hard_max,
612 }
613 for field_name, val in fields.items():
614 if val is not None and (not isinstance(val, int) or val < 1):
615 errors.append(
616 ValidationError(
617 message=f"'throttle.{field_name}' must be a positive integer, got {val!r}",
618 path=path,
619 )
620 )
621 # Enforce ordering when all three are set
622 if t.normal_max is not None and t.warn_max is not None and t.normal_max >= t.warn_max:
623 errors.append(
624 ValidationError(
625 message=(
626 f"'throttle.normal_max' ({t.normal_max}) must be less than "
627 f"'throttle.warn_max' ({t.warn_max})"
628 ),
629 path=path,
630 )
631 )
632 if t.warn_max is not None and t.hard_max is not None and t.warn_max >= t.hard_max:
633 errors.append(
634 ValidationError(
635 message=(
636 f"'throttle.warn_max' ({t.warn_max}) must be less than "
637 f"'throttle.hard_max' ({t.hard_max})"
638 ),
639 path=path,
640 )
641 )
643 return errors
646def _validate_targets(fsm: FSMLoop) -> list[ValidationError]:
647 """Validate top-level targets[] entries (ENH-1552).
649 Rejects any targets[].states[] entry whose sibling file: value does not
650 end with a .yaml extension.
651 """
652 errors: list[ValidationError] = []
653 for i, target in enumerate(fsm.targets):
654 if target.file is not None and not target.file.endswith(".yaml"):
655 errors.append(
656 ValidationError(
657 message=(f"targets[{i}].file must be a .yaml file, got '{target.file}'"),
658 path=f"targets[{i}].file",
659 )
660 )
661 return errors
664def _validate_failure_terminal_action(fsm: FSMLoop) -> list[ValidationError]:
665 """Warn when a failure-named terminal state has no diagnostic predecessor.
667 Failure terminals (failed, error, aborted) should have at least one
668 predecessor state with an action or sub-loop that provides diagnostic
669 output before termination. Otherwise the failure is silent — the
670 executor calls _finish("terminal") before any action on the terminal
671 itself can execute.
673 Severity is WARNING (not ERROR) so that existing loops with bare
674 failure terminals continue to load, and test_terminal_only_state_valid
675 (which filters by ERROR) passes without modification.
676 """
677 FAILURE_TERMINAL_NAMES: frozenset[str] = frozenset({"failed", "error", "aborted"})
678 errors: list[ValidationError] = []
680 terminal_states = fsm.get_terminal_states()
681 failure_terminals = terminal_states & FAILURE_TERMINAL_NAMES
683 for ft_name in failure_terminals:
684 has_diagnostic_predecessor = False
685 for state_name, state in fsm.states.items():
686 if state_name == ft_name:
687 continue
688 if ft_name in state.get_referenced_states():
689 if state.action is not None or state.loop is not None:
690 has_diagnostic_predecessor = True
691 break
693 if not has_diagnostic_predecessor:
694 errors.append(
695 ValidationError(
696 message=(
697 f"Failure-named terminal state '{ft_name}' has no predecessor "
698 "state with a diagnostic action. Add a non-terminal diagnostic "
699 "state (e.g. 'diagnose') with an action or sub-loop that routes "
700 f"to '{ft_name}'."
701 ),
702 path=f"states.{ft_name}",
703 severity=ValidationSeverity.WARNING,
704 )
705 )
707 return errors
710def validate_fsm(fsm: FSMLoop) -> list[ValidationError]:
711 """Validate FSM structure and return list of errors.
713 Performs comprehensive validation:
714 - Initial state exists
715 - All referenced states exist
716 - At least one terminal state
717 - Evaluator configurations are valid
718 - Routing configurations are valid
719 - Numeric fields are in valid ranges (max_iterations > 0, backoff >= 0, timeout > 0)
721 Args:
722 fsm: The FSM loop to validate
724 Returns:
725 List of validation errors (empty if valid)
726 """
727 errors: list[ValidationError] = []
728 defined_states = fsm.get_all_state_names()
730 # Warn when no top-level description: field is set. The field is optional
731 # for FSM execution but required for goal-alignment skills (debug-loop-run,
732 # audit-loop-run) and for ll-loop show --json to surface intent text.
733 if not fsm.description:
734 errors.append(
735 ValidationError(
736 path="<root>",
737 message=("No 'description' field defined. Add a top-level description: key."),
738 severity=ValidationSeverity.WARNING,
739 )
740 )
742 # Validate parameters block
743 errors.extend(_validate_parameters(fsm))
745 # Validate targets block (ENH-1552)
746 errors.extend(_validate_targets(fsm))
748 # Check initial state exists
749 if fsm.initial not in defined_states:
750 errors.append(
751 ValidationError(
752 message=f"Initial state '{fsm.initial}' not found in states",
753 path="initial",
754 )
755 )
757 # Check at least one terminal state
758 terminal_states = fsm.get_terminal_states()
759 if not terminal_states:
760 errors.append(
761 ValidationError(
762 message="No terminal state defined. At least one state must have 'terminal: true'",
763 path="states",
764 )
765 )
767 # Validate each state
768 for state_name, state in fsm.states.items():
769 # Check all referenced states exist
770 refs = state.get_referenced_states()
771 for ref in refs:
772 # $current is a special token for retry
773 if ref != "$current" and ref not in defined_states:
774 errors.append(
775 ValidationError(
776 message=f"References unknown state '{ref}'",
777 path=f"states.{state_name}",
778 )
779 )
781 # Validate action configuration
782 errors.extend(_validate_state_action(state_name, state))
784 # Validate evaluator if present
785 if state.evaluate is not None:
786 errors.extend(_validate_evaluator(state_name, state.evaluate))
788 # Validate routing configuration
789 errors.extend(_validate_state_routing(state_name, state))
791 # Check numeric field ranges
792 if fsm.max_iterations <= 0:
793 errors.append(
794 ValidationError(
795 message=f"max_iterations must be > 0, got {fsm.max_iterations}",
796 path="max_iterations",
797 )
798 )
799 if fsm.max_edge_revisits <= 0:
800 errors.append(
801 ValidationError(
802 message=f"max_edge_revisits must be > 0, got {fsm.max_edge_revisits}",
803 path="max_edge_revisits",
804 )
805 )
806 if fsm.backoff is not None and fsm.backoff < 0:
807 errors.append(
808 ValidationError(
809 message=f"backoff must be >= 0, got {fsm.backoff}",
810 path="backoff",
811 )
812 )
813 if fsm.timeout is not None and fsm.timeout <= 0:
814 errors.append(
815 ValidationError(
816 message=f"timeout must be > 0, got {fsm.timeout}",
817 path="timeout",
818 )
819 )
820 if fsm.llm.max_tokens <= 0:
821 errors.append(
822 ValidationError(
823 message=f"llm.max_tokens must be > 0, got {fsm.llm.max_tokens}",
824 path="llm.max_tokens",
825 )
826 )
827 if fsm.llm.timeout <= 0:
828 errors.append(
829 ValidationError(
830 message=f"llm.timeout must be > 0, got {fsm.llm.timeout}",
831 path="llm.timeout",
832 )
833 )
835 # Check for unreachable states (warning only)
836 reachable = _find_reachable_states(fsm)
837 unreachable = defined_states - reachable
838 for state_name in unreachable:
839 errors.append(
840 ValidationError(
841 message="State is not reachable from initial state",
842 path=f"states.{state_name}",
843 severity=ValidationSeverity.WARNING,
844 )
845 )
847 errors.extend(_validate_failure_terminal_action(fsm))
849 errors.extend(_validate_meta_loop_evaluation(fsm))
851 errors.extend(_validate_on_max_iterations(fsm, defined_states))
853 errors.extend(_validate_circuit(fsm, defined_states))
855 return errors
858def _is_meta_loop(fsm: FSMLoop) -> bool:
859 """Return True if fsm is classified as a meta-loop.
861 A loop is meta if ANY of the following match:
862 1. Any state action string matches a harness-artifact path regex
863 (writes another loop YAML, skill, agent, command, or project config)
864 2. The loop's import list contains lib/benchmark.yaml
865 3. Any state action references yaml_state_editor or replace_action
866 """
867 # Condition 2: imports lib/benchmark.yaml
868 if any(imp in _META_LOOP_IMPORT_TRIGGERS for imp in fsm.imports):
869 return True
870 # Conditions 1 and 3: scan action strings
871 for state in fsm.states.values():
872 if state.action is None:
873 continue
874 for pattern in _META_LOOP_ACTION_PATTERNS:
875 if pattern.search(state.action):
876 return True
877 for token in _META_LOOP_ACTION_TOKENS:
878 if token in state.action:
879 return True
880 return False
883def _validate_meta_loop_evaluation(fsm: FSMLoop) -> list[ValidationError]:
884 """Validate meta-loop evaluation rules MR-1 and MR-2.
886 MR-1 (ERROR): meta-loop must have at least one non-LLM evaluator.
887 MR-2 (WARNING): meta-loop should reference a captured baseline in an evaluator.
889 Both rules are suppressed by ``meta_self_eval_ok: true`` at the loop top-level.
890 """
891 errors: list[ValidationError] = []
892 if fsm.meta_self_eval_ok or not _is_meta_loop(fsm):
893 return errors
895 # Collect all evaluator types used across all states
896 evaluator_types: set[str] = set()
897 for state in fsm.states.values():
898 if state.evaluate is not None:
899 evaluator_types.add(state.evaluate.type)
901 # MR-1: must have at least one non-LLM evaluator
902 if not evaluator_types & NON_LLM_EVALUATOR_TYPES:
903 errors.append(
904 ValidationError(
905 message=(
906 "Loop modifies harness artifacts but has no non-LLM evaluator. "
907 "LLM self-grades on harness updates are unreliable (SHOR Table 1: "
908 "33-55% accuracy). Pair every check_semantic state with at least one "
909 "of: exit_code, output_numeric, convergence, diff_stall, mcp_result. "
910 "To suppress with justification, set `meta_self_eval_ok: true` at the "
911 "loop top-level."
912 ),
913 path="<root>",
914 severity=ValidationSeverity.ERROR,
915 )
916 )
918 # MR-2: should reference a captured baseline in a later evaluator
919 capture_names: set[str] = {state.capture for state in fsm.states.values() if state.capture}
920 if capture_names and not _has_baseline_reference(fsm, capture_names):
921 errors.append(
922 ValidationError(
923 message=(
924 "Meta-loop appears to lack a measure→propose→apply→re-measure "
925 "spine: no captured baseline value is referenced by a later evaluator. "
926 "Meta-loops should compare a post-change score against a pre-change "
927 "baseline (see loops/harness-optimize.yaml as reference template). "
928 "To suppress, set `meta_self_eval_ok: true`."
929 ),
930 path="<root>",
931 severity=ValidationSeverity.WARNING,
932 )
933 )
935 return errors
938def _has_baseline_reference(fsm: FSMLoop, capture_names: set[str]) -> bool:
939 """Return True if any evaluate block references a captured variable."""
940 for state in fsm.states.values():
941 ev = state.evaluate
942 if ev is None:
943 continue
944 # Check string fields that may interpolate captured values
945 candidates = [ev.previous, ev.source]
946 if isinstance(ev.target, str):
947 candidates.append(ev.target)
948 for field_val in candidates:
949 if not field_val:
950 continue
951 for name in capture_names:
952 if f"captured.{name}" in field_val:
953 return True
954 return False
957def _validate_on_max_iterations(fsm: FSMLoop, defined_states: set[str]) -> list[ValidationError]:
958 """Validate the top-level `on_max_iterations` field (ENH-1631).
960 Checks that the named state exists when `on_max_iterations` is set.
961 """
962 errors: list[ValidationError] = []
963 if fsm.on_max_iterations is None:
964 return errors
965 if fsm.on_max_iterations not in defined_states:
966 errors.append(
967 ValidationError(
968 message=(
969 f"on_max_iterations references unknown state "
970 f"'{fsm.on_max_iterations}' (must be a declared state)"
971 ),
972 path="on_max_iterations",
973 )
974 )
975 return errors
978def _validate_circuit(fsm: FSMLoop, defined_states: set[str]) -> list[ValidationError]:
979 """Validate the top-level `circuit:` block (FEAT-1637).
981 Checks:
982 - `circuit.repeated_failure.window` is a positive integer.
983 - `circuit.repeated_failure.on_repeated_failure` is either the special
984 token ``"abort"`` or the name of a declared state.
985 """
986 errors: list[ValidationError] = []
987 if fsm.circuit is None or fsm.circuit.repeated_failure is None:
988 return errors
990 rf = fsm.circuit.repeated_failure
991 if rf.window < 1:
992 errors.append(
993 ValidationError(
994 message=f"circuit.repeated_failure.window must be >= 1, got {rf.window}",
995 path="circuit.repeated_failure.window",
996 )
997 )
999 target = rf.on_repeated_failure
1000 if target not in STALL_SPECIAL_TOKENS and target not in defined_states:
1001 errors.append(
1002 ValidationError(
1003 message=(
1004 f"circuit.repeated_failure.on_repeated_failure references "
1005 f"unknown state '{target}' (must be a declared state or "
1006 f'the literal "abort")'
1007 ),
1008 path="circuit.repeated_failure.on_repeated_failure",
1009 )
1010 )
1012 return errors
1015def _find_reachable_states(fsm: FSMLoop) -> set[str]:
1016 """Find all states reachable from the initial state.
1018 Uses breadth-first search to find all reachable states.
1020 Args:
1021 fsm: The FSM loop to analyze
1023 Returns:
1024 Set of reachable state names
1025 """
1026 reachable: set[str] = set()
1027 to_visit: deque[str] = deque([fsm.initial])
1029 while to_visit:
1030 current = to_visit.popleft()
1031 if current in reachable or current not in fsm.states:
1032 continue
1034 reachable.add(current)
1035 state = fsm.states[current]
1036 refs = state.get_referenced_states()
1038 for ref in refs:
1039 if ref != "$current" and ref not in reachable:
1040 to_visit.append(ref)
1042 return reachable
1045def is_runnable_loop(path: Path) -> bool:
1046 """Cheap check for whether a YAML file is a runnable FSM loop definition.
1048 Returns True iff the file parses as a YAML mapping with the required
1049 top-level keys ``name``, ``initial``, and either ``states`` or ``flow``
1050 (the shorthand resolved by :func:`resolve_flow`). This matches the
1051 required-fields gate in :func:`load_and_validate` (lines 885-894) so
1052 "counted by the verifier" stays in sync with "runnable by ll-loop validate".
1054 No fragment/inheritance resolution is performed — library fragments under
1055 ``loops/lib/`` that omit ``initial`` or ``states`` correctly return False.
1056 """
1057 try:
1058 data = yaml.safe_load(path.read_text())
1059 except (OSError, yaml.YAMLError):
1060 return False
1061 if not isinstance(data, dict):
1062 return False
1063 has_flow = "states" in data or "flow" in data
1064 return "name" in data and "initial" in data and has_flow
1067def load_and_validate(path: Path) -> tuple[FSMLoop, list[ValidationError]]:
1068 """Load YAML file and validate FSM structure.
1070 Args:
1071 path: Path to the YAML file to load
1073 Returns:
1074 Tuple of (validated FSMLoop instance, list of WARNING-severity ValidationErrors)
1076 Raises:
1077 FileNotFoundError: If the file doesn't exist
1078 yaml.YAMLError: If the file is not valid YAML
1079 ValueError: If validation fails (contains error details)
1080 """
1081 if not path.exists():
1082 raise FileNotFoundError(f"FSM file not found: {path}")
1084 with open(path) as f:
1085 data: dict[str, Any] = yaml.safe_load(f)
1087 if not isinstance(data, dict):
1088 raise ValueError(f"FSM file must contain a YAML mapping, got {type(data)}")
1090 # Resolve `from:` inheritance before any further checks, so a child loop
1091 # can omit fields its parent provides (including `initial`/`states`) and
1092 # so a parent's `import:`/`fragments:` blocks survive into the merged
1093 # result for the subsequent `resolve_fragments` pass.
1094 data = resolve_inheritance(data, path.parent)
1096 # Expand flow: linear shorthand into states: before required-fields check
1097 data = resolve_flow(data)
1099 # Check required fields before parsing
1100 missing = []
1101 for field in ["name", "initial"]:
1102 if field not in data:
1103 missing.append(field)
1104 if "states" not in data:
1105 missing.append("states (or flow)")
1107 if missing:
1108 raise ValueError(f"FSM file missing required fields: {', '.join(missing)}")
1110 # Check for unknown top-level keys before parsing
1111 unknown_key_warnings: list[ValidationError] = []
1112 unknown = set(data.keys()) - KNOWN_TOP_LEVEL_KEYS
1113 if unknown:
1114 unknown_key_warnings.append(
1115 ValidationError(
1116 path="<root>",
1117 message=f"Unknown top-level keys: {', '.join(sorted(unknown))}",
1118 severity=ValidationSeverity.WARNING,
1119 )
1120 )
1122 # Resolve fragment libraries before parsing into dataclass
1123 data = resolve_fragments(data, path.parent)
1125 # Parse into dataclass
1126 fsm = FSMLoop.from_dict(data)
1128 # Validate
1129 errors = validate_fsm(fsm)
1131 # Validate with: bindings against child loop parameters (requires file-system access)
1132 errors.extend(_validate_with_bindings(fsm, path.parent))
1134 # Filter to errors only (not warnings) for raising
1135 error_list = [e for e in errors if e.severity == ValidationSeverity.ERROR]
1137 if error_list:
1138 error_messages = "\n ".join(str(e) for e in error_list)
1139 raise ValueError(f"FSM validation failed:\n {error_messages}")
1141 # Collect all warnings (unknown-key warnings + structural warnings)
1142 struct_warnings = [e for e in errors if e.severity == ValidationSeverity.WARNING]
1143 all_warnings = unknown_key_warnings + struct_warnings
1144 for warning in all_warnings:
1145 logger.warning(str(warning))
1147 return fsm, all_warnings