Coverage for little_loops / fsm / schema.py: 44%
552 statements
« prev ^ index » next coverage.py v7.12.0, created at 2026-06-16 13:12 -0500
1"""FSM loop schema dataclasses.
3This module defines the type-safe dataclasses that represent FSM loop
4definitions. These match the universal FSM schema described in the
5design documentation.
7The schema supports:
8- Multiple evaluator types (exit_code, output_numeric, etc.)
9- Two-layer transition system (evaluate + route)
10- Both shorthand (on_success/on_failure) and full routing syntax
11- Context variables and captured values
12- LLM evaluation configuration
13"""
15from __future__ import annotations
17from dataclasses import dataclass, field
18from typing import Any, Literal
20# Default LLM model for structured evaluation
21DEFAULT_LLM_MODEL: str = "sonnet"
24@dataclass
25class EvaluateConfig:
26 """Evaluator configuration for action result interpretation.
28 The evaluator determines how to interpret an action's output and
29 produce a verdict string for routing.
31 Attributes:
32 type: Evaluator type. One of:
33 - exit_code: Map exit codes to verdicts (default for shell)
34 - output_numeric: Compare numeric output to target
35 - output_json: Extract and compare JSON path value
36 - output_contains: Pattern matching on stdout
37 - convergence: Compare current vs previous value toward target
38 - llm_structured: Use LLM with structured output (default for slash)
39 operator: Comparison operator (eq, ne, lt, le, gt, ge)
40 target: Target value for comparison
41 tolerance: Acceptable distance from target (for convergence)
42 pattern: Pattern string for output_contains
43 negate: If True, invert the match result (output_contains)
44 path: JSON path for output_json (jq-style)
45 prompt: Custom prompt for llm_structured
46 schema: Custom JSON schema for llm_structured response
47 min_confidence: Minimum confidence threshold for llm_structured
48 uncertain_suffix: If True, append _uncertain to low-confidence verdicts
49 source: Override default source (current action output)
50 previous: Previous value reference for convergence
51 direction: Optimization direction for convergence (minimize/maximize)
52 scope: Paths to limit git diff to for diff_stall evaluator
53 max_stall: Consecutive no-change iterations before failure (diff_stall)
54 pairs: List of producer/consumer pair dicts for contract evaluator
55 line: Line selector for classify evaluator (last/first/<int index>)
56 """
58 type: Literal[
59 "exit_code",
60 "output_numeric",
61 "output_json",
62 "output_contains",
63 "convergence",
64 "diff_stall",
65 "action_stall",
66 "llm_structured",
67 "mcp_result",
68 "harbor_scorer",
69 "comparator",
70 "contract",
71 "classify",
72 ]
73 operator: str | None = None
74 target: int | float | str | None = None
75 tolerance: float | str | None = None # str for interpolation (e.g., "${context.tolerance}")
76 pattern: str | None = None
77 negate: bool = False
78 path: str | None = None
79 prompt: str | None = None
80 schema: dict[str, Any] | None = None
81 min_confidence: float = 0.5
82 uncertain_suffix: bool = False
83 source: str | None = None
84 previous: str | None = None
85 direction: Literal["minimize", "maximize"] = "minimize"
86 scope: list[str] | None = None # for diff_stall: limit git diff to these paths
87 max_stall: int = 1 # for diff_stall: consecutive no-change iterations before failure
88 track: list[str] | None = None # for action_stall: context keys to track (default: ["action"])
89 max_repeat: int = 2 # for action_stall: consecutive identical iterations before failure
90 baseline_path: str | None = None # for comparator: path to .loops/baselines/<loop>/ dir
91 auto_promote: bool = False # for comparator: write output to baseline on yes verdict
92 min_pairs: int = 1 # for comparator: number of blind A/B comparisons to run
93 pairs: list[dict] | None = None # for contract: list of producer/consumer pair dicts
94 line: str | int | None = None # for classify: which line to read (last/first/<int index>)
96 def to_dict(self) -> dict[str, Any]:
97 """Convert to dictionary for JSON/YAML serialization."""
98 result: dict[str, Any] = {"type": self.type}
100 # Only include non-None optional fields
101 if self.operator is not None:
102 result["operator"] = self.operator
103 if self.target is not None:
104 result["target"] = self.target
105 if self.tolerance is not None:
106 result["tolerance"] = self.tolerance
107 if self.pattern is not None:
108 result["pattern"] = self.pattern
109 if self.negate:
110 result["negate"] = self.negate
111 if self.path is not None:
112 result["path"] = self.path
113 if self.prompt is not None:
114 result["prompt"] = self.prompt
115 if self.schema is not None:
116 result["schema"] = self.schema
117 if self.min_confidence != 0.5:
118 result["min_confidence"] = self.min_confidence
119 if self.uncertain_suffix:
120 result["uncertain_suffix"] = self.uncertain_suffix
121 if self.source is not None:
122 result["source"] = self.source
123 if self.previous is not None:
124 result["previous"] = self.previous
125 if self.direction != "minimize":
126 result["direction"] = self.direction
127 if self.scope is not None:
128 result["scope"] = self.scope
129 if self.max_stall != 1:
130 result["max_stall"] = self.max_stall
131 if self.track is not None:
132 result["track"] = self.track
133 if self.max_repeat != 2:
134 result["max_repeat"] = self.max_repeat
135 if self.baseline_path is not None:
136 result["baseline_path"] = self.baseline_path
137 if self.auto_promote:
138 result["auto_promote"] = self.auto_promote
139 if self.min_pairs != 1:
140 result["min_pairs"] = self.min_pairs
141 if self.pairs is not None:
142 result["pairs"] = self.pairs
143 if self.line is not None:
144 result["line"] = self.line
146 return result
148 @classmethod
149 def from_dict(cls, data: dict[str, Any]) -> EvaluateConfig:
150 """Create from dictionary (JSON/YAML deserialization)."""
151 return cls(
152 type=data["type"],
153 operator=data.get("operator"),
154 target=data.get("target"),
155 tolerance=data.get("tolerance"),
156 pattern=data.get("pattern"),
157 negate=data.get("negate", False),
158 path=data.get("path"),
159 prompt=data.get("prompt"),
160 schema=data.get("schema"),
161 min_confidence=data.get("min_confidence", 0.5),
162 uncertain_suffix=data.get("uncertain_suffix", False),
163 source=data.get("source"),
164 previous=data.get("previous"),
165 direction=data.get("direction", "minimize"),
166 scope=data.get("scope"),
167 max_stall=data.get("max_stall", 1),
168 track=data.get("track"),
169 max_repeat=data.get("max_repeat", 2),
170 baseline_path=data.get("baseline_path"),
171 auto_promote=data.get("auto_promote", False),
172 min_pairs=data.get("min_pairs", 1),
173 pairs=data.get("pairs"),
174 line=data.get("line"),
175 )
178@dataclass
179class RouteConfig:
180 """Routing table configuration for verdict-to-state mapping.
182 Maps verdict strings from evaluators to next state names.
184 Attributes:
185 routes: Mapping from verdict string to next state name
186 default: Default state for unmatched verdicts (the "_" key)
187 error: State for evaluation/execution errors (the "_error" key)
188 """
190 routes: dict[str, str] = field(default_factory=dict)
191 default: str | None = None
192 error: str | None = None
194 def to_dict(self) -> dict[str, Any]:
195 """Convert to dictionary for JSON/YAML serialization."""
196 result = dict(self.routes)
197 if self.default is not None:
198 result["_"] = self.default
199 if self.error is not None:
200 result["_error"] = self.error
201 return result
203 @classmethod
204 def from_dict(cls, data: dict[str, Any]) -> RouteConfig:
205 """Create from dictionary (JSON/YAML deserialization)."""
206 routes = {k: v for k, v in data.items() if not k.startswith("_")}
207 return cls(
208 routes=routes,
209 default=data.get("_"),
210 error=data.get("_error"),
211 )
214@dataclass
215class ParameterSpec:
216 """Specification for a single loop input parameter.
218 Declares a typed input that callers bind via the 'with:' block on sub-loop states.
220 Attributes:
221 type: Parameter type. One of: string, integer, number, boolean, enum, path.
222 required: If True, callers must supply this parameter in 'with:' (mutually
223 exclusive with 'default').
224 default: Default value used when the caller does not bind the parameter.
225 Only valid when required is False.
226 description: Human-readable description of the parameter.
227 values: Allowed values for 'enum' type parameters.
228 """
230 type: str
231 required: bool = False
232 default: Any = None
233 description: str | None = None
234 values: list[Any] | None = None
236 def to_dict(self) -> dict[str, Any]:
237 """Convert to dictionary for JSON/YAML serialization."""
238 result: dict[str, Any] = {"type": self.type}
239 if self.required:
240 result["required"] = self.required
241 if self.default is not None:
242 result["default"] = self.default
243 if self.description is not None:
244 result["description"] = self.description
245 if self.values is not None:
246 result["values"] = self.values
247 return result
249 @classmethod
250 def from_dict(cls, data: dict[str, Any]) -> ParameterSpec:
251 """Create from dictionary (JSON/YAML deserialization)."""
252 return cls(
253 type=data["type"],
254 required=data.get("required", False),
255 default=data.get("default"),
256 description=data.get("description"),
257 values=data.get("values"),
258 )
261@dataclass
262class ThrottleConfig:
263 """Per-state tool-call progressive throttling configuration.
265 Counts successful tool calls within a single state visit and escalates
266 restrictions to self-throttle runaway states before provider limits are reached.
268 Thresholds fall back to executor module defaults when not set:
269 - normal_max: _DEFAULT_THROTTLE_NORMAL_MAX (3) — calls 1..normal_max pass through
270 - warn_max: _DEFAULT_THROTTLE_WARN_MAX (8) — calls at warn_max inject a warning event
271 - hard_max: _DEFAULT_THROTTLE_HARD_MAX (12) — calls at hard_max route to on_throttle_hard
272 - calls > hard_max: hard stop, loop marked stuck
274 States with type="learning" (FEAT-1283) are exempt from the hard_max hard-stop because
275 they legitimately make N tool calls per visit (one per unproven target). The warn_max
276 warning still applies so users can see the state is doing significant work.
277 """
279 normal_max: int | None = None
280 warn_max: int | None = None
281 hard_max: int | None = None
283 def to_dict(self) -> dict[str, Any]:
284 """Convert to dictionary for JSON/YAML serialization."""
285 result: dict[str, Any] = {}
286 if self.normal_max is not None:
287 result["normal_max"] = self.normal_max
288 if self.warn_max is not None:
289 result["warn_max"] = self.warn_max
290 if self.hard_max is not None:
291 result["hard_max"] = self.hard_max
292 return result
294 @classmethod
295 def from_dict(cls, data: dict[str, Any]) -> ThrottleConfig:
296 """Create from dictionary (JSON/YAML deserialization)."""
297 return cls(
298 normal_max=data.get("normal_max"),
299 warn_max=data.get("warn_max"),
300 hard_max=data.get("hard_max"),
301 )
304@dataclass
305class LearningConfig:
306 """Per-state configuration for FEAT-1283 `type: learning` dispatch.
308 Declares the list of external-API/SDK targets a learning state must prove
309 against the learning-tests registry (ENH-1282) before advancing. On a
310 missing or stale record, the state invokes `/ll:explore-api <target>` up to
311 `max_retries` times before transitioning to a blocked target.
313 Attributes:
314 targets: Ordered list of target identifiers (e.g. "Anthropic SDK
315 streaming"). Each is slugified internally when looking up the
316 registry record. All targets must reach status="proven" for the
317 state to advance via on_yes.
318 targets_csv: Runtime-interpolated comma-separated target string (e.g.
319 "${context.targets}"). Resolved and CSV-split by the executor at
320 execution time. Alternative to the static ``targets`` list for loops
321 that receive targets as a context CSV string. Exactly one of
322 ``targets`` (non-empty) or ``targets_csv`` must be set.
323 max_retries: Maximum number of `/ll:explore-api` invocations per target
324 before the state routes to on_blocked / on_no with reason
325 ``retries_exhausted``. Counts only re-exploration attempts; the
326 initial registry lookup is free. Distinct from ENH-1115's throttle
327 counter, which measures tool-call volume; learning states are
328 already exempt from throttle hard_max via FSMExecutor._check_throttle.
329 max_retries_expr: Runtime-interpolated retry limit (e.g.
330 "${context.max_retries}"). Resolved via interpolate() and int()-cast
331 at execution time. Takes precedence over ``max_retries`` when set.
332 """
334 targets: list[str] = field(default_factory=list)
335 targets_csv: str | None = None
336 max_retries: int = 2
337 max_retries_expr: str | None = None
339 def to_dict(self) -> dict[str, Any]:
340 """Convert to dictionary for JSON/YAML serialization."""
341 result: dict[str, Any] = {"targets": list(self.targets), "max_retries": self.max_retries}
342 if self.targets_csv is not None:
343 result["targets_csv"] = self.targets_csv
344 if self.max_retries_expr is not None:
345 result["max_retries_expr"] = self.max_retries_expr
346 return result
348 @classmethod
349 def from_dict(cls, data: dict[str, Any]) -> LearningConfig:
350 """Create from dictionary (JSON/YAML deserialization)."""
351 return cls(
352 targets=list(data.get("targets") or []),
353 targets_csv=data.get("targets_csv") or None,
354 max_retries=int(data.get("max_retries", 2)),
355 max_retries_expr=data.get("max_retries_expr") or None,
356 )
359@dataclass
360class StateConfig:
361 """Configuration for a single FSM state.
363 States can have actions, evaluators, and routing. Supports both
364 shorthand (on_success/on_failure) and full routing table syntax.
366 Attributes:
367 action: Command to execute (shell, slash command, or "server/tool-name" for mcp_tool)
368 action_type: How to execute the action (prompt, slash_command, shell, mcp_tool).
369 If None, uses heuristic: / prefix = slash_command, else = shell.
370 params: MCP tool arguments (only used with action_type: mcp_tool). Supports
371 ${variable} interpolation in string values.
372 evaluate: Evaluator configuration for result interpretation
373 route: Full routing table (verdict -> state mapping)
374 on_yes: Shorthand for yes verdict routing
375 on_no: Shorthand for no verdict routing
376 on_error: Shorthand for error verdict routing
377 on_partial: Shorthand for partial verdict routing
378 next: Unconditional transition (no evaluation)
379 terminal: If True, this is an end state
380 capture: Variable name to store action output
381 timeout: Action-level timeout in seconds
382 on_maintain: State to transition to when maintain=True and loop completes
383 max_retries: Max consecutive re-entries before transitioning to on_retry_exhausted.
384 A value of N allows N retries after the initial execution (N+1 total entries).
385 Requires on_retry_exhausted to also be set.
386 on_retry_exhausted: State to transition to when max_retries consecutive re-entries
387 are exceeded. Required when max_retries is set.
388 max_rate_limit_retries: Max consecutive 429/rate-limit in-place retries for this
389 state before transitioning to on_rate_limit_exhausted. Requires
390 on_rate_limit_exhausted to also be set.
391 on_rate_limit_exhausted: State to transition to when max_rate_limit_retries
392 consecutive rate-limit retries are exceeded. Required when
393 max_rate_limit_retries is set.
394 rate_limit_backoff_base_seconds: Base seconds for exponential backoff between
395 rate-limit retries; actual sleep is base * 2^(attempt-1) + uniform(0, base).
396 Defaults to 30.
397 rate_limit_max_wait_seconds: Total wall-clock budget (seconds) for rate-limit
398 handling in this state before routing to on_rate_limit_exhausted. When unset,
399 defaults from commands.rate_limits.max_wait_seconds (21600 = 6h).
400 rate_limit_long_wait_ladder: Backoff ladder (seconds) for the long-wait tier used
401 once the short-tier retry budget is spent. Each entry is the sleep before the
402 next retry attempt. When unset, defaults from
403 commands.rate_limits.long_wait_ladder ([300, 900, 1800, 3600]).
404 loop: Name of a loop YAML to execute as a sub-FSM. Mutually exclusive with action.
405 context_passthrough: When True, pass parent context variables to child loop and
406 merge child captures back into parent context. Legacy escape hatch; prefer
407 'with_' for explicit named bindings.
408 with_: Explicit parameter bindings for sub-loop calls (YAML key: 'with'). Maps
409 declared child parameter names to parent expressions. Only valid when 'loop'
410 is set. Mutually exclusive with context_passthrough.
411 fragment_name: Original fragment name from the 'fragment:' YAML key. Populated by
412 resolve_fragments; None for non-fragment states. Used for validation and debugging.
413 fragment_bindings: Parameter bindings for fragment references (YAML key 'with' on a
414 fragment state, renamed to 'fragment_bindings' by resolve_fragments). Distinct from
415 with_ (sub-loop bindings) and params (MCP tool args). Only valid when fragment_name
416 is set.
417 fragment_parameters: Parsed ParameterSpec declarations from the fragment definition's
418 'parameters:' block. Populated by resolve_fragments for validation and runtime checks.
419 """
421 action: str | None = None
422 action_type: str | None = None
423 params: dict[str, Any] = field(default_factory=dict)
424 evaluate: EvaluateConfig | None = None
425 route: RouteConfig | None = None
426 on_yes: str | None = None
427 on_no: str | None = None
428 on_error: str | None = None
429 on_partial: str | None = None
430 on_blocked: str | None = None
431 next: str | None = None
432 terminal: bool = False
433 capture: str | None = None
434 append_to_messages: str | None = None
435 timeout: int | None = None
436 on_maintain: str | None = None
437 max_retries: int | None = None
438 on_retry_exhausted: str | None = None
439 retryable_exit_codes: list[int] | None = None
440 max_rate_limit_retries: int | None = None
441 on_rate_limit_exhausted: str | None = None
442 rate_limit_backoff_base_seconds: int | None = None
443 rate_limit_max_wait_seconds: int | None = None
444 rate_limit_long_wait_ladder: list[int] | None = None
445 loop: str | None = None
446 context_passthrough: bool = False
447 with_: dict[str, Any] = field(default_factory=dict)
448 fragment_name: str | None = None
449 fragment_bindings: dict[str, Any] = field(default_factory=dict)
450 fragment_parameters: dict[str, ParameterSpec] = field(default_factory=dict)
451 agent: str | None = None
452 tools: list[str] | None = None
453 model: str | None = None
454 extra_routes: dict[str, str] = field(default_factory=dict)
455 type: str | None = None
456 throttle: ThrottleConfig | None = None
457 on_throttle_hard: str | None = None
458 learning: LearningConfig | None = None
460 def to_dict(self) -> dict[str, Any]:
461 """Convert to dictionary for JSON/YAML serialization."""
462 result: dict[str, Any] = {}
464 if self.action is not None:
465 result["action"] = self.action
466 if self.action_type is not None:
467 result["action_type"] = self.action_type
468 if self.params:
469 result["params"] = self.params
470 if self.evaluate is not None:
471 result["evaluate"] = self.evaluate.to_dict()
472 if self.route is not None:
473 result["route"] = self.route.to_dict()
474 if self.on_yes is not None:
475 result["on_yes"] = self.on_yes
476 if self.on_no is not None:
477 result["on_no"] = self.on_no
478 if self.on_error is not None:
479 result["on_error"] = self.on_error
480 if self.on_partial is not None:
481 result["on_partial"] = self.on_partial
482 if self.on_blocked is not None:
483 result["on_blocked"] = self.on_blocked
484 if self.next is not None:
485 result["next"] = self.next
486 if self.terminal:
487 result["terminal"] = self.terminal
488 if self.capture is not None:
489 result["capture"] = self.capture
490 if self.append_to_messages is not None:
491 result["append_to_messages"] = self.append_to_messages
492 if self.timeout is not None:
493 result["timeout"] = self.timeout
494 if self.on_maintain is not None:
495 result["on_maintain"] = self.on_maintain
496 if self.max_retries is not None:
497 result["max_retries"] = self.max_retries
498 if self.on_retry_exhausted is not None:
499 result["on_retry_exhausted"] = self.on_retry_exhausted
500 if self.retryable_exit_codes is not None:
501 result["retryable_exit_codes"] = self.retryable_exit_codes
502 if self.max_rate_limit_retries is not None:
503 result["max_rate_limit_retries"] = self.max_rate_limit_retries
504 if self.on_rate_limit_exhausted is not None:
505 result["on_rate_limit_exhausted"] = self.on_rate_limit_exhausted
506 if self.rate_limit_backoff_base_seconds is not None:
507 result["rate_limit_backoff_base_seconds"] = self.rate_limit_backoff_base_seconds
508 if self.rate_limit_max_wait_seconds is not None:
509 result["rate_limit_max_wait_seconds"] = self.rate_limit_max_wait_seconds
510 if self.rate_limit_long_wait_ladder is not None:
511 result["rate_limit_long_wait_ladder"] = self.rate_limit_long_wait_ladder
512 if self.loop is not None:
513 result["loop"] = self.loop
514 if self.context_passthrough:
515 result["context_passthrough"] = self.context_passthrough
516 if self.with_:
517 result["with"] = self.with_
518 if self.fragment_bindings:
519 result["fragment_bindings"] = self.fragment_bindings
520 if self.agent is not None:
521 result["agent"] = self.agent
522 if self.tools is not None:
523 result["tools"] = self.tools
524 if self.model is not None:
525 result["model"] = self.model
526 for verdict, target in self.extra_routes.items():
527 result[f"on_{verdict}"] = target
528 if self.type is not None:
529 result["type"] = self.type
530 if self.throttle is not None:
531 result["throttle"] = self.throttle.to_dict()
532 if self.on_throttle_hard is not None:
533 result["on_throttle_hard"] = self.on_throttle_hard
534 if self.learning is not None:
535 result["learning"] = self.learning.to_dict()
537 return result
539 @classmethod
540 def from_dict(cls, data: dict[str, Any]) -> StateConfig:
541 """Create from dictionary (JSON/YAML deserialization)."""
542 evaluate = None
543 if "evaluate" in data:
544 evaluate = EvaluateConfig.from_dict(data["evaluate"])
546 route = None
547 if "route" in data:
548 route = RouteConfig.from_dict(data["route"])
550 throttle = None
551 if "throttle" in data:
552 throttle = ThrottleConfig.from_dict(data["throttle"])
554 learning = None
555 if "learning" in data:
556 learning = LearningConfig.from_dict(data["learning"])
558 _known_on_keys = {
559 "on_yes",
560 "on_success",
561 "on_no",
562 "on_failure",
563 "on_error",
564 "on_partial",
565 "on_blocked",
566 "on_maintain",
567 "on_retry_exhausted",
568 "on_rate_limit_exhausted",
569 "on_throttle_hard",
570 }
571 extra_routes = {
572 key[3:]: val
573 for key, val in data.items()
574 if key.startswith("on_") and key not in _known_on_keys and isinstance(val, str)
575 }
577 return cls(
578 action=data.get("action"),
579 action_type=data.get("action_type"),
580 params=data.get("params", {}),
581 evaluate=evaluate,
582 route=route,
583 on_yes=data.get("on_yes") or data.get("on_success"),
584 on_no=data.get("on_no") or data.get("on_failure"),
585 on_error=data.get("on_error"),
586 on_partial=data.get("on_partial"),
587 on_blocked=data.get("on_blocked"),
588 next=data.get("next"),
589 terminal=data.get("terminal", False),
590 capture=data.get("capture"),
591 append_to_messages=data.get("append_to_messages"),
592 timeout=data.get("timeout"),
593 on_maintain=data.get("on_maintain"),
594 max_retries=data.get("max_retries"),
595 on_retry_exhausted=data.get("on_retry_exhausted"),
596 retryable_exit_codes=data.get("retryable_exit_codes"),
597 max_rate_limit_retries=data.get("max_rate_limit_retries"),
598 on_rate_limit_exhausted=data.get("on_rate_limit_exhausted"),
599 rate_limit_backoff_base_seconds=data.get("rate_limit_backoff_base_seconds"),
600 rate_limit_max_wait_seconds=data.get("rate_limit_max_wait_seconds"),
601 rate_limit_long_wait_ladder=data.get("rate_limit_long_wait_ladder"),
602 loop=data.get("loop"),
603 context_passthrough=data.get("context_passthrough", False),
604 with_=data.get("with", {}),
605 agent=data.get("agent"),
606 tools=data.get("tools"),
607 model=data.get("model"),
608 extra_routes=extra_routes,
609 type=data.get("type"),
610 throttle=throttle,
611 on_throttle_hard=data.get("on_throttle_hard"),
612 learning=learning,
613 fragment_name=data.get("fragment_name"),
614 fragment_bindings=data.get("fragment_bindings", {}),
615 fragment_parameters={
616 name: ParameterSpec.from_dict(spec)
617 for name, spec in data.get("fragment_parameters", {}).items()
618 },
619 )
621 def get_referenced_states(self) -> set[str]:
622 """Get all state names referenced by this state configuration.
624 Returns:
625 Set of state names that this state can transition to.
626 """
627 refs: set[str] = set()
629 if self.on_yes is not None:
630 refs.add(self.on_yes)
631 if self.on_no is not None:
632 refs.add(self.on_no)
633 if self.on_error is not None:
634 refs.add(self.on_error)
635 if self.on_partial is not None:
636 refs.add(self.on_partial)
637 if self.on_blocked is not None:
638 refs.add(self.on_blocked)
639 if self.next is not None:
640 refs.add(self.next)
641 if self.on_maintain is not None:
642 refs.add(self.on_maintain)
643 if self.on_retry_exhausted is not None:
644 refs.add(self.on_retry_exhausted)
645 if self.on_rate_limit_exhausted is not None:
646 refs.add(self.on_rate_limit_exhausted)
647 if self.on_throttle_hard is not None:
648 refs.add(self.on_throttle_hard)
649 if self.route is not None:
650 refs.update(self.route.routes.values())
651 if self.route.default is not None:
652 refs.add(self.route.default)
653 if self.route.error is not None:
654 refs.add(self.route.error)
655 refs.update(self.extra_routes.values())
657 return refs
660@dataclass
661class LLMConfig:
662 """LLM evaluation configuration.
664 Settings for the llm_structured evaluator.
666 Attributes:
667 enabled: If False, disable LLM evaluation entirely
668 model: Model identifier for LLM calls
669 max_tokens: Maximum tokens for evaluation response
670 timeout: Timeout for LLM calls in seconds
671 """
673 enabled: bool = True
674 model: str = DEFAULT_LLM_MODEL
675 max_tokens: int = 256
676 timeout: int = 1800
678 def to_dict(self) -> dict[str, Any]:
679 """Convert to dictionary for JSON/YAML serialization."""
680 result: dict[str, Any] = {}
682 if not self.enabled:
683 result["enabled"] = self.enabled
684 if self.model != DEFAULT_LLM_MODEL:
685 result["model"] = self.model
686 if self.max_tokens != 256:
687 result["max_tokens"] = self.max_tokens
688 if self.timeout != 1800:
689 result["timeout"] = self.timeout
691 return result if result else {}
693 @classmethod
694 def from_dict(cls, data: dict[str, Any]) -> LLMConfig:
695 """Create from dictionary (JSON/YAML deserialization)."""
696 return cls(
697 enabled=data.get("enabled", True),
698 model=data.get("model", DEFAULT_LLM_MODEL),
699 max_tokens=data.get("max_tokens", 256),
700 timeout=data.get("timeout", 1800),
701 )
704@dataclass
705class LoopConfigOverrides:
706 """Per-loop ll-config overrides embedded in the loop YAML definition.
708 All fields are optional (None = use global ll-config default).
709 Precedence: CLI flags > YAML config block > global ll-config > schema defaults.
711 Attributes:
712 handoff_threshold: Override for LL_HANDOFF_THRESHOLD env var (1-100)
713 readiness_threshold: Override for commands.confidence_gate.readiness_threshold (1-100)
714 outcome_threshold: Override for commands.confidence_gate.outcome_threshold (1-100)
715 max_continuations: Override for automation.max_continuations (>=1)
716 """
718 handoff_threshold: int | None = None
719 readiness_threshold: int | None = None
720 outcome_threshold: int | None = None
721 max_continuations: int | None = None
723 def to_dict(self) -> dict[str, Any]:
724 """Convert to dictionary for JSON/YAML serialization (skip-if-None)."""
725 result: dict[str, Any] = {}
727 if self.handoff_threshold is not None:
728 result["handoff_threshold"] = self.handoff_threshold
730 confidence_gate: dict[str, Any] = {}
731 if self.readiness_threshold is not None:
732 confidence_gate["readiness_threshold"] = self.readiness_threshold
733 if self.outcome_threshold is not None:
734 confidence_gate["outcome_threshold"] = self.outcome_threshold
735 if confidence_gate:
736 result["commands"] = {"confidence_gate": confidence_gate}
738 if self.max_continuations is not None:
739 result["automation"] = {"max_continuations": self.max_continuations}
741 return result
743 @classmethod
744 def from_dict(cls, data: dict[str, Any]) -> LoopConfigOverrides:
745 """Create from dictionary (JSON/YAML deserialization)."""
746 commands = data.get("commands", {})
747 confidence_gate = commands.get("confidence_gate", {}) if isinstance(commands, dict) else {}
748 automation = data.get("automation", {})
749 continuation = data.get("continuation", {})
751 max_continuations = None
752 if isinstance(automation, dict) and "max_continuations" in automation:
753 max_continuations = automation["max_continuations"]
754 elif isinstance(continuation, dict) and "max_continuations" in continuation:
755 max_continuations = continuation["max_continuations"]
757 return cls(
758 handoff_threshold=data.get("handoff_threshold"),
759 readiness_threshold=confidence_gate.get("readiness_threshold")
760 if isinstance(confidence_gate, dict)
761 else None,
762 outcome_threshold=confidence_gate.get("outcome_threshold")
763 if isinstance(confidence_gate, dict)
764 else None,
765 max_continuations=max_continuations,
766 )
769@dataclass
770class CommandEntry:
771 """A single command entry for the loop's Commands display section.
773 Attributes:
774 cmd: Full command string to display (e.g., "ll-loop run my-loop --param x=1")
775 comment: Short description shown as a comment (e.g., "run with parameter x")
776 """
778 cmd: str
779 comment: str
782@dataclass
783class TargetStateSpec:
784 """Per-state targeting specification for harness-optimize APO (ENH-1552).
786 Names a single FSM state inside a target loop file and associates it with
787 the examples file and eval fragment used during that state's optimization.
789 Attributes:
790 name: State name within the target loop
791 examples_file: Path to the examples YAML file for this state
792 eval_fragment: Eval fragment identifier used during optimization
793 """
795 name: str
796 examples_file: str
797 eval_fragment: str
799 def to_dict(self) -> dict[str, Any]:
800 """Convert to dictionary for JSON/YAML serialization."""
801 return {
802 "name": self.name,
803 "examples_file": self.examples_file,
804 "eval": self.eval_fragment,
805 }
807 @classmethod
808 def from_dict(cls, data: dict[str, Any]) -> TargetStateSpec:
809 """Create from dictionary (JSON/YAML deserialization)."""
810 return cls(
811 name=data["name"],
812 examples_file=data["examples_file"],
813 eval_fragment=data["eval"],
814 )
817@dataclass
818class TargetFileSpec:
819 """Per-file targeting specification for harness-optimize APO (ENH-1552).
821 Associates a loop YAML file (or glob pattern) with the list of states
822 to optimize within that file.
824 Attributes:
825 file: Explicit path to a loop YAML file (mutually exclusive with glob)
826 glob: Glob pattern matching one or more loop YAML files
827 states: States within the matched file(s) to target
828 """
830 file: str | None = None
831 glob: str | None = None
832 states: list[TargetStateSpec] = field(default_factory=list)
834 def to_dict(self) -> dict[str, Any]:
835 """Convert to dictionary for JSON/YAML serialization."""
836 result: dict[str, Any] = {}
837 if self.file is not None:
838 result["file"] = self.file
839 if self.glob is not None:
840 result["glob"] = self.glob
841 if self.states:
842 result["states"] = [s.to_dict() for s in self.states]
843 return result
845 @classmethod
846 def from_dict(cls, data: dict[str, Any]) -> TargetFileSpec:
847 """Create from dictionary (JSON/YAML deserialization)."""
848 return cls(
849 file=data.get("file"),
850 glob=data.get("glob"),
851 states=[TargetStateSpec.from_dict(s) for s in (data.get("states") or [])],
852 )
855@dataclass
856class RepeatedFailureConfig:
857 """Configuration for the FSM stall detector (FEAT-1637).
859 When N consecutive iterations produce an identical
860 `(state_name, exit_code, eval_verdict)` triple, the FSM either
861 aborts the run or routes to a configured recovery state.
863 Attributes:
864 window: Consecutive iterations with identical triple required to
865 fire (default 3).
866 on_repeated_failure: Either the literal ``"abort"`` (terminate
867 with ``terminated_by="stall_detected"``) or the name of a
868 declared state to route to.
869 """
871 window: int = 3
872 on_repeated_failure: str = "abort"
873 progress_paths: list[str] = field(default_factory=list)
874 exclude_paths: list[str] = field(default_factory=list)
876 def to_dict(self) -> dict[str, Any]:
877 """Convert to dictionary for JSON/YAML serialization (skip-if-default)."""
878 result: dict[str, Any] = {}
879 if self.window != 3:
880 result["window"] = self.window
881 if self.on_repeated_failure != "abort":
882 result["on_repeated_failure"] = self.on_repeated_failure
883 if self.progress_paths:
884 result["progress_paths"] = self.progress_paths
885 if self.exclude_paths:
886 result["exclude_paths"] = self.exclude_paths
887 return result
889 @classmethod
890 def from_dict(cls, data: dict[str, Any]) -> RepeatedFailureConfig:
891 """Create from dictionary (JSON/YAML deserialization)."""
892 return cls(
893 window=data.get("window", 3),
894 on_repeated_failure=data.get("on_repeated_failure", "abort"),
895 progress_paths=data.get("progress_paths", []),
896 exclude_paths=data.get("exclude_paths", []),
897 )
900@dataclass
901class CircuitConfig:
902 """Top-level ``circuit:`` block grouping loop-level safety knobs.
904 Currently exposes ``repeated_failure`` (the stall detector). Future
905 safety knobs (e.g. global timeouts, panic-stop guards) should be
906 added here rather than as separate top-level keys.
908 Attributes:
909 repeated_failure: Stall-detector configuration, or None to disable.
910 """
912 repeated_failure: RepeatedFailureConfig | None = None
914 def to_dict(self) -> dict[str, Any]:
915 """Convert to dictionary for JSON/YAML serialization (skip-if-None)."""
916 result: dict[str, Any] = {}
917 if self.repeated_failure is not None:
918 rf_dict = self.repeated_failure.to_dict()
919 # Always emit the key when configured (even if all fields are defaults)
920 # so the round-trip preserves "repeated_failure was set" intent.
921 result["repeated_failure"] = rf_dict
922 return result
924 @classmethod
925 def from_dict(cls, data: dict[str, Any]) -> CircuitConfig:
926 """Create from dictionary (JSON/YAML deserialization)."""
927 rf = None
928 if "repeated_failure" in data and data["repeated_failure"] is not None:
929 rf = RepeatedFailureConfig.from_dict(data["repeated_failure"])
930 return cls(repeated_failure=rf)
933@dataclass
934class FSMLoop:
935 """Complete FSM loop definition.
937 The main dataclass representing a loop configuration.
939 Attributes:
940 name: Unique loop identifier
941 initial: Starting state name
942 states: Mapping from state name to StateConfig
943 context: User-defined shared variables
944 scope: Paths this loop operates on (for concurrency control). Supports ${context.<var>} template variables that are resolved at runtime.
945 max_iterations: Safety limit for loop iterations
946 backoff: Seconds between iterations
947 timeout: Max total runtime in seconds (loop-level)
948 maintain: If True, restart after completion
949 llm: LLM evaluation configuration
950 on_handoff: Behavior when handoff signal detected (pause/spawn/terminate)
951 commands: Optional override for the Commands section in ll-loop show
952 """
954 name: str
955 initial: str
956 states: dict[str, StateConfig]
957 description: str | None = None
958 context: dict[str, Any] = field(default_factory=dict)
959 parameters: dict[str, ParameterSpec] = field(default_factory=dict)
960 scope: list[str] = field(default_factory=list)
961 max_iterations: int = 50
962 on_max_iterations: str | None = None
963 max_edge_revisits: int = 100
964 backoff: float | None = None
965 timeout: int | None = None
966 default_timeout: int | None = None
967 maintain: bool = False
968 llm: LLMConfig = field(default_factory=LLMConfig)
969 on_handoff: Literal["pause", "spawn", "terminate"] = "pause"
970 input_key: str = "input"
971 config: LoopConfigOverrides | None = None
972 category: str = ""
973 labels: list[str] = field(default_factory=list)
974 # Audience axis for `ll-loop list` filtering (orthogonal to `category`, which
975 # is topical). "public" = user-facing (default); "internal" = delegated-only
976 # sub-loop, never run directly; "example" = demo/template. See ENH (loop-list
977 # visibility tiers).
978 visibility: str = "public"
979 required_inputs: list[str] = field(default_factory=list)
980 commands: list[CommandEntry] = field(default_factory=list)
981 targets: list[TargetFileSpec] = field(default_factory=list)
982 circuit: CircuitConfig | None = None
983 meta_self_eval_ok: bool = False
984 shared_state_ok: bool = False
985 partial_route_ok: bool = False
986 artifact_versioning: bool = False
987 artifact_versioning_ok: bool = False
988 generator_fix_ok: bool = False
989 # Populated from the raw `import:` list by from_dict(); not serialized by to_dict()
990 imports: list[str] = field(default_factory=list)
992 def to_dict(self) -> dict[str, Any]:
993 """Convert to dictionary for JSON/YAML serialization."""
994 result: dict[str, Any] = {
995 "name": self.name,
996 "initial": self.initial,
997 "states": {name: state.to_dict() for name, state in self.states.items()},
998 }
1000 if self.description is not None:
1001 result["description"] = self.description
1002 if self.context:
1003 result["context"] = self.context
1004 if self.parameters:
1005 result["parameters"] = {name: spec.to_dict() for name, spec in self.parameters.items()}
1006 if self.scope:
1007 result["scope"] = self.scope
1008 if self.max_iterations != 50:
1009 result["max_iterations"] = self.max_iterations
1010 if self.max_edge_revisits != 100:
1011 result["max_edge_revisits"] = self.max_edge_revisits
1012 if self.backoff is not None:
1013 result["backoff"] = self.backoff
1014 if self.timeout is not None:
1015 result["timeout"] = self.timeout
1016 if self.default_timeout is not None:
1017 result["default_timeout"] = self.default_timeout
1018 if self.maintain:
1019 result["maintain"] = self.maintain
1020 if self.on_handoff != "pause":
1021 result["on_handoff"] = self.on_handoff
1022 if self.on_max_iterations is not None:
1023 result["on_max_iterations"] = self.on_max_iterations
1024 if self.input_key != "input":
1025 result["input_key"] = self.input_key
1027 llm_dict = self.llm.to_dict()
1028 if llm_dict:
1029 result["llm"] = llm_dict
1031 if self.config is not None:
1032 config_dict = self.config.to_dict()
1033 if config_dict:
1034 result["config"] = config_dict
1036 if self.category:
1037 result["category"] = self.category
1038 if self.labels:
1039 result["labels"] = self.labels
1040 if self.visibility and self.visibility != "public":
1041 result["visibility"] = self.visibility
1042 if self.required_inputs:
1043 result["required_inputs"] = self.required_inputs
1044 if self.commands:
1045 result["commands"] = [{"cmd": e.cmd, "comment": e.comment} for e in self.commands]
1046 if self.targets:
1047 result["targets"] = [t.to_dict() for t in self.targets]
1049 if self.circuit is not None:
1050 circuit_dict = self.circuit.to_dict()
1051 if circuit_dict:
1052 result["circuit"] = circuit_dict
1054 if self.meta_self_eval_ok:
1055 result["meta_self_eval_ok"] = self.meta_self_eval_ok
1056 if self.shared_state_ok:
1057 result["shared_state_ok"] = self.shared_state_ok
1058 if self.partial_route_ok:
1059 result["partial_route_ok"] = self.partial_route_ok
1060 if self.artifact_versioning:
1061 result["artifact_versioning"] = self.artifact_versioning
1062 if self.artifact_versioning_ok:
1063 result["artifact_versioning_ok"] = self.artifact_versioning_ok
1064 if self.generator_fix_ok:
1065 result["generator_fix_ok"] = self.generator_fix_ok
1067 return result
1069 @classmethod
1070 def from_dict(cls, data: dict[str, Any]) -> FSMLoop:
1071 """Create from dictionary (JSON/YAML deserialization)."""
1072 states = {
1073 name: StateConfig.from_dict(state_data)
1074 for name, state_data in data.get("states", {}).items()
1075 }
1077 llm = LLMConfig()
1078 if "llm" in data:
1079 llm = LLMConfig.from_dict(data["llm"])
1081 loop_config = None
1082 if "config" in data:
1083 loop_config = LoopConfigOverrides.from_dict(data["config"])
1085 circuit = None
1086 if "circuit" in data and data["circuit"] is not None:
1087 circuit = CircuitConfig.from_dict(data["circuit"])
1089 parameters = {
1090 name: ParameterSpec.from_dict(spec) for name, spec in data.get("parameters", {}).items()
1091 }
1093 return cls(
1094 name=data["name"],
1095 initial=data["initial"],
1096 states=states,
1097 description=data.get("description"),
1098 context=data.get("context", {}),
1099 parameters=parameters,
1100 scope=data.get("scope", []),
1101 max_iterations=data.get("max_iterations", 50),
1102 on_max_iterations=data.get("on_max_iterations"),
1103 max_edge_revisits=data.get("max_edge_revisits", 100),
1104 backoff=data.get("backoff"),
1105 timeout=data.get("timeout"),
1106 default_timeout=data.get("default_timeout"),
1107 maintain=data.get("maintain", False),
1108 llm=llm,
1109 on_handoff=data.get("on_handoff", "pause"),
1110 input_key=data.get("input_key", "input"),
1111 config=loop_config,
1112 category=data.get("category", ""),
1113 labels=data.get("labels", []),
1114 visibility=data.get("visibility", "public"),
1115 required_inputs=data.get("required_inputs", []),
1116 commands=[CommandEntry(**e) for e in data.get("commands", [])],
1117 targets=[TargetFileSpec.from_dict(t) for t in (data.get("targets") or [])],
1118 circuit=circuit,
1119 meta_self_eval_ok=data.get("meta_self_eval_ok", False),
1120 shared_state_ok=data.get("shared_state_ok", False),
1121 partial_route_ok=data.get("partial_route_ok", False),
1122 artifact_versioning=data.get("artifact_versioning", False),
1123 artifact_versioning_ok=data.get("artifact_versioning_ok", False),
1124 generator_fix_ok=data.get("generator_fix_ok", False),
1125 imports=data.get("import", []),
1126 )
1128 def get_all_state_names(self) -> set[str]:
1129 """Get all defined state names.
1131 Returns:
1132 Set of all state names in this FSM.
1133 """
1134 return set(self.states.keys())
1136 def get_terminal_states(self) -> set[str]:
1137 """Get all terminal state names.
1139 Returns:
1140 Set of state names where terminal=True.
1141 """
1142 return {name for name, state in self.states.items() if state.terminal}
1144 def get_all_referenced_states(self) -> set[str]:
1145 """Get all state names referenced by transitions.
1147 This includes the initial state and all states referenced
1148 in routing configurations.
1150 Returns:
1151 Set of all referenced state names.
1152 """
1153 refs: set[str] = {self.initial}
1154 for state in self.states.values():
1155 refs.update(state.get_referenced_states())
1156 if self.on_max_iterations is not None:
1157 refs.add(self.on_max_iterations)
1158 return refs