Coverage for little_loops / cli / loop / lifecycle.py: 0%
428 statements
« prev ^ index » next coverage.py v7.12.0, created at 2026-06-04 12:21 -0500
1"""ll-loop lifecycle subcommands: status, stop, resume."""
3from __future__ import annotations
5import argparse
6import atexit
7import json
8import os
9import signal
10import time
11from pathlib import Path
13from little_loops.cli.loop._helpers import (
14 load_loop,
15 register_loop_signal_handlers,
16 resolve_loop_path,
17 run_background,
18)
19from little_loops.fsm.concurrency import _process_alive
20from little_loops.fsm.persistence import (
21 LoopState,
22 StatePersistence,
23 _find_instances,
24 _read_pid_file,
25 _reconcile_stale_running,
26)
27from little_loops.logger import Logger
30def _format_relative_time(seconds: float) -> str:
31 """Format seconds as a human-readable relative time string (e.g., '3m ago').
33 Delegates to the shared ``format_relative_time`` in ``cli.output``.
34 """
35 from little_loops.cli.output import format_relative_time
37 return format_relative_time(seconds)
40def _format_log_label(running_dir: Path, stem: str) -> str:
41 """Return one of three Log: labels based on run-mode signals.
43 - log exists → path string (background run, normal)
44 - no log, no pid → foreground run (output went to terminal)
45 - no log, pid exists → background run whose log was deleted (something wrong)
46 """
47 log_file = running_dir / f"{stem}.log"
48 if log_file.exists():
49 return str(log_file)
50 pid_file = running_dir / f"{stem}.pid"
51 if pid_file.exists():
52 return f"(expected {log_file}, missing)"
53 return "(foreground run — output went to terminal)"
56def _get_events_info(running_dir: Path, stem: str) -> tuple[str | None, str | None]:
57 """Return (events_file_str, formatted_detail_line) for the events.jsonl file.
59 Returns (None, None) if the file doesn't exist.
60 Detail line format: 'Events: <path> (N events, last Xm ago)'
61 """
62 events_file = running_dir / f"{stem}.events.jsonl"
63 if not events_file.exists():
64 return None, None
65 try:
66 lines = [ln for ln in events_file.read_text().splitlines() if ln.strip()]
67 count = len(lines)
68 detail = f"({count} events)"
69 if lines:
70 try:
71 last_entry = json.loads(lines[-1])
72 last_ts = last_entry.get("ts")
73 if last_ts is not None:
74 from datetime import UTC, datetime
76 dt = datetime.fromisoformat(last_ts)
77 age = (datetime.now(tz=UTC) - dt).total_seconds()
78 detail = f"({count} events, last {_format_relative_time(age)})"
79 except (json.JSONDecodeError, AttributeError, ValueError, OverflowError):
80 pass
81 return str(events_file), f"Events: {events_file} {detail}"
82 except OSError:
83 return str(events_file), f"Events: {events_file}"
86def _kill_with_timeout(pid: int, label: str, logger: Logger) -> None:
87 """Send SIGTERM to pid; escalate to SIGKILL after 10 s if still alive."""
88 os.kill(pid, signal.SIGTERM)
89 for _ in range(10):
90 time.sleep(1)
91 if not _process_alive(pid):
92 return
93 try:
94 os.kill(pid, signal.SIGKILL)
95 logger.warning(f"Sent SIGKILL to {label} (PID: {pid})")
96 except OSError:
97 pass
100def _status_single(
101 instance_id: str | None,
102 state: LoopState,
103 loop_name: str,
104 running_dir: Path,
105 args: argparse.Namespace | None,
106) -> int:
107 """Render status for one instance (human-readable or JSON)."""
108 from little_loops.cli.output import print_json
110 stem = instance_id or loop_name
111 persistence = StatePersistence(loop_name, running_dir.parent, instance_id=instance_id)
112 state = _reconcile_stale_running(state, persistence, running_dir, stem)
114 pid_file = running_dir / f"{stem}.pid"
115 pid = _read_pid_file(pid_file)
116 pid_source: str | None = "pid_file" if pid is not None else None
118 if pid is None:
119 lock_file_path = running_dir / f"{stem}.lock"
120 if lock_file_path.exists():
121 try:
122 with open(lock_file_path) as _lf:
123 lock_data = json.load(_lf)
124 pid = lock_data.get("pid")
125 if pid is not None:
126 pid_source = "lock_file"
127 except (json.JSONDecodeError, KeyError, OSError):
128 pass
130 log_file = running_dir / f"{stem}.log"
131 log_file_str: str | None = None
132 log_updated_ago: str | None = None
133 last_event: str | None = None
134 if log_file.exists():
135 log_file_str = str(log_file)
136 age_seconds = time.time() - log_file.stat().st_mtime
137 log_updated_ago = _format_relative_time(age_seconds)
138 try:
139 lines = log_file.read_text().splitlines()
140 non_empty = [ln for ln in lines if ln.strip()]
141 last_event = non_empty[-1] if non_empty else None
142 except OSError:
143 last_event = None
145 events_file_str, events_detail_line = _get_events_info(running_dir, stem)
147 if getattr(args, "json", False):
148 d = state.to_dict()
149 d["pid"] = pid
150 d["pid_source"] = pid_source
151 d["log_file"] = log_file_str
152 d["log_updated_ago"] = log_updated_ago
153 d["last_event"] = last_event
154 d["events_file"] = events_file_str
155 print_json(d)
156 return 0
158 print(f"Loop: {state.loop_name}")
159 print(f"Status: {state.status}")
160 print(f"Current state: {state.current_state}")
161 print(f"Iteration: {state.iteration}")
162 print(f"Started: {state.started_at}")
163 print(f"Updated: {state.updated_at}")
165 if pid is not None:
166 if _process_alive(pid):
167 print(f"PID: {pid} (running)")
168 else:
169 print(f"PID: {pid} (not running - stale PID file)")
171 log_label = _format_log_label(running_dir, stem)
172 print(f"Log: {log_label}")
173 if log_file_str is not None:
174 print(f"Log updated: {log_updated_ago}")
175 if last_event:
176 print(f"Last event: {last_event}")
178 if events_detail_line is not None:
179 print(events_detail_line)
181 if state.continuation_prompt:
182 prompt_preview = state.continuation_prompt[:200]
183 if len(state.continuation_prompt) > 200:
184 prompt_preview += "..."
185 print(f"Continuation context: {prompt_preview}")
186 return 0
189def cmd_status(
190 loop_name: str,
191 loops_dir: Path,
192 logger: Logger,
193 args: argparse.Namespace | None = None,
194) -> int:
195 """Show loop status."""
196 from little_loops.cli.output import print_json
198 running_dir = loops_dir / ".running"
199 instances = _find_instances(loop_name, running_dir)
201 if not instances:
202 logger.error(f"No state found for: {loop_name}")
203 return 1
205 if len(instances) == 1:
206 instance_id, state = instances[0]
207 return _status_single(instance_id, state, loop_name, running_dir, args)
209 # Multiple instances: aggregate display
210 if getattr(args, "json", False):
211 result_list = []
212 for instance_id, state in instances:
213 stem = instance_id or loop_name
214 persistence = StatePersistence(loop_name, loops_dir, instance_id=instance_id)
215 state = _reconcile_stale_running(state, persistence, running_dir, stem)
216 pid_file = running_dir / f"{stem}.pid"
217 pid = _read_pid_file(pid_file)
218 pid_source: str | None = "pid_file" if pid is not None else None
219 if pid is None:
220 lock_file_path = running_dir / f"{stem}.lock"
221 if lock_file_path.exists():
222 try:
223 with open(lock_file_path) as _lf:
224 lock_data = json.load(_lf)
225 pid = lock_data.get("pid")
226 if pid is not None:
227 pid_source = "lock_file"
228 except (json.JSONDecodeError, KeyError, OSError):
229 pass
230 log_file = running_dir / f"{stem}.log"
231 d = state.to_dict()
232 d["instance_id"] = instance_id
233 d["pid"] = pid
234 d["pid_source"] = pid_source
235 if log_file.exists():
236 d["log_file"] = str(log_file)
237 d["log_updated_ago"] = _format_relative_time(time.time() - log_file.stat().st_mtime)
238 else:
239 d["log_file"] = None
240 d["log_updated_ago"] = None
241 events_file_str, _ = _get_events_info(running_dir, stem)
242 d["events_file"] = events_file_str
243 result_list.append(d)
244 print_json(result_list)
245 return 0
247 print(f"{len(instances)} instances of '{loop_name}':")
248 for i, (instance_id, state) in enumerate(instances, 1):
249 stem = instance_id or loop_name
250 persistence = StatePersistence(loop_name, loops_dir, instance_id=instance_id)
251 state = _reconcile_stale_running(state, persistence, running_dir, stem)
252 pid_file = running_dir / f"{stem}.pid"
253 pid = _read_pid_file(pid_file)
254 if pid is None:
255 lock_file_path = running_dir / f"{stem}.lock"
256 if lock_file_path.exists():
257 try:
258 with open(lock_file_path) as _lf:
259 lock_data = json.load(_lf)
260 pid = lock_data.get("pid")
261 except (json.JSONDecodeError, KeyError, OSError):
262 pass
263 log_file = running_dir / f"{stem}.log"
265 print()
266 print(f"[{i}] {stem}")
267 print(f" Status: {state.status}")
268 print(f" Current state: {state.current_state}")
269 print(f" Iteration: {state.iteration}")
270 if pid is not None:
271 if _process_alive(pid):
272 print(f" PID: {pid} (running)")
273 else:
274 print(f" PID: {pid} (not running - stale PID file)")
275 log_label = _format_log_label(running_dir, stem)
276 print(f" Log: {log_label}")
277 if log_file.exists():
278 age_seconds = time.time() - log_file.stat().st_mtime
279 print(f" Log updated: {_format_relative_time(age_seconds)}")
280 _, events_detail_line = _get_events_info(running_dir, stem)
281 if events_detail_line is not None:
282 print(f" {events_detail_line}")
283 return 0
286def cmd_stop(
287 loop_name: str,
288 loops_dir: Path,
289 logger: Logger,
290) -> int:
291 """Stop a running loop."""
292 from little_loops.fsm.persistence import StatePersistence # noqa: PLC0415
294 running_dir = loops_dir / ".running"
295 instances = _find_instances(loop_name, running_dir)
297 if not instances:
298 logger.error(f"No state found for: {loop_name}")
299 return 1
301 running_instances = [(iid, s) for iid, s in instances if s.status == "running"]
303 if not running_instances:
304 # Secondary check: an orphaned lock-file with a live PID blocks scope
305 # acquisition even when state is not "running". Kill the holder and release.
306 for instance_id, state in instances:
307 stem = instance_id or loop_name
308 lock_file = running_dir / f"{stem}.lock"
309 if lock_file.exists():
310 lock_pid: int | None = None
311 try:
312 with open(lock_file) as _lf:
313 lock_data = json.load(_lf)
314 lock_pid = lock_data.get("pid")
315 except (json.JSONDecodeError, KeyError, OSError):
316 pass
317 if lock_pid and _process_alive(lock_pid):
318 logger.warning(
319 f"Loop state is '{state.status}' but lock file holds live PID {lock_pid}. "
320 "Killing orphaned lock holder..."
321 )
322 _kill_with_timeout(lock_pid, stem, logger)
323 lock_file.unlink(missing_ok=True)
324 logger.success(f"Released orphaned scope lock for {stem}")
325 return 0
326 elif lock_pid is not None:
327 # Dead PID: stale lock file, just remove it
328 lock_file.unlink(missing_ok=True)
329 logger.info(f"Removed stale lock file for {stem}")
330 return 0
331 _, state = instances[0]
332 logger.error(f"Loop not running: {loop_name} (status: {state.status})")
333 return 1
335 for instance_id, state in running_instances:
336 persistence = StatePersistence(loop_name, loops_dir, instance_id=instance_id)
337 stem = instance_id or loop_name
338 pid_file = running_dir / f"{stem}.pid"
339 pid = _read_pid_file(pid_file)
341 if pid is not None:
342 if _process_alive(pid):
343 _kill_with_timeout(pid, stem, logger)
344 state.status = "interrupted"
345 persistence.save_state(state)
346 pid_file.unlink(missing_ok=True)
347 logger.success(f"Stopped {stem} (PID: {pid})")
348 else:
349 # Process already exited: preserve its final status, only clean up PID file
350 logger.info(f"Process {pid} not running, cleaning up PID file")
351 pid_file.unlink(missing_ok=True)
352 else:
353 # No PID file: no background process tracked, update state only
354 state.status = "interrupted"
355 persistence.save_state(state)
356 logger.success(f"Marked {stem} as interrupted")
358 return 0
361def cmd_resume(
362 loop_name: str,
363 args: argparse.Namespace,
364 loops_dir: Path,
365 logger: Logger,
366) -> int:
367 """Resume an interrupted loop."""
368 import os
370 from little_loops.fsm.persistence import RESUMABLE_STATUSES, PersistentExecutor
372 running_dir = loops_dir / ".running"
373 running_dir.mkdir(parents=True, exist_ok=True)
375 # Discover all instances and resolve to a single resumable one.
376 instances = _find_instances(loop_name, running_dir)
377 resumable = [(iid, s) for iid, s in instances if s.status in RESUMABLE_STATUSES]
379 explicit_instance_id = getattr(args, "instance_id", None)
380 if explicit_instance_id:
381 filtered: list[tuple[str | None, LoopState]] = [
382 (iid, s) for iid, s in resumable if iid == explicit_instance_id
383 ]
384 if not filtered:
385 print(
386 f"Instance '{explicit_instance_id}' not found among resumable instances of '{loop_name}'."
387 )
388 if resumable:
389 print("Resumable instances:")
390 for iid, _ in resumable:
391 print(f" {iid or loop_name}")
392 return 1
393 resumable = filtered
394 elif len(resumable) > 1:
395 # Auto-select the most recent instance by sorted filename (matches cmd_monitor()).
396 selected_id = resumable[-1][0]
397 print(f"Auto-selected latest instance: {selected_id}")
398 resumable = [resumable[-1]]
400 # Use discovered instance_id (or fall back to args / None for no-state case)
401 if resumable:
402 instance_id: str | None = resumable[0][0]
403 state_for_display = resumable[0][1]
404 else:
405 instance_id = None
406 state_for_display = None
408 # Background mode: spawn detached process and return
409 if getattr(args, "background", False):
410 if instance_id is None:
411 print(f"No resumable instances of '{loop_name}'.")
412 return 1
413 return run_background(
414 loop_name, args, loops_dir, subcommand="resume", instance_id=instance_id
415 )
417 # Register PID file for all foreground runs so cmd_stop can send SIGTERM (BUG-639).
418 # Background-spawned processes (foreground_internal=True) have their PID written by the
419 # parent in run_background(); plain foreground runs must write their own PID here.
420 if instance_id is None:
421 instance_id = getattr(args, "instance_id", None)
423 pid_file = running_dir / f"{instance_id or loop_name}.pid"
424 foreground_pid_file: Path | None = pid_file
426 if not getattr(args, "foreground_internal", False):
427 pid_file.parent.mkdir(parents=True, exist_ok=True)
428 pid_file.write_text(str(os.getpid()))
430 def _cleanup_pid() -> None:
431 pid_file.unlink(missing_ok=True)
433 atexit.register(_cleanup_pid)
435 try:
436 fsm = load_loop(loop_name, loops_dir, logger)
437 except FileNotFoundError as e:
438 logger.error(str(e))
439 return 1
440 except ValueError as e:
441 logger.error(f"Validation error: {e}")
442 return 1
444 if getattr(args, "baseline", False):
445 logger.error(
446 "--baseline is not supported with resume. Start a fresh run with 'll-loop run'."
447 )
448 return 1
450 try:
451 loop_path = resolve_loop_path(loop_name, loops_dir)
452 except FileNotFoundError:
453 loop_path = None
455 for kv in getattr(args, "context", None) or []:
456 if "=" not in kv:
457 raise SystemExit(f"Invalid --context format: {kv!r} (expected KEY=VALUE)")
458 key, _, value = kv.partition("=")
459 fsm.context[key.strip()] = value.strip()
461 # Re-inject run_dir using the same instance_id as the original run so resumed
462 # loops write artifacts to the same directory they started with.
463 if "run_dir" not in fsm.context and instance_id is not None:
464 fsm.context["run_dir"] = str(loops_dir / "runs" / instance_id) + "/"
466 if getattr(args, "delay", None) is not None:
467 fsm.backoff = args.delay
469 # Apply YAML loop config env-var overrides (CLI flags below overwrite these)
470 if fsm.config is not None and isinstance(fsm.config.handoff_threshold, int):
471 os.environ["LL_HANDOFF_THRESHOLD"] = str(fsm.config.handoff_threshold)
473 if getattr(args, "handoff_threshold", None) is not None:
474 if not (1 <= args.handoff_threshold <= 100):
475 raise SystemExit("--handoff-threshold must be between 1 and 100")
476 os.environ["LL_HANDOFF_THRESHOLD"] = str(args.handoff_threshold)
478 # Show context if resuming from a handoff
479 if state_for_display and state_for_display.status == "awaiting_continuation":
480 print(f"Resuming from context handoff (iteration {state_for_display.iteration})...")
481 if state_for_display.continuation_prompt:
482 prompt_preview = state_for_display.continuation_prompt[:500]
483 if len(state_for_display.continuation_prompt) > 500:
484 prompt_preview += "..."
485 print(f"Context: {prompt_preview}")
486 print()
488 from little_loops.config import BRConfig
489 from little_loops.design_tokens import load_design_tokens, render_as_prompt_context
490 from little_loops.extension import wire_extensions
491 from little_loops.fsm.rate_limit_circuit import RateLimitCircuit
492 from little_loops.transport import wire_transports
494 config = BRConfig(Path.cwd())
496 if "design_tokens_context" not in fsm.context:
497 _tokens = load_design_tokens(config)
498 fsm.context["design_tokens_context"] = render_as_prompt_context(_tokens) if _tokens else ""
500 circuit = (
501 RateLimitCircuit(Path(config.commands.rate_limits.circuit_breaker_path))
502 if config.commands.rate_limits.circuit_breaker_enabled
503 else None
504 )
505 executor = PersistentExecutor(
506 fsm, loops_dir=loops_dir, circuit=circuit, instance_id=instance_id
507 )
509 # Register signal handlers for graceful shutdown (same as cmd_run)
510 register_loop_signal_handlers(executor, pid_file=foreground_pid_file)
512 wire_extensions(executor.event_bus, config.extensions, executor=executor)
513 wire_transports(executor.event_bus, config.events)
515 # Route through run_foreground (BUG-1645) so the resume path shares the
516 # display-callback wiring that cmd_run gets via run_foreground. Without this
517 # the FSM event bus has no subscriber on a resumed run, leaving the terminal
518 # silent for the entire duration (no per-iteration lines, no FSM diagram).
519 from little_loops.cli.loop._helpers import run_foreground
521 _edge_label_colors = config.cli.colors.fsm_edge_labels.to_dict()
522 _highlight_color = config.cli.colors.fsm_active_state
523 _badges = config.loops.glyphs.to_dict()
525 try:
526 return run_foreground(
527 executor,
528 fsm,
529 args,
530 highlight_color=_highlight_color,
531 edge_label_colors=_edge_label_colors,
532 badges=_badges,
533 mode="resume",
534 instance_id=instance_id,
535 running_dir=running_dir,
536 loop_path=loop_path,
537 model=fsm.llm.model,
538 )
539 finally:
540 executor.close_transports()
543def _print_last_state(state: LoopState) -> None:
544 """Print last-known state (used when not actively tailing)."""
545 print(f"Loop: {state.loop_name}")
546 print(f"Status: {state.status}")
547 print(f"Current state: {state.current_state}")
548 print(f"Iteration: {state.iteration}")
551def cmd_monitor(args: argparse.Namespace, loops_dir: Path) -> int:
552 """Attach to a running loop and render its FSM state in realtime.
554 Read-only attach: tails ``<stem>.events.jsonl`` and forwards events to a
555 ``StateFeedRenderer``. Ctrl-C detaches without sending any signal to the
556 loop process (FEAT-1764).
557 """
558 loop_name = args.loop
559 running_dir = loops_dir / ".running"
560 instances = _find_instances(loop_name, running_dir)
562 if not instances:
563 print(f"No instances of '{loop_name}' found")
564 return 1
566 instance_id, state = instances[-1]
567 stem = instance_id or loop_name
569 pid = _read_pid_file(running_dir / f"{stem}.pid")
570 if pid is None:
571 pid = getattr(state, "pid", None)
573 if pid is None or not _process_alive(pid):
574 _print_last_state(state)
575 return 0
577 # Fabricate missing Namespace attrs that StateFeedRenderer reads.
578 for attr, default in (
579 ("quiet", False),
580 ("verbose", False),
581 ("show_diagrams", None),
582 ("clear", True),
583 ("diagram_edge_labels", None),
584 ("diagram_state_detail", None),
585 ("diagram_scope", None),
586 ("follow", False),
587 ):
588 if not hasattr(args, attr):
589 setattr(args, attr, default)
591 try:
592 fsm = load_loop(loop_name, loops_dir, Logger())
593 except (FileNotFoundError, ValueError) as e:
594 print(f"Cannot load loop '{loop_name}': {e}")
595 return 1
597 try:
598 loop_path = resolve_loop_path(loop_name, loops_dir)
599 except FileNotFoundError:
600 loop_path = None
602 # Late import: tests patch StateFeedRenderer at its module-of-origin
603 # (little_loops.cli.loop._helpers.StateFeedRenderer); using a function-local
604 # import ensures the patch takes effect at call time.
605 from little_loops.cli.loop._helpers import (
606 StateFeedRenderer,
607 _install_sigwinch_handler,
608 _restore_sigwinch_handler,
609 )
611 renderer = StateFeedRenderer(
612 fsm, args, loops_dir=loops_dir, loop_path=loop_path, model=fsm.llm.model
613 )
615 events_file = running_dir / f"{stem}.events.jsonl"
617 if not events_file.exists():
618 _print_last_state(state)
619 return 0
621 sigwinch_installed = False
622 if renderer.in_pinned_mode:
623 _install_sigwinch_handler()
624 sigwinch_installed = True
626 try:
627 with open(events_file, encoding="utf-8") as ev_f:
628 ev_f.seek(0, 2)
629 while True:
630 progressed = False
631 try:
632 line = ev_f.readline()
633 except FileNotFoundError:
634 break
635 if line:
636 progressed = True
637 stripped = line.strip()
638 if stripped:
639 try:
640 event = json.loads(stripped)
641 except json.JSONDecodeError:
642 event = None
643 if event is not None:
644 renderer.handle_event(event)
645 if not progressed:
646 if not _process_alive(pid):
647 break
648 if not events_file.exists():
649 break
650 time.sleep(0.1)
651 except KeyboardInterrupt:
652 return 0
653 finally:
654 if sigwinch_installed:
655 _restore_sigwinch_handler()
657 return 0