Coverage for little_loops / cli / sprint / run.py: 9%
350 statements
« prev ^ index » next coverage.py v7.12.0, created at 2026-06-15 17:30 -0500
1"""ll-sprint run subcommand with signal handling and state management."""
3from __future__ import annotations
5import os
6import signal
7import subprocess
8import sys
9import time
10from pathlib import Path
11from types import FrameType
12from typing import TYPE_CHECKING
14from little_loops.cli.output import use_color_enabled
15from little_loops.cli.sprint._helpers import _build_issue_contents, _render_dependency_analysis
16from little_loops.cli_args import _id_matches, parse_issue_ids, parse_issue_types, parse_labels
17from little_loops.dependency_graph import DependencyGraph, refine_waves_for_contention
18from little_loops.issue_manager import IssueProcessingResult, process_issue_inplace
19from little_loops.logger import Logger, format_duration
20from little_loops.parallel.orchestrator import ParallelOrchestrator
21from little_loops.parallel.types import SprintWorkerContext
22from little_loops.sprint import SprintManager, SprintState
24if TYPE_CHECKING:
25 import argparse
27 from little_loops.config import BRConfig
28 from little_loops.issue_parser import IssueInfo
30# Module-level shutdown flag for ll-sprint signal handling (ENH-183)
31_sprint_shutdown_requested: bool = False
32_sprint_logger: Logger | None = None
35class IssueWallClockTimeout(Exception):
36 """Raised when an issue exceeds its per-issue wall-clock time budget (BUG-2144)."""
38 def __init__(self, issue_id: str) -> None:
39 self.issue_id = issue_id
40 super().__init__(f"{issue_id}: wall-clock timeout")
43def _run_issue_with_wall_clock_timeout(
44 issue: IssueInfo,
45 config: BRConfig,
46 logger: Logger,
47 dry_run: bool,
48 max_seconds: int,
49 sprint_context: SprintWorkerContext | None = None,
50) -> IssueProcessingResult:
51 """Wrap process_issue_inplace with a SIGALRM-based wall-clock timeout.
53 If the issue takes longer than max_seconds (across all subprocess spawns,
54 including Option J continuations), SIGALRM fires, IssueWallClockTimeout is
55 raised, and a failed IssueProcessingResult with failure_reason=WALL_CLOCK_TIMEOUT
56 is returned. The alarm is always cleared in the finally block.
57 """
59 def _timeout_handler(signum: int, frame: FrameType | None) -> None:
60 raise IssueWallClockTimeout(issue.issue_id)
62 prev_handler = signal.signal(signal.SIGALRM, _timeout_handler)
63 signal.alarm(max_seconds)
64 issue_start = time.monotonic()
65 try:
66 return process_issue_inplace(
67 info=issue,
68 config=config,
69 logger=logger,
70 dry_run=dry_run,
71 sprint_context=sprint_context,
72 )
73 except IssueWallClockTimeout:
74 elapsed = time.monotonic() - issue_start
75 logger.error(
76 f" {issue.issue_id}: wall-clock timeout after {int(elapsed)}s "
77 f"(limit {max_seconds}s) — marking failed"
78 )
79 return IssueProcessingResult(
80 success=False,
81 duration=elapsed,
82 issue_id=issue.issue_id,
83 failure_reason="WALL_CLOCK_TIMEOUT",
84 )
85 finally:
86 signal.alarm(0)
87 signal.signal(signal.SIGALRM, prev_handler)
90def _detect_current_branch() -> str:
91 """Return the current git branch name, defaulting to 'main' on failure."""
92 result = subprocess.run(
93 ["git", "rev-parse", "--abbrev-ref", "HEAD"],
94 capture_output=True,
95 text=True,
96 )
97 return result.stdout.strip() if result.returncode == 0 else "main"
100def _sprint_signal_handler(signum: int, frame: FrameType | None) -> None:
101 """Handle shutdown signals gracefully for ll-sprint.
103 First signal: Set shutdown flag for graceful exit after current wave.
104 Second signal: Force immediate exit.
105 """
106 global _sprint_shutdown_requested
107 if _sprint_shutdown_requested:
108 # Second signal - force exit
109 msg = "\nForce shutdown requested"
110 if _sprint_logger is not None:
111 _sprint_logger.warning(msg)
112 else:
113 print(msg, file=sys.stderr)
114 sys.exit(1)
115 _sprint_shutdown_requested = True
116 msg = "\nShutdown requested, will exit after current wave..."
117 if _sprint_logger is not None:
118 _sprint_logger.warning(msg)
119 else:
120 print(msg, file=sys.stderr)
123def _get_sprint_state_file() -> Path:
124 """Get path to sprint state file."""
125 return Path.cwd() / ".sprint-state.json"
128def _load_sprint_state(logger: Logger) -> SprintState | None:
129 """Load sprint state from file."""
130 import json
132 state_file = _get_sprint_state_file()
133 if not state_file.exists():
134 return None
135 try:
136 data = json.loads(state_file.read_text())
137 state = SprintState.from_dict(data)
138 logger.info(f"State loaded from {state_file}")
139 return state
140 except (json.JSONDecodeError, KeyError) as e:
141 logger.warning(f"Failed to load state: {e}")
142 return None
145def _save_sprint_state(state: SprintState, logger: Logger) -> None:
146 """Save sprint state to file."""
147 import json
148 from datetime import datetime
150 state.last_checkpoint = datetime.now().isoformat()
151 state_file = _get_sprint_state_file()
152 state_file.write_text(json.dumps(state.to_dict(), indent=2))
153 logger.info(f"State saved to {state_file}")
156def _cleanup_sprint_state(logger: Logger) -> None:
157 """Remove sprint state file."""
158 state_file = _get_sprint_state_file()
159 if state_file.exists():
160 state_file.unlink()
161 logger.info("Sprint state file cleaned up")
164def _cmd_sprint_run(
165 args: argparse.Namespace,
166 manager: SprintManager,
167 config: BRConfig,
168) -> int:
169 """Execute a sprint with dependency-aware scheduling."""
170 from datetime import datetime
172 global _sprint_logger
173 logger = Logger(verbose=not args.quiet, use_color=use_color_enabled())
174 _sprint_logger = logger
176 # Setup signal handlers for graceful shutdown (ENH-183)
177 global _sprint_shutdown_requested
178 _sprint_shutdown_requested = False # Reset in case of multiple runs
179 signal.signal(signal.SIGINT, _sprint_signal_handler)
180 signal.signal(signal.SIGTERM, _sprint_signal_handler)
182 handoff_threshold = getattr(args, "handoff_threshold", None)
183 if handoff_threshold is not None:
184 os.environ["LL_HANDOFF_THRESHOLD"] = str(handoff_threshold)
186 context_limit = getattr(args, "context_limit", None)
187 if context_limit is not None:
188 os.environ["LL_CONTEXT_LIMIT"] = str(context_limit)
190 sprint = manager.load_or_resolve(args.sprint)
191 if not sprint:
192 logger.error(f"Sprint not found: {args.sprint}")
193 return 1
195 if not sprint.issues:
196 logger.info(f"Sprint '{sprint.name}' has no active issues — nothing to process")
197 return 0
199 if getattr(args, "save", False):
200 saved_path = sprint.save(manager.sprints_dir)
201 logger.info(f"Sprint saved to {saved_path}")
203 # Apply skip filter if provided
204 issues_to_process = list(sprint.issues)
205 skip_ids = parse_issue_ids(args.skip)
206 if skip_ids:
207 original_count = len(issues_to_process)
208 issues_to_process = [i for i in issues_to_process if i not in skip_ids]
209 skipped = original_count - len(issues_to_process)
210 if skipped > 0:
211 logger.info(f"Skipping {skipped} issue(s): {', '.join(sorted(skip_ids))}")
213 # Apply only filter if provided
214 only_ids = parse_issue_ids(getattr(args, "only", None))
215 if only_ids:
216 invalid_only = {p for p in only_ids if not any(_id_matches(i, p) for i in sprint.issues)}
217 if invalid_only:
218 logger.error(
219 f"Issue(s) not found in sprint definition: {', '.join(sorted(invalid_only))}"
220 )
221 return 1
222 issues_to_process = [
223 i for i in issues_to_process if any(_id_matches(i, p) for p in only_ids)
224 ]
225 logger.info(
226 f"Processing only {len(issues_to_process)} issue(s): {', '.join(sorted(only_ids))}"
227 )
229 # Apply type filter if provided
230 type_prefixes = parse_issue_types(getattr(args, "type", None))
231 if type_prefixes:
232 original_count = len(issues_to_process)
233 issues_to_process = [i for i in issues_to_process if i.split("-", 1)[0] in type_prefixes]
234 filtered = original_count - len(issues_to_process)
235 if filtered > 0:
236 logger.info(f"Filtered {filtered} issue(s) by type: {', '.join(sorted(type_prefixes))}")
238 # Pre-validate: skip issues with status: done/cancelled (ENH-1424)
239 pre_completed_skipped: list[str] = []
241 # Validate issues exist
242 valid = manager.validate_issues(issues_to_process)
243 invalid = set(issues_to_process) - set(valid.keys())
245 if invalid:
246 logger.error(f"Issue IDs not found: {', '.join(sorted(invalid))}")
247 logger.info("Cannot execute sprint with missing issues")
248 return 1
250 # Write milestone: field back to issue files for all valid issues in this sprint
251 from little_loops.frontmatter import parse_frontmatter, update_frontmatter
253 for _issue_id, path in valid.items():
254 content = path.read_text(encoding="utf-8")
255 updated = update_frontmatter(content, {"milestone": sprint.name})
256 if updated != content:
257 path.write_text(updated, encoding="utf-8")
258 logger.info(f"Set milestone: {sprint.name!r} on {len(valid)} issue file(s)")
260 # Skip issues already completed via frontmatter status (ENH-1424)
261 still_active: list[str] = []
262 for issue_id in issues_to_process:
263 path = valid[issue_id]
264 fm = parse_frontmatter(path.read_text(encoding="utf-8"))
265 if fm.get("status", "open") in ("done", "cancelled"):
266 logger.info(f" {issue_id}: already completed (status: {fm.get('status')}), skipping")
267 pre_completed_skipped.append(issue_id)
268 else:
269 still_active.append(issue_id)
270 if pre_completed_skipped:
271 logger.info(
272 f"Pre-validation: {len(pre_completed_skipped)} issue(s) already completed, "
273 f"{len(still_active)} active"
274 )
275 issues_to_process = still_active
277 if pre_completed_skipped and not issues_to_process:
278 logger.info("All sprint issues already completed - nothing to process")
279 return 0
281 # Load full IssueInfo objects for dependency analysis
282 issue_infos = manager.load_issue_infos(issues_to_process)
283 if not issue_infos:
284 logger.error("No issue files found")
285 return 1
287 # Apply label filter if provided (must run after IssueInfo load since labels come from frontmatter)
288 label_filter = parse_labels(getattr(args, "label", None))
289 if label_filter:
290 original_count = len(issue_infos)
291 issue_infos = [i for i in issue_infos if any(lb.lower() in label_filter for lb in i.labels)]
292 filtered = original_count - len(issue_infos)
293 if filtered > 0:
294 logger.info(f"Filtered {filtered} issue(s) by label: {', '.join(sorted(label_filter))}")
295 issues_to_process = [i.issue_id for i in issue_infos]
297 # Gather all issue IDs on disk to avoid false "nonexistent" warnings
298 from little_loops.dependency_mapper import gather_all_issue_ids
300 issues_dir = config.project_root / config.issues.base_dir
301 all_known_ids = gather_all_issue_ids(issues_dir, config=config)
303 # Dependency analysis (ENH-301)
304 if not getattr(args, "skip_analysis", False):
305 from little_loops.dependency_mapper import analyze_dependencies
307 issue_contents = _build_issue_contents(issue_infos)
308 dep_config = config.dependency_mapping
309 dep_report = analyze_dependencies(
310 issue_infos, issue_contents, all_known_ids=all_known_ids, config=dep_config
311 )
312 _render_dependency_analysis(dep_report, logger, config=dep_config)
314 # Build dependency graph
315 dep_graph = DependencyGraph.from_issues(issue_infos, all_known_ids=all_known_ids)
317 # Detect cycles
318 if dep_graph.has_cycles():
319 cycles = dep_graph.detect_cycles()
320 for cycle in cycles:
321 logger.error(f"Dependency cycle detected: {' -> '.join(cycle)}")
322 return 1
324 # Get execution waves
325 try:
326 waves = dep_graph.get_execution_waves()
327 except ValueError as e:
328 logger.error(str(e))
329 return 1
331 # Refine waves for file overlap (ENH-306)
332 waves, contention_notes = refine_waves_for_contention(waves, config=config.dependency_mapping)
334 # Display execution plan
335 logger.info(f"Running sprint: {sprint.name}")
336 logger.info("Dependency analysis:")
337 for i, wave in enumerate(waves, 1):
338 issue_ids = ", ".join(issue.issue_id for issue in wave)
339 note = contention_notes[i - 1] if contention_notes else None
340 if note:
341 logger.info(
342 f" Wave {i}: {issue_ids}"
343 f" [sub-wave {note.sub_wave_index + 1}/{note.total_sub_waves}]"
344 )
345 else:
346 logger.info(f" Wave {i}: {issue_ids}")
348 # Warn when issues were serialized conservatively due to missing file hints (BUG-2142)
349 if any(n is not None and getattr(n, "has_unknown_hints", False) for n in contention_notes):
350 logger.warning(
351 "Some issues lack '### Files to Modify' sections — serialized conservatively."
352 " Add file hints for more accurate wave scheduling."
353 )
355 if args.dry_run:
356 logger.info("\nDry run mode - no changes will be made")
357 return 0
359 # Initialize or load state
360 state: SprintState
361 start_wave = 1
363 if args.resume:
364 loaded_state = _load_sprint_state(logger)
365 if loaded_state and loaded_state.sprint_name == sprint.name:
366 state = loaded_state
367 # Find first incomplete wave by checking completed issues
368 completed_set = set(state.completed_issues)
369 for i, wave in enumerate(waves, 1):
370 wave_issue_ids = {issue.issue_id for issue in wave}
371 if not wave_issue_ids.issubset(completed_set):
372 start_wave = i
373 break
374 else:
375 # All waves completed
376 logger.info("Sprint already completed - nothing to resume")
377 _cleanup_sprint_state(logger)
378 return 0
379 logger.info(f"Resuming from wave {start_wave}/{len(waves)}")
380 logger.info(f" Previously completed: {len(state.completed_issues)} issues")
381 else:
382 if loaded_state:
383 logger.warning(
384 f"State file is for sprint '{loaded_state.sprint_name}', "
385 f"not '{sprint.name}' - starting fresh"
386 )
387 else:
388 logger.warning("No valid state found - starting fresh")
389 state = SprintState(
390 sprint_name=sprint.name,
391 started_at=datetime.now().isoformat(),
392 )
393 else:
394 # Fresh start - delete any old state
395 _cleanup_sprint_state(logger)
396 state = SprintState(
397 sprint_name=sprint.name,
398 started_at=datetime.now().isoformat(),
399 )
401 # Track exit status for error handling (ENH-185)
402 exit_code = 0
404 try:
405 # Determine max workers
406 max_workers = args.max_workers or (sprint.options.max_workers if sprint.options else 2)
408 # Execute wave by wave
409 completed: set[str] = set(state.completed_issues)
410 failed_waves = 0
411 total_duration = 0.0
412 total_waves = len(waves)
414 for wave_num, wave in enumerate(waves, 1):
415 # Check for shutdown request (ENH-183)
416 if _sprint_shutdown_requested:
417 logger.warning("Shutdown requested - saving state and exiting")
418 _save_sprint_state(state, logger)
419 exit_code = 1
420 return exit_code
422 # Skip already-completed waves when resuming
423 if wave_num < start_wave:
424 continue
426 wave_ids = [issue.issue_id for issue in wave]
427 state.current_wave = wave_num
428 logger.info(f"\nProcessing wave {wave_num}/{total_waves}: {', '.join(wave_ids)}")
430 wave_note = (
431 contention_notes[wave_num - 1]
432 if contention_notes and wave_num - 1 < len(contention_notes)
433 else None
434 )
435 is_contention_subwave = wave_note is not None
437 if len(wave) == 1 or is_contention_subwave:
438 # Single issue OR contention sub-wave — process in-place sequentially
439 # (contention sub-waves are displayed as "serialized steps" so must run that way)
440 max_wall_clock = config.sprints.max_issue_wall_clock_time
442 wave_failed = False
443 _current_branch = _detect_current_branch()
444 for issue in wave:
445 # TODO(ENH-1686): sprint sequential path not yet live-written
446 issue_result = _run_issue_with_wall_clock_timeout(
447 issue=issue,
448 config=config,
449 logger=logger,
450 dry_run=args.dry_run,
451 max_seconds=max_wall_clock,
452 sprint_context=SprintWorkerContext(
453 issue_id=issue.issue_id,
454 branch=_current_branch,
455 ),
456 )
457 total_duration += issue_result.duration
458 if issue_result.success:
459 completed.add(issue.issue_id)
460 state.completed_issues.append(issue.issue_id)
461 state.timing[issue.issue_id] = {"total": issue_result.duration}
462 logger.success(f" {issue.issue_id}: completed")
463 elif issue_result.was_blocked:
464 completed.add(issue.issue_id)
465 state.skipped_blocked_issues[issue.issue_id] = issue_result.failure_reason
466 logger.warning(f" {issue.issue_id}: skipped (blocked by open dependency)")
467 else:
468 wave_failed = True
469 completed.add(issue.issue_id)
470 state.failed_issues[issue.issue_id] = (
471 issue_result.failure_reason or "Issue processing failed"
472 )
473 logger.warning(f" {issue.issue_id}: failed")
474 if wave_failed:
475 failed_waves += 1
476 logger.warning(f"Wave {wave_num}/{total_waves} had failures")
477 else:
478 logger.success(
479 f"Wave {wave_num}/{total_waves} completed: {', '.join(wave_ids)}"
480 )
481 _save_sprint_state(state, logger)
482 if wave_num < total_waves:
483 logger.info(f"Continuing to wave {wave_num + 1}/{total_waves}...")
484 # Check for shutdown before next wave (ENH-183)
485 if _sprint_shutdown_requested:
486 logger.warning("Shutdown requested - exiting after wave completion")
487 exit_code = 1
488 return exit_code
489 else:
490 # Multi-issue — use ParallelOrchestrator with worktrees
491 only_ids = set(wave_ids)
492 # Detect current branch for rebase/merge operations (BUG-439)
493 _br = subprocess.run(
494 ["git", "rev-parse", "--abbrev-ref", "HEAD"],
495 capture_output=True,
496 text=True,
497 cwd=Path.cwd(),
498 )
499 _base_branch = _br.stdout.strip() if _br.returncode == 0 else "main"
500 # Runtime overlap detection disabled for sprints (ENH-512):
501 # refine_waves_for_contention() already splits overlapping
502 # issues into separate sub-waves before dispatch.
503 parallel_config = config.create_parallel_config(
504 max_workers=min(max_workers, len(wave)),
505 only_ids=only_ids,
506 dry_run=args.dry_run,
507 overlap_detection=False,
508 serialize_overlapping=True,
509 base_branch=_base_branch,
510 clean_start=True, # Sprint manages its own state; don't load stale orchestrator state
511 )
513 from little_loops.events import EventBus
514 from little_loops.extension import wire_extensions
515 from little_loops.transport import wire_transports
517 event_bus = EventBus()
518 wire_extensions(event_bus, config.extensions)
519 wire_transports(event_bus, config.events)
520 orchestrator = ParallelOrchestrator(
521 parallel_config,
522 config,
523 Path.cwd(),
524 wave_label=f"Wave {wave_num}/{total_waves}",
525 event_bus=event_bus,
526 )
527 result = orchestrator.run()
528 total_duration += orchestrator.execution_duration
530 # Track completed/failed from this wave using per-issue results
531 actually_completed = set(orchestrator.queue.completed_ids)
532 actually_failed = set(orchestrator.queue.failed_ids)
534 for issue_id in wave_ids:
535 if issue_id in actually_completed:
536 completed.add(issue_id)
537 state.completed_issues.append(issue_id)
538 state.timing[issue_id] = {
539 "total": orchestrator.execution_duration / len(wave)
540 }
541 elif issue_id in actually_failed:
542 completed.add(issue_id)
543 state.failed_issues[issue_id] = "Issue failed during wave execution"
544 # else: issue was neither completed nor failed (interrupted/stranded)
545 # — leave untracked so it can be retried on resume
547 # Sequential retry for failed issues (ENH-308)
548 if actually_failed:
549 logger.info(f"Retrying {len(actually_failed)} failed issue(s) sequentially...")
550 retried_ok = 0
551 for issue in wave:
552 if issue.issue_id not in actually_failed:
553 continue
554 logger.info(f" Retrying {issue.issue_id} in-place...")
555 # TODO(ENH-1686): sprint sequential path not yet live-written
556 retry_result = process_issue_inplace(
557 info=issue,
558 config=config,
559 logger=logger,
560 dry_run=args.dry_run,
561 sprint_context=SprintWorkerContext(
562 issue_id=issue.issue_id,
563 branch=_detect_current_branch(),
564 ),
565 )
566 total_duration += retry_result.duration
567 if retry_result.success:
568 retried_ok += 1
569 state.failed_issues.pop(issue.issue_id, None)
570 state.completed_issues.append(issue.issue_id)
571 state.timing[issue.issue_id] = {"total": retry_result.duration}
572 logger.success(f" Retry succeeded: {issue.issue_id}")
573 elif retry_result.was_blocked:
574 state.failed_issues.pop(issue.issue_id, None)
575 state.skipped_blocked_issues[issue.issue_id] = (
576 retry_result.failure_reason
577 )
578 logger.warning(
579 f" Retry skipped: {issue.issue_id} (blocked by open dependency)"
580 )
581 else:
582 logger.warning(f" Retry failed: {issue.issue_id}")
583 if retried_ok > 0:
584 logger.info(
585 f"Sequential retry recovered {retried_ok}/{len(actually_failed)} issue(s)"
586 )
588 # Check whether failures remain after retry (ENH-308)
589 remaining_failures = {iid for iid in actually_failed if iid in state.failed_issues}
590 if result == 0 or not remaining_failures:
591 logger.success(
592 f"Wave {wave_num}/{total_waves} completed: {', '.join(wave_ids)}"
593 )
594 else:
595 failed_waves += 1
596 logger.warning(f"Wave {wave_num}/{total_waves} had failures")
597 _save_sprint_state(state, logger)
598 if wave_num < total_waves:
599 logger.info(f"Continuing to wave {wave_num + 1}/{total_waves}...")
600 # Check for shutdown before next wave (ENH-183)
601 if _sprint_shutdown_requested:
602 logger.warning("Shutdown requested - exiting after wave completion")
603 exit_code = 1
604 return exit_code
606 wave_word = "wave" if len(waves) == 1 else "waves"
607 skip_msg = (
608 f", {len(pre_completed_skipped)} already completed (skipped)"
609 if pre_completed_skipped
610 else ""
611 )
612 blocked_msg = (
613 f", {len(state.skipped_blocked_issues)} skipped (blocked)"
614 if state.skipped_blocked_issues
615 else ""
616 )
617 logger.info(
618 f"\nSprint completed: {len(completed)} issues processed "
619 f"({len(waves)} {wave_word}){skip_msg}{blocked_msg}"
620 )
621 logger.timing(f"Total execution time: {format_duration(total_duration)}")
622 if failed_waves > 0:
623 logger.warning(f"{failed_waves} wave(s) had failures")
624 exit_code = 1
625 else:
626 # Clean up state on successful completion
627 _cleanup_sprint_state(logger)
629 except KeyboardInterrupt:
630 # Belt-and-suspenders with signal handler (ENH-185)
631 logger.warning("Sprint interrupted by user (KeyboardInterrupt)")
632 exit_code = 130
634 except Exception as e:
635 # Catch unexpected exceptions (ENH-185)
636 logger.error(f"Sprint failed unexpectedly: {e}")
637 exit_code = 1
639 finally:
640 # Guaranteed state save on any non-success exit (ENH-185)
641 if exit_code != 0:
642 _save_sprint_state(state, logger)
643 logger.info("State saved before exit")
645 _sprint_logger = None # Clear for test isolation
646 return exit_code