Coverage for little_loops / cli / sprint / run.py: 8%
355 statements
« prev ^ index » next coverage.py v7.12.0, created at 2026-06-16 13:11 -0500
1"""ll-sprint run subcommand with signal handling and state management."""
3from __future__ import annotations
5import os
6import signal
7import subprocess
8import sys
9import time
10from pathlib import Path
11from types import FrameType
12from typing import TYPE_CHECKING
14from little_loops.cli.output import use_color_enabled
15from little_loops.cli.sprint._helpers import _build_issue_contents, _render_dependency_analysis
16from little_loops.cli_args import _id_matches, parse_issue_ids, parse_issue_types, parse_labels
17from little_loops.dependency_graph import DependencyGraph, refine_waves_for_contention
18from little_loops.issue_manager import IssueProcessingResult, process_issue_inplace
19from little_loops.logger import Logger, format_duration
20from little_loops.parallel.orchestrator import ParallelOrchestrator
21from little_loops.parallel.types import SprintWorkerContext
22from little_loops.sprint import SprintManager, SprintState
24if TYPE_CHECKING:
25 import argparse
27 from little_loops.config import BRConfig
28 from little_loops.issue_parser import IssueInfo
30# Module-level shutdown flag for ll-sprint signal handling (ENH-183)
31_sprint_shutdown_requested: bool = False
32_sprint_logger: Logger | None = None
35class IssueWallClockTimeout(Exception):
36 """Raised when an issue exceeds its per-issue wall-clock time budget (BUG-2144)."""
38 def __init__(self, issue_id: str) -> None:
39 self.issue_id = issue_id
40 super().__init__(f"{issue_id}: wall-clock timeout")
43def _run_issue_with_wall_clock_timeout(
44 issue: IssueInfo,
45 config: BRConfig,
46 logger: Logger,
47 dry_run: bool,
48 max_seconds: int,
49 sprint_context: SprintWorkerContext | None = None,
50) -> IssueProcessingResult:
51 """Wrap process_issue_inplace with a SIGALRM-based wall-clock timeout.
53 If the issue takes longer than max_seconds (across all subprocess spawns,
54 including Option J continuations), SIGALRM fires, IssueWallClockTimeout is
55 raised, and a failed IssueProcessingResult with failure_reason=WALL_CLOCK_TIMEOUT
56 is returned. The alarm is always cleared in the finally block.
57 """
59 def _timeout_handler(signum: int, frame: FrameType | None) -> None:
60 raise IssueWallClockTimeout(issue.issue_id)
62 prev_handler = signal.signal(signal.SIGALRM, _timeout_handler)
63 signal.alarm(max_seconds)
64 issue_start = time.monotonic()
65 try:
66 return process_issue_inplace(
67 info=issue,
68 config=config,
69 logger=logger,
70 dry_run=dry_run,
71 sprint_context=sprint_context,
72 )
73 except IssueWallClockTimeout:
74 elapsed = time.monotonic() - issue_start
75 logger.error(
76 f" {issue.issue_id}: wall-clock timeout after {int(elapsed)}s "
77 f"(limit {max_seconds}s) — marking failed"
78 )
79 return IssueProcessingResult(
80 success=False,
81 duration=elapsed,
82 issue_id=issue.issue_id,
83 failure_reason="WALL_CLOCK_TIMEOUT",
84 )
85 finally:
86 signal.alarm(0)
87 signal.signal(signal.SIGALRM, prev_handler)
90def _detect_current_branch() -> str:
91 """Return the current git branch name, defaulting to 'main' on failure."""
92 result = subprocess.run(
93 ["git", "rev-parse", "--abbrev-ref", "HEAD"],
94 capture_output=True,
95 text=True,
96 )
97 return result.stdout.strip() if result.returncode == 0 else "main"
100def _sprint_signal_handler(signum: int, frame: FrameType | None) -> None:
101 """Handle shutdown signals gracefully for ll-sprint.
103 First signal: Set shutdown flag for graceful exit after current wave.
104 Second signal: Force immediate exit.
105 """
106 global _sprint_shutdown_requested
107 if _sprint_shutdown_requested:
108 # Second signal - force exit
109 msg = "\nForce shutdown requested"
110 if _sprint_logger is not None:
111 _sprint_logger.warning(msg)
112 else:
113 print(msg, file=sys.stderr)
114 sys.exit(1)
115 _sprint_shutdown_requested = True
116 msg = "\nShutdown requested, will exit after current wave..."
117 if _sprint_logger is not None:
118 _sprint_logger.warning(msg)
119 else:
120 print(msg, file=sys.stderr)
123def _get_sprint_state_file() -> Path:
124 """Get path to sprint state file."""
125 return Path.cwd() / ".sprint-state.json"
128def _load_sprint_state(logger: Logger) -> SprintState | None:
129 """Load sprint state from file."""
130 import json
132 state_file = _get_sprint_state_file()
133 if not state_file.exists():
134 return None
135 try:
136 data = json.loads(state_file.read_text())
137 state = SprintState.from_dict(data)
138 logger.info(f"State loaded from {state_file}")
139 return state
140 except (json.JSONDecodeError, KeyError) as e:
141 logger.warning(f"Failed to load state: {e}")
142 return None
145def _save_sprint_state(state: SprintState, logger: Logger) -> None:
146 """Save sprint state to file."""
147 import json
148 from datetime import datetime
150 state.last_checkpoint = datetime.now().isoformat()
151 state_file = _get_sprint_state_file()
152 state_file.write_text(json.dumps(state.to_dict(), indent=2))
153 logger.info(f"State saved to {state_file}")
156def _cleanup_sprint_state(logger: Logger) -> None:
157 """Remove sprint state file."""
158 state_file = _get_sprint_state_file()
159 if state_file.exists():
160 state_file.unlink()
161 logger.info("Sprint state file cleaned up")
164def _cmd_sprint_run(
165 args: argparse.Namespace,
166 manager: SprintManager,
167 config: BRConfig,
168) -> int:
169 """Execute a sprint with dependency-aware scheduling."""
170 from datetime import datetime
172 global _sprint_logger
173 logger = Logger(verbose=not args.quiet, use_color=use_color_enabled())
174 _sprint_logger = logger
176 # Setup signal handlers for graceful shutdown (ENH-183)
177 global _sprint_shutdown_requested
178 _sprint_shutdown_requested = False # Reset in case of multiple runs
179 signal.signal(signal.SIGINT, _sprint_signal_handler)
180 signal.signal(signal.SIGTERM, _sprint_signal_handler)
182 handoff_threshold = getattr(args, "handoff_threshold", None)
183 if handoff_threshold is not None:
184 os.environ["LL_HANDOFF_THRESHOLD"] = str(handoff_threshold)
186 context_limit = getattr(args, "context_limit", None)
187 if context_limit is not None:
188 os.environ["LL_CONTEXT_LIMIT"] = str(context_limit)
190 sprint = manager.load_or_resolve(args.sprint)
191 if not sprint:
192 logger.error(f"Sprint not found: {args.sprint}")
193 return 1
195 if not sprint.issues:
196 logger.info(f"Sprint '{sprint.name}' has no active issues — nothing to process")
197 return 0
199 if getattr(args, "save", False):
200 saved_path = sprint.save(manager.sprints_dir)
201 logger.info(f"Sprint saved to {saved_path}")
203 # Apply skip filter if provided
204 issues_to_process = list(sprint.issues)
205 skip_ids = parse_issue_ids(args.skip)
206 if skip_ids:
207 original_count = len(issues_to_process)
208 issues_to_process = [i for i in issues_to_process if i not in skip_ids]
209 skipped = original_count - len(issues_to_process)
210 if skipped > 0:
211 logger.info(f"Skipping {skipped} issue(s): {', '.join(sorted(skip_ids))}")
213 # Apply only filter if provided
214 only_ids = parse_issue_ids(getattr(args, "only", None))
215 if only_ids:
216 invalid_only = {p for p in only_ids if not any(_id_matches(i, p) for i in sprint.issues)}
217 if invalid_only:
218 logger.error(
219 f"Issue(s) not found in sprint definition: {', '.join(sorted(invalid_only))}"
220 )
221 return 1
222 issues_to_process = [
223 i for i in issues_to_process if any(_id_matches(i, p) for p in only_ids)
224 ]
225 logger.info(
226 f"Processing only {len(issues_to_process)} issue(s): {', '.join(sorted(only_ids))}"
227 )
229 # Apply type filter if provided
230 type_prefixes = parse_issue_types(getattr(args, "type", None))
231 if type_prefixes:
232 original_count = len(issues_to_process)
233 issues_to_process = [i for i in issues_to_process if i.split("-", 1)[0] in type_prefixes]
234 filtered = original_count - len(issues_to_process)
235 if filtered > 0:
236 logger.info(f"Filtered {filtered} issue(s) by type: {', '.join(sorted(type_prefixes))}")
238 # Pre-validate: skip issues with status: done/cancelled (ENH-1424)
239 pre_completed_skipped: list[str] = []
241 # Validate issues exist
242 valid = manager.validate_issues(issues_to_process)
243 invalid = set(issues_to_process) - set(valid.keys())
245 if invalid:
246 logger.error(f"Issue IDs not found: {', '.join(sorted(invalid))}")
247 logger.info("Cannot execute sprint with missing issues")
248 return 1
250 # Write milestone: field back to issue files for all valid issues in this sprint
251 from little_loops.frontmatter import parse_frontmatter, update_frontmatter
253 for _issue_id, path in valid.items():
254 content = path.read_text(encoding="utf-8")
255 updated = update_frontmatter(content, {"milestone": sprint.name})
256 if updated != content:
257 path.write_text(updated, encoding="utf-8")
258 logger.info(f"Set milestone: {sprint.name!r} on {len(valid)} issue file(s)")
260 # Skip issues already completed via frontmatter status (ENH-1424)
261 still_active: list[str] = []
262 for issue_id in issues_to_process:
263 path = valid[issue_id]
264 fm = parse_frontmatter(path.read_text(encoding="utf-8"))
265 if fm.get("status", "open") in ("done", "cancelled"):
266 logger.info(f" {issue_id}: already completed (status: {fm.get('status')}), skipping")
267 pre_completed_skipped.append(issue_id)
268 else:
269 still_active.append(issue_id)
270 if pre_completed_skipped:
271 logger.info(
272 f"Pre-validation: {len(pre_completed_skipped)} issue(s) already completed, "
273 f"{len(still_active)} active"
274 )
275 issues_to_process = still_active
277 if pre_completed_skipped and not issues_to_process:
278 logger.info("All sprint issues already completed - nothing to process")
279 return 0
281 # Load full IssueInfo objects for dependency analysis
282 issue_infos = manager.load_issue_infos(issues_to_process)
283 if not issue_infos:
284 logger.error("No issue files found")
285 return 1
287 # Apply label filter if provided (must run after IssueInfo load since labels come from frontmatter)
288 label_filter = parse_labels(getattr(args, "label", None))
289 if label_filter:
290 original_count = len(issue_infos)
291 issue_infos = [i for i in issue_infos if any(lb.lower() in label_filter for lb in i.labels)]
292 filtered = original_count - len(issue_infos)
293 if filtered > 0:
294 logger.info(f"Filtered {filtered} issue(s) by label: {', '.join(sorted(label_filter))}")
295 issues_to_process = [i.issue_id for i in issue_infos]
297 # Gather all issue IDs on disk to avoid false "nonexistent" warnings
298 from little_loops.dependency_mapper import gather_all_issue_ids
300 issues_dir = config.project_root / config.issues.base_dir
301 all_known_ids = gather_all_issue_ids(issues_dir, config=config)
303 # Dependency analysis (ENH-301)
304 if not getattr(args, "skip_analysis", False):
305 from little_loops.dependency_mapper import analyze_dependencies
307 issue_contents = _build_issue_contents(issue_infos)
308 dep_config = config.dependency_mapping
309 dep_report = analyze_dependencies(
310 issue_infos, issue_contents, all_known_ids=all_known_ids, config=dep_config
311 )
312 _render_dependency_analysis(dep_report, logger, config=dep_config)
314 # Build dependency graph
315 dep_graph = DependencyGraph.from_issues(issue_infos, all_known_ids=all_known_ids)
317 # Detect cycles
318 if dep_graph.has_cycles():
319 cycles = dep_graph.detect_cycles()
320 for cycle in cycles:
321 logger.error(f"Dependency cycle detected: {' -> '.join(cycle)}")
322 return 1
324 # Get execution waves
325 try:
326 waves = dep_graph.get_execution_waves()
327 except ValueError as e:
328 logger.error(str(e))
329 return 1
331 # Refine waves for file overlap (ENH-306)
332 waves, contention_notes = refine_waves_for_contention(waves, config=config.dependency_mapping)
334 # Display execution plan
335 logger.info(f"Running sprint: {sprint.name}")
336 logger.info("Dependency analysis:")
337 for i, wave in enumerate(waves, 1):
338 issue_ids = ", ".join(issue.issue_id for issue in wave)
339 note = contention_notes[i - 1] if contention_notes else None
340 if note:
341 logger.info(
342 f" Wave {i}: {issue_ids}"
343 f" [sub-wave {note.sub_wave_index + 1}/{note.total_sub_waves}]"
344 )
345 else:
346 logger.info(f" Wave {i}: {issue_ids}")
348 # Warn when issues were serialized conservatively due to missing file hints (BUG-2142)
349 if any(n is not None and getattr(n, "has_unknown_hints", False) for n in contention_notes):
350 logger.warning(
351 "Some issues lack '### Files to Modify' sections — serialized conservatively."
352 " Add file hints for more accurate wave scheduling."
353 )
355 if args.dry_run:
356 logger.info("\nDry run mode - no changes will be made")
357 return 0
359 # Initialize or load state
360 state: SprintState
361 start_wave = 1
363 if args.resume:
364 loaded_state = _load_sprint_state(logger)
365 if loaded_state and loaded_state.sprint_name == sprint.name:
366 state = loaded_state
367 # Find first incomplete wave by checking completed issues
368 completed_set = set(state.completed_issues)
369 for i, wave in enumerate(waves, 1):
370 wave_issue_ids = {issue.issue_id for issue in wave}
371 if not wave_issue_ids.issubset(completed_set):
372 start_wave = i
373 break
374 else:
375 # All waves completed
376 logger.info("Sprint already completed - nothing to resume")
377 _cleanup_sprint_state(logger)
378 return 0
379 logger.info(f"Resuming from wave {start_wave}/{len(waves)}")
380 logger.info(f" Previously completed: {len(state.completed_issues)} issues")
381 else:
382 if loaded_state:
383 logger.warning(
384 f"State file is for sprint '{loaded_state.sprint_name}', "
385 f"not '{sprint.name}' - starting fresh"
386 )
387 else:
388 logger.warning("No valid state found - starting fresh")
389 state = SprintState(
390 sprint_name=sprint.name,
391 started_at=datetime.now().isoformat(),
392 )
393 else:
394 # Fresh start - delete any old state
395 _cleanup_sprint_state(logger)
396 state = SprintState(
397 sprint_name=sprint.name,
398 started_at=datetime.now().isoformat(),
399 )
401 # Track exit status for error handling (ENH-185)
402 exit_code = 0
404 try:
405 # Determine max workers
406 max_workers = args.max_workers or (sprint.options.max_workers if sprint.options else 2)
408 # Execute wave by wave
409 completed: set[str] = set(state.completed_issues)
410 failed_waves = 0
411 total_duration = 0.0
412 total_waves = len(waves)
413 _fb_warning_emitted = False # deduplicate: fire once per sprint run (ENH-2176)
415 for wave_num, wave in enumerate(waves, 1):
416 # Check for shutdown request (ENH-183)
417 if _sprint_shutdown_requested:
418 logger.warning("Shutdown requested - saving state and exiting")
419 _save_sprint_state(state, logger)
420 exit_code = 1
421 return exit_code
423 # Skip already-completed waves when resuming
424 if wave_num < start_wave:
425 continue
427 wave_ids = [issue.issue_id for issue in wave]
428 state.current_wave = wave_num
429 logger.info(f"\nProcessing wave {wave_num}/{total_waves}: {', '.join(wave_ids)}")
431 wave_note = (
432 contention_notes[wave_num - 1]
433 if contention_notes and wave_num - 1 < len(contention_notes)
434 else None
435 )
436 is_contention_subwave = wave_note is not None
438 if len(wave) == 1 or is_contention_subwave:
439 # Single issue OR contention sub-wave — process in-place sequentially
440 # (contention sub-waves are displayed as "serialized steps" so must run that way)
441 max_wall_clock = config.sprints.max_issue_wall_clock_time
443 wave_failed = False
444 _current_branch = _detect_current_branch()
445 effective_feature_branches = (
446 args.feature_branches
447 if args.feature_branches is not None
448 else config.parallel.use_feature_branches
449 )
450 if effective_feature_branches and not _fb_warning_emitted:
451 logger.warning(
452 "feature-branch mode does not apply to single-issue / contention"
453 f" sub-waves; these issues run in-place on '{_current_branch}'"
454 )
455 _fb_warning_emitted = True
456 for issue in wave:
457 # TODO(ENH-1686): sprint sequential path not yet live-written
458 issue_result = _run_issue_with_wall_clock_timeout(
459 issue=issue,
460 config=config,
461 logger=logger,
462 dry_run=args.dry_run,
463 max_seconds=max_wall_clock,
464 sprint_context=SprintWorkerContext(
465 issue_id=issue.issue_id,
466 branch=_current_branch,
467 ),
468 )
469 total_duration += issue_result.duration
470 if issue_result.success:
471 completed.add(issue.issue_id)
472 state.completed_issues.append(issue.issue_id)
473 state.timing[issue.issue_id] = {"total": issue_result.duration}
474 logger.success(f" {issue.issue_id}: completed")
475 elif issue_result.was_blocked:
476 completed.add(issue.issue_id)
477 state.skipped_blocked_issues[issue.issue_id] = issue_result.failure_reason
478 logger.warning(f" {issue.issue_id}: skipped (blocked by open dependency)")
479 else:
480 wave_failed = True
481 completed.add(issue.issue_id)
482 state.failed_issues[issue.issue_id] = (
483 issue_result.failure_reason or "Issue processing failed"
484 )
485 logger.warning(f" {issue.issue_id}: failed")
486 if wave_failed:
487 failed_waves += 1
488 logger.warning(f"Wave {wave_num}/{total_waves} had failures")
489 else:
490 logger.success(
491 f"Wave {wave_num}/{total_waves} completed: {', '.join(wave_ids)}"
492 )
493 _save_sprint_state(state, logger)
494 if wave_num < total_waves:
495 logger.info(f"Continuing to wave {wave_num + 1}/{total_waves}...")
496 # Check for shutdown before next wave (ENH-183)
497 if _sprint_shutdown_requested:
498 logger.warning("Shutdown requested - exiting after wave completion")
499 exit_code = 1
500 return exit_code
501 else:
502 # Multi-issue — use ParallelOrchestrator with worktrees
503 only_ids = set(wave_ids)
504 # Detect current branch for rebase/merge operations (BUG-439)
505 _br = subprocess.run(
506 ["git", "rev-parse", "--abbrev-ref", "HEAD"],
507 capture_output=True,
508 text=True,
509 cwd=Path.cwd(),
510 )
511 _base_branch = _br.stdout.strip() if _br.returncode == 0 else "main"
512 # Runtime overlap detection disabled for sprints (ENH-512):
513 # refine_waves_for_contention() already splits overlapping
514 # issues into separate sub-waves before dispatch.
515 parallel_config = config.create_parallel_config(
516 max_workers=min(max_workers, len(wave)),
517 only_ids=only_ids,
518 dry_run=args.dry_run,
519 overlap_detection=False,
520 serialize_overlapping=True,
521 base_branch=_base_branch,
522 clean_start=True, # Sprint manages its own state; don't load stale orchestrator state
523 use_feature_branches=args.feature_branches,
524 )
526 from little_loops.events import EventBus
527 from little_loops.extension import wire_extensions
528 from little_loops.transport import wire_transports
530 event_bus = EventBus()
531 wire_extensions(event_bus, config.extensions)
532 wire_transports(event_bus, config.events)
533 orchestrator = ParallelOrchestrator(
534 parallel_config,
535 config,
536 Path.cwd(),
537 wave_label=f"Wave {wave_num}/{total_waves}",
538 event_bus=event_bus,
539 )
540 result = orchestrator.run()
541 total_duration += orchestrator.execution_duration
543 # Track completed/failed from this wave using per-issue results
544 actually_completed = set(orchestrator.queue.completed_ids)
545 actually_failed = set(orchestrator.queue.failed_ids)
547 for issue_id in wave_ids:
548 if issue_id in actually_completed:
549 completed.add(issue_id)
550 state.completed_issues.append(issue_id)
551 state.timing[issue_id] = {
552 "total": orchestrator.execution_duration / len(wave)
553 }
554 elif issue_id in actually_failed:
555 completed.add(issue_id)
556 state.failed_issues[issue_id] = "Issue failed during wave execution"
557 # else: issue was neither completed nor failed (interrupted/stranded)
558 # — leave untracked so it can be retried on resume
560 # Sequential retry for failed issues (ENH-308)
561 if actually_failed:
562 logger.info(f"Retrying {len(actually_failed)} failed issue(s) sequentially...")
563 retried_ok = 0
564 for issue in wave:
565 if issue.issue_id not in actually_failed:
566 continue
567 logger.info(f" Retrying {issue.issue_id} in-place...")
568 # TODO(ENH-1686): sprint sequential path not yet live-written
569 retry_result = process_issue_inplace(
570 info=issue,
571 config=config,
572 logger=logger,
573 dry_run=args.dry_run,
574 sprint_context=SprintWorkerContext(
575 issue_id=issue.issue_id,
576 branch=_detect_current_branch(),
577 ),
578 )
579 total_duration += retry_result.duration
580 if retry_result.success:
581 retried_ok += 1
582 state.failed_issues.pop(issue.issue_id, None)
583 state.completed_issues.append(issue.issue_id)
584 state.timing[issue.issue_id] = {"total": retry_result.duration}
585 logger.success(f" Retry succeeded: {issue.issue_id}")
586 elif retry_result.was_blocked:
587 state.failed_issues.pop(issue.issue_id, None)
588 state.skipped_blocked_issues[issue.issue_id] = (
589 retry_result.failure_reason
590 )
591 logger.warning(
592 f" Retry skipped: {issue.issue_id} (blocked by open dependency)"
593 )
594 else:
595 logger.warning(f" Retry failed: {issue.issue_id}")
596 if retried_ok > 0:
597 logger.info(
598 f"Sequential retry recovered {retried_ok}/{len(actually_failed)} issue(s)"
599 )
601 # Check whether failures remain after retry (ENH-308)
602 remaining_failures = {iid for iid in actually_failed if iid in state.failed_issues}
603 if result == 0 or not remaining_failures:
604 logger.success(
605 f"Wave {wave_num}/{total_waves} completed: {', '.join(wave_ids)}"
606 )
607 else:
608 failed_waves += 1
609 logger.warning(f"Wave {wave_num}/{total_waves} had failures")
610 _save_sprint_state(state, logger)
611 if wave_num < total_waves:
612 logger.info(f"Continuing to wave {wave_num + 1}/{total_waves}...")
613 # Check for shutdown before next wave (ENH-183)
614 if _sprint_shutdown_requested:
615 logger.warning("Shutdown requested - exiting after wave completion")
616 exit_code = 1
617 return exit_code
619 wave_word = "wave" if len(waves) == 1 else "waves"
620 skip_msg = (
621 f", {len(pre_completed_skipped)} already completed (skipped)"
622 if pre_completed_skipped
623 else ""
624 )
625 blocked_msg = (
626 f", {len(state.skipped_blocked_issues)} skipped (blocked)"
627 if state.skipped_blocked_issues
628 else ""
629 )
630 logger.info(
631 f"\nSprint completed: {len(completed)} issues processed "
632 f"({len(waves)} {wave_word}){skip_msg}{blocked_msg}"
633 )
634 logger.timing(f"Total execution time: {format_duration(total_duration)}")
635 if failed_waves > 0:
636 logger.warning(f"{failed_waves} wave(s) had failures")
637 exit_code = 1
638 else:
639 # Clean up state on successful completion
640 _cleanup_sprint_state(logger)
642 except KeyboardInterrupt:
643 # Belt-and-suspenders with signal handler (ENH-185)
644 logger.warning("Sprint interrupted by user (KeyboardInterrupt)")
645 exit_code = 130
647 except Exception as e:
648 # Catch unexpected exceptions (ENH-185)
649 logger.error(f"Sprint failed unexpectedly: {e}")
650 exit_code = 1
652 finally:
653 # Guaranteed state save on any non-success exit (ENH-185)
654 if exit_code != 0:
655 _save_sprint_state(state, logger)
656 logger.info("State saved before exit")
658 _sprint_logger = None # Clear for test isolation
659 return exit_code