Coverage for little_loops / cli / sprint / run.py: 8%

389 statements  

« prev     ^ index     » next       coverage.py v7.12.0, created at 2026-06-29 00:55 -0500

1"""ll-sprint run subcommand with signal handling and state management.""" 

2 

3from __future__ import annotations 

4 

5import os 

6import signal 

7import subprocess 

8import sys 

9import time 

10from pathlib import Path 

11from types import FrameType 

12from typing import TYPE_CHECKING 

13 

14from little_loops.cli.output import use_color_enabled 

15from little_loops.cli.sprint._helpers import _build_issue_contents, _render_dependency_analysis 

16from little_loops.cli_args import _id_matches, parse_issue_ids, parse_issue_types, parse_labels 

17from little_loops.dependency_graph import DependencyGraph, refine_waves_for_contention 

18from little_loops.issue_manager import IssueProcessingResult, process_issue_inplace 

19from little_loops.logger import Logger, format_duration 

20from little_loops.parallel.orchestrator import ParallelOrchestrator 

21from little_loops.parallel.types import SprintWorkerContext 

22from little_loops.sprint import SprintManager, SprintState 

23 

24if TYPE_CHECKING: 

25 import argparse 

26 

27 from little_loops.config import BRConfig 

28 from little_loops.issue_parser import IssueInfo 

29 

30# Module-level shutdown flag for ll-sprint signal handling (ENH-183) 

31_sprint_shutdown_requested: bool = False 

32_sprint_logger: Logger | None = None 

33 

34 

35class IssueWallClockTimeout(Exception): 

36 """Raised when an issue exceeds its per-issue wall-clock time budget (BUG-2144).""" 

37 

38 def __init__(self, issue_id: str) -> None: 

39 self.issue_id = issue_id 

40 super().__init__(f"{issue_id}: wall-clock timeout") 

41 

42 

43def _run_issue_with_wall_clock_timeout( 

44 issue: IssueInfo, 

45 config: BRConfig, 

46 logger: Logger, 

47 dry_run: bool, 

48 max_seconds: int, 

49 sprint_context: SprintWorkerContext | None = None, 

50) -> IssueProcessingResult: 

51 """Wrap process_issue_inplace with a SIGALRM-based wall-clock timeout. 

52 

53 If the issue takes longer than max_seconds (across all subprocess spawns, 

54 including Option J continuations), SIGALRM fires, IssueWallClockTimeout is 

55 raised, and a failed IssueProcessingResult with failure_reason=WALL_CLOCK_TIMEOUT 

56 is returned. The alarm is always cleared in the finally block. 

57 """ 

58 

59 def _timeout_handler(signum: int, frame: FrameType | None) -> None: 

60 raise IssueWallClockTimeout(issue.issue_id) 

61 

62 prev_handler = signal.signal(signal.SIGALRM, _timeout_handler) 

63 signal.alarm(max_seconds) 

64 issue_start = time.monotonic() 

65 try: 

66 return process_issue_inplace( 

67 info=issue, 

68 config=config, 

69 logger=logger, 

70 dry_run=dry_run, 

71 sprint_context=sprint_context, 

72 ) 

73 except IssueWallClockTimeout: 

74 elapsed = time.monotonic() - issue_start 

75 logger.error( 

76 f" {issue.issue_id}: wall-clock timeout after {int(elapsed)}s " 

77 f"(limit {max_seconds}s) — marking failed" 

78 ) 

79 return IssueProcessingResult( 

80 success=False, 

81 duration=elapsed, 

82 issue_id=issue.issue_id, 

83 failure_reason="WALL_CLOCK_TIMEOUT", 

84 ) 

85 finally: 

86 signal.alarm(0) 

87 signal.signal(signal.SIGALRM, prev_handler) 

88 

89 

90def _detect_current_branch() -> str: 

91 """Return the current git branch name, defaulting to 'main' on failure.""" 

92 result = subprocess.run( 

93 ["git", "rev-parse", "--abbrev-ref", "HEAD"], 

94 capture_output=True, 

95 text=True, 

96 ) 

97 return result.stdout.strip() if result.returncode == 0 else "main" 

98 

99 

100def _sprint_signal_handler(signum: int, frame: FrameType | None) -> None: 

101 """Handle shutdown signals gracefully for ll-sprint. 

102 

103 First signal: Set shutdown flag for graceful exit after current wave. 

104 Second signal: Force immediate exit. 

105 """ 

106 global _sprint_shutdown_requested 

107 if _sprint_shutdown_requested: 

108 # Second signal - force exit 

109 msg = "\nForce shutdown requested" 

110 if _sprint_logger is not None: 

111 _sprint_logger.warning(msg) 

112 else: 

113 print(msg, file=sys.stderr) 

114 sys.exit(1) 

115 _sprint_shutdown_requested = True 

116 msg = "\nShutdown requested, will exit after current wave..." 

117 if _sprint_logger is not None: 

118 _sprint_logger.warning(msg) 

119 else: 

120 print(msg, file=sys.stderr) 

121 

122 

123def _get_sprint_state_file() -> Path: 

124 """Get path to sprint state file.""" 

125 return Path.cwd() / ".sprint-state.json" 

126 

127 

128def _load_sprint_state(logger: Logger) -> SprintState | None: 

129 """Load sprint state from file.""" 

130 import json 

131 

132 state_file = _get_sprint_state_file() 

133 if not state_file.exists(): 

134 return None 

135 try: 

136 data = json.loads(state_file.read_text()) 

137 state = SprintState.from_dict(data) 

138 logger.info(f"State loaded from {state_file}") 

139 return state 

140 except (json.JSONDecodeError, KeyError) as e: 

141 logger.warning(f"Failed to load state: {e}") 

142 return None 

143 

144 

145def _save_sprint_state(state: SprintState, logger: Logger) -> None: 

146 """Save sprint state to file.""" 

147 import json 

148 from datetime import datetime 

149 

150 state.last_checkpoint = datetime.now().isoformat() 

151 state_file = _get_sprint_state_file() 

152 state_file.write_text(json.dumps(state.to_dict(), indent=2)) 

153 logger.info(f"State saved to {state_file}") 

154 

155 

156def _cleanup_sprint_state(logger: Logger) -> None: 

157 """Remove sprint state file.""" 

158 state_file = _get_sprint_state_file() 

159 if state_file.exists(): 

160 state_file.unlink() 

161 logger.info("Sprint state file cleaned up") 

162 

163 

164def _run_learning_gate_preflight( 

165 args: argparse.Namespace, 

166 issue_infos: list[IssueInfo], 

167 config: BRConfig, 

168 logger: Logger, 

169) -> int: 

170 """Run the learning-test pre-flight gate before sprint execution (ENH-2210). 

171 

172 Aggregates all learning_tests_required targets across sprint issues, 

173 deduplicates them (preserving first-occurrence order), and runs 

174 ready-to-implement-gate once. If any target is unproven the sprint aborts 

175 with a message identifying the blocking targets and the issue IDs that 

176 contributed each one. 

177 

178 Returns: 

179 0 if the gate passes or is skipped; 1 if the gate blocks the sprint. 

180 """ 

181 if not config.learning_tests.enabled: 

182 return 0 

183 

184 if getattr(args, "skip_learning_gate", False): 

185 logger.info("Learning gate pre-flight skipped (--skip-learning-gate)") 

186 return 0 

187 

188 from little_loops.learning_tests.extractor import resolve_learning_targets 

189 

190 # Map target → contributing issue IDs (for attribution in error output) 

191 target_to_issues: dict[str, list[str]] = {} 

192 seen: set[str] = set() 

193 all_targets: list[str] = [] 

194 

195 for info in issue_infos: 

196 targets = resolve_learning_targets(info) 

197 

198 for target in targets: 

199 if target not in seen: 

200 seen.add(target) 

201 all_targets.append(target) 

202 target_to_issues.setdefault(target, []).append(info.issue_id) 

203 

204 if not all_targets: 

205 return 0 

206 

207 logger.info( 

208 f"Learning gate pre-flight: checking {len(all_targets)} target(s): {', '.join(all_targets)}" 

209 ) 

210 

211 cmd = [ 

212 "ll-loop", 

213 "run", 

214 "ready-to-implement-gate", 

215 "--context", 

216 f"targets={','.join(all_targets)}", 

217 ] 

218 result = subprocess.run(cmd, capture_output=True, text=True) 

219 

220 if result.returncode != 0: 

221 logger.error("Learning gate pre-flight FAILED — sprint aborted") 

222 for target in all_targets: 

223 contributing = ", ".join(target_to_issues.get(target, [])) 

224 logger.error(f" Blocking target '{target}': required by {contributing}") 

225 return 1 

226 

227 logger.info("Learning gate pre-flight: all targets proven") 

228 return 0 

229 

230 

231def _cmd_sprint_run( 

232 args: argparse.Namespace, 

233 manager: SprintManager, 

234 config: BRConfig, 

235) -> int: 

236 """Execute a sprint with dependency-aware scheduling.""" 

237 from datetime import datetime 

238 

239 global _sprint_logger 

240 logger = Logger(verbose=not args.quiet, use_color=use_color_enabled()) 

241 _sprint_logger = logger 

242 

243 # Setup signal handlers for graceful shutdown (ENH-183) 

244 global _sprint_shutdown_requested 

245 _sprint_shutdown_requested = False # Reset in case of multiple runs 

246 signal.signal(signal.SIGINT, _sprint_signal_handler) 

247 signal.signal(signal.SIGTERM, _sprint_signal_handler) 

248 

249 handoff_threshold = getattr(args, "handoff_threshold", None) 

250 if handoff_threshold is not None: 

251 os.environ["LL_HANDOFF_THRESHOLD"] = str(handoff_threshold) 

252 

253 context_limit = getattr(args, "context_limit", None) 

254 if context_limit is not None: 

255 os.environ["LL_CONTEXT_LIMIT"] = str(context_limit) 

256 

257 sprint = manager.load_or_resolve(args.sprint) 

258 if not sprint: 

259 logger.error(f"Sprint not found: {args.sprint}") 

260 return 1 

261 

262 if not sprint.issues: 

263 logger.info(f"Sprint '{sprint.name}' has no active issues — nothing to process") 

264 return 0 

265 

266 if getattr(args, "save", False): 

267 saved_path = sprint.save(manager.sprints_dir) 

268 logger.info(f"Sprint saved to {saved_path}") 

269 

270 # Apply skip filter if provided 

271 issues_to_process = list(sprint.issues) 

272 skip_ids = parse_issue_ids(args.skip) 

273 if skip_ids: 

274 original_count = len(issues_to_process) 

275 issues_to_process = [i for i in issues_to_process if i not in skip_ids] 

276 skipped = original_count - len(issues_to_process) 

277 if skipped > 0: 

278 logger.info(f"Skipping {skipped} issue(s): {', '.join(sorted(skip_ids))}") 

279 

280 # Apply only filter if provided 

281 only_ids = parse_issue_ids(getattr(args, "only", None)) 

282 if only_ids: 

283 invalid_only = {p for p in only_ids if not any(_id_matches(i, p) for i in sprint.issues)} 

284 if invalid_only: 

285 logger.error( 

286 f"Issue(s) not found in sprint definition: {', '.join(sorted(invalid_only))}" 

287 ) 

288 return 1 

289 issues_to_process = [ 

290 i for i in issues_to_process if any(_id_matches(i, p) for p in only_ids) 

291 ] 

292 logger.info( 

293 f"Processing only {len(issues_to_process)} issue(s): {', '.join(sorted(only_ids))}" 

294 ) 

295 

296 # Apply type filter if provided 

297 type_prefixes = parse_issue_types(getattr(args, "type", None)) 

298 if type_prefixes: 

299 original_count = len(issues_to_process) 

300 issues_to_process = [i for i in issues_to_process if i.split("-", 1)[0] in type_prefixes] 

301 filtered = original_count - len(issues_to_process) 

302 if filtered > 0: 

303 logger.info(f"Filtered {filtered} issue(s) by type: {', '.join(sorted(type_prefixes))}") 

304 

305 # Pre-validate: skip issues with status: done/cancelled (ENH-1424) 

306 pre_completed_skipped: list[str] = [] 

307 

308 # Validate issues exist 

309 valid = manager.validate_issues(issues_to_process) 

310 invalid = set(issues_to_process) - set(valid.keys()) 

311 

312 if invalid: 

313 logger.error(f"Issue IDs not found: {', '.join(sorted(invalid))}") 

314 logger.info("Cannot execute sprint with missing issues") 

315 return 1 

316 

317 # Write milestone: field back to issue files for all valid issues in this sprint 

318 from little_loops.frontmatter import parse_frontmatter, update_frontmatter 

319 

320 for _issue_id, path in valid.items(): 

321 content = path.read_text(encoding="utf-8") 

322 updated = update_frontmatter(content, {"milestone": sprint.name}) 

323 if updated != content: 

324 path.write_text(updated, encoding="utf-8") 

325 logger.info(f"Set milestone: {sprint.name!r} on {len(valid)} issue file(s)") 

326 

327 # Skip issues already completed via frontmatter status (ENH-1424) 

328 still_active: list[str] = [] 

329 for issue_id in issues_to_process: 

330 path = valid[issue_id] 

331 fm = parse_frontmatter(path.read_text(encoding="utf-8")) 

332 if fm.get("status", "open") in ("done", "cancelled"): 

333 logger.info(f" {issue_id}: already completed (status: {fm.get('status')}), skipping") 

334 pre_completed_skipped.append(issue_id) 

335 else: 

336 still_active.append(issue_id) 

337 if pre_completed_skipped: 

338 logger.info( 

339 f"Pre-validation: {len(pre_completed_skipped)} issue(s) already completed, " 

340 f"{len(still_active)} active" 

341 ) 

342 issues_to_process = still_active 

343 

344 if pre_completed_skipped and not issues_to_process: 

345 logger.info("All sprint issues already completed - nothing to process") 

346 return 0 

347 

348 # Load full IssueInfo objects for dependency analysis 

349 issue_infos = manager.load_issue_infos(issues_to_process) 

350 if not issue_infos: 

351 logger.error("No issue files found") 

352 return 1 

353 

354 # Apply label filter if provided (must run after IssueInfo load since labels come from frontmatter) 

355 label_filter = parse_labels(getattr(args, "label", None)) 

356 if label_filter: 

357 original_count = len(issue_infos) 

358 issue_infos = [i for i in issue_infos if any(lb.lower() in label_filter for lb in i.labels)] 

359 filtered = original_count - len(issue_infos) 

360 if filtered > 0: 

361 logger.info(f"Filtered {filtered} issue(s) by label: {', '.join(sorted(label_filter))}") 

362 issues_to_process = [i.issue_id for i in issue_infos] 

363 

364 # Learning-test pre-flight gate (ENH-2210) 

365 preflight_result = _run_learning_gate_preflight(args, issue_infos, config, logger) 

366 if preflight_result != 0: 

367 return preflight_result 

368 

369 # Gather all issue IDs on disk to avoid false "nonexistent" warnings 

370 from little_loops.dependency_mapper import gather_all_issue_ids 

371 

372 issues_dir = config.project_root / config.issues.base_dir 

373 all_known_ids = gather_all_issue_ids(issues_dir, config=config) 

374 

375 # Dependency analysis (ENH-301) 

376 if not getattr(args, "skip_analysis", False): 

377 from little_loops.dependency_mapper import analyze_dependencies 

378 

379 issue_contents = _build_issue_contents(issue_infos) 

380 dep_config = config.dependency_mapping 

381 dep_report = analyze_dependencies( 

382 issue_infos, issue_contents, all_known_ids=all_known_ids, config=dep_config 

383 ) 

384 _render_dependency_analysis(dep_report, logger, config=dep_config) 

385 

386 # Build dependency graph 

387 dep_graph = DependencyGraph.from_issues(issue_infos, all_known_ids=all_known_ids) 

388 

389 # Detect cycles 

390 if dep_graph.has_cycles(): 

391 cycles = dep_graph.detect_cycles() 

392 for cycle in cycles: 

393 logger.error(f"Dependency cycle detected: {' -> '.join(cycle)}") 

394 return 1 

395 

396 # Get execution waves 

397 try: 

398 waves = dep_graph.get_execution_waves() 

399 except ValueError as e: 

400 logger.error(str(e)) 

401 return 1 

402 

403 # Refine waves for file overlap (ENH-306) 

404 waves, contention_notes = refine_waves_for_contention(waves, config=config.dependency_mapping) 

405 

406 # Display execution plan 

407 logger.info(f"Running sprint: {sprint.name}") 

408 logger.info("Dependency analysis:") 

409 for i, wave in enumerate(waves, 1): 

410 issue_ids = ", ".join(issue.issue_id for issue in wave) 

411 note = contention_notes[i - 1] if contention_notes else None 

412 if note: 

413 logger.info( 

414 f" Wave {i}: {issue_ids}" 

415 f" [sub-wave {note.sub_wave_index + 1}/{note.total_sub_waves}]" 

416 ) 

417 else: 

418 logger.info(f" Wave {i}: {issue_ids}") 

419 

420 # Warn when issues were serialized conservatively due to missing file hints (BUG-2142) 

421 if any(n is not None and getattr(n, "has_unknown_hints", False) for n in contention_notes): 

422 logger.warning( 

423 "Some issues lack '### Files to Modify' sections — serialized conservatively." 

424 " Add file hints for more accurate wave scheduling." 

425 ) 

426 

427 if args.dry_run: 

428 logger.info("\nDry run mode - no changes will be made") 

429 return 0 

430 

431 # Initialize or load state 

432 state: SprintState 

433 start_wave = 1 

434 

435 if args.resume: 

436 loaded_state = _load_sprint_state(logger) 

437 if loaded_state and loaded_state.sprint_name == sprint.name: 

438 state = loaded_state 

439 # Find first incomplete wave by checking completed issues 

440 completed_set = set(state.completed_issues) 

441 for i, wave in enumerate(waves, 1): 

442 wave_issue_ids = {issue.issue_id for issue in wave} 

443 if not wave_issue_ids.issubset(completed_set): 

444 start_wave = i 

445 break 

446 else: 

447 # All waves completed 

448 logger.info("Sprint already completed - nothing to resume") 

449 _cleanup_sprint_state(logger) 

450 return 0 

451 logger.info(f"Resuming from wave {start_wave}/{len(waves)}") 

452 logger.info(f" Previously completed: {len(state.completed_issues)} issues") 

453 else: 

454 if loaded_state: 

455 logger.warning( 

456 f"State file is for sprint '{loaded_state.sprint_name}', " 

457 f"not '{sprint.name}' - starting fresh" 

458 ) 

459 else: 

460 logger.warning("No valid state found - starting fresh") 

461 state = SprintState( 

462 sprint_name=sprint.name, 

463 started_at=datetime.now().isoformat(), 

464 ) 

465 else: 

466 # Fresh start - delete any old state 

467 _cleanup_sprint_state(logger) 

468 state = SprintState( 

469 sprint_name=sprint.name, 

470 started_at=datetime.now().isoformat(), 

471 ) 

472 

473 # Track exit status for error handling (ENH-185) 

474 exit_code = 0 

475 

476 try: 

477 # Determine max workers 

478 max_workers = args.max_workers or (sprint.options.max_workers if sprint.options else 2) 

479 

480 # Execute wave by wave 

481 completed: set[str] = set(state.completed_issues) 

482 failed_waves = 0 

483 total_duration = 0.0 

484 total_waves = len(waves) 

485 _fb_warning_emitted = False # deduplicate: fire once per sprint run (ENH-2176) 

486 

487 for wave_num, wave in enumerate(waves, 1): 

488 # Check for shutdown request (ENH-183) 

489 if _sprint_shutdown_requested: 

490 logger.warning("Shutdown requested - saving state and exiting") 

491 _save_sprint_state(state, logger) 

492 exit_code = 1 

493 return exit_code 

494 

495 # Skip already-completed waves when resuming 

496 if wave_num < start_wave: 

497 continue 

498 

499 wave_ids = [issue.issue_id for issue in wave] 

500 state.current_wave = wave_num 

501 logger.info(f"\nProcessing wave {wave_num}/{total_waves}: {', '.join(wave_ids)}") 

502 

503 wave_note = ( 

504 contention_notes[wave_num - 1] 

505 if contention_notes and wave_num - 1 < len(contention_notes) 

506 else None 

507 ) 

508 is_contention_subwave = wave_note is not None 

509 

510 if len(wave) == 1 or is_contention_subwave: 

511 # Single issue OR contention sub-wave — process in-place sequentially 

512 # (contention sub-waves are displayed as "serialized steps" so must run that way) 

513 max_wall_clock = config.sprints.max_issue_wall_clock_time 

514 

515 wave_failed = False 

516 _current_branch = _detect_current_branch() 

517 _feature_branches_arg = getattr(args, "feature_branches", None) 

518 effective_feature_branches = ( 

519 _feature_branches_arg 

520 if _feature_branches_arg is not None 

521 else config.parallel.use_feature_branches 

522 ) 

523 if effective_feature_branches and not _fb_warning_emitted: 

524 logger.warning( 

525 "feature-branch mode does not apply to single-issue / contention" 

526 f" sub-waves; these issues run in-place on '{_current_branch}'" 

527 ) 

528 _fb_warning_emitted = True 

529 for issue in wave: 

530 # TODO(ENH-1686): sprint sequential path not yet live-written 

531 issue_result = _run_issue_with_wall_clock_timeout( 

532 issue=issue, 

533 config=config, 

534 logger=logger, 

535 dry_run=args.dry_run, 

536 max_seconds=max_wall_clock, 

537 sprint_context=SprintWorkerContext( 

538 issue_id=issue.issue_id, 

539 branch=_current_branch, 

540 ), 

541 ) 

542 total_duration += issue_result.duration 

543 if issue_result.success: 

544 completed.add(issue.issue_id) 

545 state.completed_issues.append(issue.issue_id) 

546 state.timing[issue.issue_id] = {"total": issue_result.duration} 

547 logger.success(f" {issue.issue_id}: completed") 

548 elif issue_result.was_blocked: 

549 completed.add(issue.issue_id) 

550 state.skipped_blocked_issues[issue.issue_id] = issue_result.failure_reason 

551 logger.warning(f" {issue.issue_id}: skipped (blocked by open dependency)") 

552 else: 

553 wave_failed = True 

554 completed.add(issue.issue_id) 

555 state.failed_issues[issue.issue_id] = ( 

556 issue_result.failure_reason or "Issue processing failed" 

557 ) 

558 logger.warning(f" {issue.issue_id}: failed") 

559 if wave_failed: 

560 failed_waves += 1 

561 logger.warning(f"Wave {wave_num}/{total_waves} had failures") 

562 else: 

563 logger.success( 

564 f"Wave {wave_num}/{total_waves} completed: {', '.join(wave_ids)}" 

565 ) 

566 _save_sprint_state(state, logger) 

567 if wave_num < total_waves: 

568 logger.info(f"Continuing to wave {wave_num + 1}/{total_waves}...") 

569 # Check for shutdown before next wave (ENH-183) 

570 if _sprint_shutdown_requested: 

571 logger.warning("Shutdown requested - exiting after wave completion") 

572 exit_code = 1 

573 return exit_code 

574 else: 

575 # Multi-issue — use ParallelOrchestrator with worktrees 

576 only_ids = set(wave_ids) 

577 # Detect current branch for rebase/merge operations (BUG-439) 

578 _br = subprocess.run( 

579 ["git", "rev-parse", "--abbrev-ref", "HEAD"], 

580 capture_output=True, 

581 text=True, 

582 cwd=Path.cwd(), 

583 ) 

584 _base_branch = _br.stdout.strip() if _br.returncode == 0 else "main" 

585 # Runtime overlap detection disabled for sprints (ENH-512): 

586 # refine_waves_for_contention() already splits overlapping 

587 # issues into separate sub-waves before dispatch. 

588 parallel_config = config.create_parallel_config( 

589 max_workers=min(max_workers, len(wave)), 

590 only_ids=only_ids, 

591 dry_run=args.dry_run, 

592 overlap_detection=False, 

593 serialize_overlapping=True, 

594 base_branch=_base_branch, 

595 clean_start=True, # Sprint manages its own state; don't load stale orchestrator state 

596 use_feature_branches=getattr(args, "feature_branches", None), 

597 ) 

598 

599 from little_loops.events import EventBus 

600 from little_loops.extension import wire_extensions 

601 from little_loops.transport import wire_transports 

602 

603 event_bus = EventBus() 

604 wire_extensions(event_bus, config.extensions) 

605 wire_transports(event_bus, config.events) 

606 orchestrator = ParallelOrchestrator( 

607 parallel_config, 

608 config, 

609 Path.cwd(), 

610 wave_label=f"Wave {wave_num}/{total_waves}", 

611 event_bus=event_bus, 

612 ) 

613 result = orchestrator.run() 

614 total_duration += orchestrator.execution_duration 

615 

616 # Track completed/failed from this wave using per-issue results 

617 actually_completed = set(orchestrator.queue.completed_ids) 

618 actually_failed = set(orchestrator.queue.failed_ids) 

619 

620 for issue_id in wave_ids: 

621 if issue_id in actually_completed: 

622 completed.add(issue_id) 

623 state.completed_issues.append(issue_id) 

624 state.timing[issue_id] = { 

625 "total": orchestrator.execution_duration / len(wave) 

626 } 

627 elif issue_id in actually_failed: 

628 completed.add(issue_id) 

629 state.failed_issues[issue_id] = "Issue failed during wave execution" 

630 # else: issue was neither completed nor failed (interrupted/stranded) 

631 # — leave untracked so it can be retried on resume 

632 

633 # Sequential retry for failed issues (ENH-308) 

634 if actually_failed: 

635 logger.info(f"Retrying {len(actually_failed)} failed issue(s) sequentially...") 

636 retried_ok = 0 

637 for issue in wave: 

638 if issue.issue_id not in actually_failed: 

639 continue 

640 logger.info(f" Retrying {issue.issue_id} in-place...") 

641 # TODO(ENH-1686): sprint sequential path not yet live-written 

642 retry_result = process_issue_inplace( 

643 info=issue, 

644 config=config, 

645 logger=logger, 

646 dry_run=args.dry_run, 

647 sprint_context=SprintWorkerContext( 

648 issue_id=issue.issue_id, 

649 branch=_detect_current_branch(), 

650 ), 

651 ) 

652 total_duration += retry_result.duration 

653 if retry_result.success: 

654 retried_ok += 1 

655 state.failed_issues.pop(issue.issue_id, None) 

656 state.completed_issues.append(issue.issue_id) 

657 state.timing[issue.issue_id] = {"total": retry_result.duration} 

658 logger.success(f" Retry succeeded: {issue.issue_id}") 

659 elif retry_result.was_blocked: 

660 state.failed_issues.pop(issue.issue_id, None) 

661 state.skipped_blocked_issues[issue.issue_id] = ( 

662 retry_result.failure_reason 

663 ) 

664 logger.warning( 

665 f" Retry skipped: {issue.issue_id} (blocked by open dependency)" 

666 ) 

667 else: 

668 logger.warning(f" Retry failed: {issue.issue_id}") 

669 if retried_ok > 0: 

670 logger.info( 

671 f"Sequential retry recovered {retried_ok}/{len(actually_failed)} issue(s)" 

672 ) 

673 

674 # Check whether failures remain after retry (ENH-308) 

675 remaining_failures = {iid for iid in actually_failed if iid in state.failed_issues} 

676 if result == 0 or not remaining_failures: 

677 logger.success( 

678 f"Wave {wave_num}/{total_waves} completed: {', '.join(wave_ids)}" 

679 ) 

680 else: 

681 failed_waves += 1 

682 logger.warning(f"Wave {wave_num}/{total_waves} had failures") 

683 _save_sprint_state(state, logger) 

684 if wave_num < total_waves: 

685 logger.info(f"Continuing to wave {wave_num + 1}/{total_waves}...") 

686 # Check for shutdown before next wave (ENH-183) 

687 if _sprint_shutdown_requested: 

688 logger.warning("Shutdown requested - exiting after wave completion") 

689 exit_code = 1 

690 return exit_code 

691 

692 wave_word = "wave" if len(waves) == 1 else "waves" 

693 skip_msg = ( 

694 f", {len(pre_completed_skipped)} already completed (skipped)" 

695 if pre_completed_skipped 

696 else "" 

697 ) 

698 blocked_msg = ( 

699 f", {len(state.skipped_blocked_issues)} skipped (blocked)" 

700 if state.skipped_blocked_issues 

701 else "" 

702 ) 

703 logger.info( 

704 f"\nSprint completed: {len(completed)} issues processed " 

705 f"({len(waves)} {wave_word}){skip_msg}{blocked_msg}" 

706 ) 

707 logger.timing(f"Total execution time: {format_duration(total_duration)}") 

708 if failed_waves > 0: 

709 logger.warning(f"{failed_waves} wave(s) had failures") 

710 exit_code = 1 

711 else: 

712 # Clean up state on successful completion 

713 _cleanup_sprint_state(logger) 

714 

715 except KeyboardInterrupt: 

716 # Belt-and-suspenders with signal handler (ENH-185) 

717 logger.warning("Sprint interrupted by user (KeyboardInterrupt)") 

718 exit_code = 130 

719 

720 except Exception as e: 

721 # Catch unexpected exceptions (ENH-185) 

722 logger.error(f"Sprint failed unexpectedly: {e}") 

723 exit_code = 1 

724 

725 finally: 

726 # Guaranteed state save on any non-success exit (ENH-185) 

727 if exit_code != 0: 

728 _save_sprint_state(state, logger) 

729 logger.info("State saved before exit") 

730 

731 _sprint_logger = None # Clear for test isolation 

732 return exit_code