1"""Issue history parsing and scanning functions.

2

3 Provides functions to parse completed issue files, extract metadata

4 from frontmatter and content, scan directories for issues, and

5 extract file paths from issue content.

6"""

7

8 from __future__ import annotations

9

10 import logging

11 import re

12 import subprocess

13 from datetime import date, datetime

14 from pathlib import Path

15 from typing import Any

16

17 from little_loops.frontmatter import parse_frontmatter

18 from little_loops.issue_history.models import CompletedIssue

19 from little_loops.text_utils import extract_file_paths

20

21 logger = logging.getLogger(__name__)

22

23

24 def parse_completed_issue(

25 file_path: Path, *, batch_dates: dict[str, date] | None = None

26) -> CompletedIssue:

27 """Parse a completed issue file.

28

29 Args:

30 file_path: Path to the issue markdown file

31 batch_dates: Optional pre-fetched mapping of filename → add-date from a batch

32 git log call; when provided, skips the per-file subprocess call.

33

34 Returns:

35 CompletedIssue with parsed metadata

36 """

37 filename = file_path.name

38 content = file_path.read_text(encoding="utf-8")

39

40 # Extract from filename: P[0-5]-[TYPE]-[NNN]-description.md

41 issue_type = "UNKNOWN"

42 priority = "P5"

43 issue_id = "UNKNOWN"

44

45 # Match priority

46 priority_match = re.match(r"^(P\d)", filename)

47 if priority_match:

48 priority = priority_match.group(1)

49

50 # Match type and ID

51 type_match = re.search(r"(BUG|ENH|FEAT|EPIC)-(\d+)", filename)

52 if type_match:

53 issue_type = type_match.group(1)

54 issue_id = f"{type_match.group(1)}-{type_match.group(2)}"

55

56 # Parse frontmatter once for discovered_by, discovered_date, captured_at

57 fm = parse_frontmatter(content)

58 discovered_by = _parse_discovered_by(fm)

59 captured_at = _parse_captured_at(fm)

60 discovered_date = _parse_discovered_date(fm)

61

62 # Parse completion date from Resolution section or file mtime

63 completed_at = _parse_completed_at(fm)

64 completed_date = _parse_completion_date(content, file_path, batch_dates=batch_dates, fm=fm)

65

66 return CompletedIssue(

67 path=file_path,

68 issue_type=issue_type,

69 priority=priority,

70 issue_id=issue_id,

71 discovered_by=discovered_by,

72 discovered_date=discovered_date,

73 completed_date=completed_date,

74 captured_at=captured_at,

75 completed_at=completed_at,

76 )

77

78

79 def _parse_iso_datetime(value: Any) -> datetime | None:

80 """Parse an ISO 8601 string into a naive datetime, or return None.

81

82 Strips a trailing ``Z`` for Python <3.11 compatibility (same convention as

83 the sibling ``cli/issues/search.py`` implementation).

84 """

85 if not isinstance(value, str) or not value:

86 return None

87 try:

88 return datetime.fromisoformat(value.rstrip("Z")).replace(tzinfo=None)

89 except ValueError:

90 return None

91

92

93 def _parse_captured_at(fm: dict[str, Any]) -> datetime | None:

94 """Extract captured_at datetime from parsed frontmatter."""

95 return _parse_iso_datetime(fm.get("captured_at"))

96

97

98 def _parse_completed_at(fm: dict[str, Any]) -> datetime | None:

99 """Extract completed_at datetime from parsed frontmatter."""

100 return _parse_iso_datetime(fm.get("completed_at"))

101

102

103 def _parse_discovered_by(fm: dict[str, Any]) -> str | None:

104 """Extract discovered_by from parsed frontmatter.

105

106 Args:

107 fm: Parsed frontmatter dictionary

108

109 Returns:

110 discovered_by value or None

111 """

112 value = fm.get("discovered_by")

113 return value if isinstance(value, str) else None

114

115

116 def _batch_completion_dates(_issues_dir: Path) -> dict[str, date]:

117 """No-op stub kept for legacy callers.

118

119 The previous implementation used ``git log --diff-filter=A`` against

120 ``completed/`` to detect when issue files were *moved* into completion.

121 With status decoupled from directory location (ENH-1418), files no

122 longer move on completion; ``completed_at:`` frontmatter is the primary

123 source of truth, with a per-file ``git log -1`` fallback in

124 ``_parse_completion_date``. ENH-1420 will backfill ``completed_at`` for

125 pre-decoupling issues, after which the per-file fallback can also be

126 removed.

127

128 Args:

129 _issues_dir: Unused (kept for signature stability).

130

131 Returns:

132 An empty mapping.

133 """

134 return {}

135

136

137 def _parse_completion_date(

138 content: str,

139 file_path: Path,

140 *,

141 batch_dates: dict[str, date] | None = None,

142 fm: dict[str, Any] | None = None,

143) -> date | None:

144 """Extract completion date from frontmatter, Resolution section, or git log.

145

146 Checks ``completed_at`` frontmatter first (coerced to ``date`` via ``.date()``

147 to preserve the existing return type); then the Resolution section regex;

148 then falls back to batch_dates or a per-file git log call.

149

150 Args:

151 content: File content

152 file_path: Path for git log fallback

153 batch_dates: Optional pre-fetched mapping of filename → add-date from a batch

154 git log call; when provided, skips the per-file subprocess call if the

155 file is found in the mapping.

156 fm: Optional pre-parsed frontmatter dict. When absent, frontmatter is

157 parsed from ``content`` so external callers with no ``fm`` benefit

158 from the ``completed_at`` check transparently.

159

160 Returns:

161 Completion date or None

162 """

163 # Try completed_at frontmatter first (sub-day resolution source of truth)

164 if fm is None:

165 fm = parse_frontmatter(content)

166 completed_at = _parse_completed_at(fm)

167 if completed_at is not None:

168 return completed_at.date()

169

170 # Try Resolution section: **Completed/Fixed/Closed/Date**: YYYY-MM-DD

171 match = re.search(r"\*\*(?:Completed|Fixed|Closed|Date)\*\*:\s*(\d{4}-\d{2}-\d{2})", content)

172 if match:

173 try:

174 return date.fromisoformat(match.group(1))

175 except ValueError:

176 pass

177

178 # Check batch map before falling back to per-file git log

179 if batch_dates is not None:

180 return batch_dates.get(file_path.name)

181

182 # Fallback to git log: most recent commit date for this file (typically

183 # the close/done commit, since status writes are the latest change).

184 try:

185 result = subprocess.run(

186 ["git", "log", "--format=%as", "-1", "--", str(file_path)],

187 capture_output=True,

188 text=True,

189 cwd=file_path.parent,

190 )

191 if result.returncode == 0 and result.stdout.strip():

192 return date.fromisoformat(result.stdout.strip())

193 except (OSError, ValueError):

194 pass

195 return None

196

197

198 def _parse_resolution_action(content: str) -> str:

199 """Extract resolution action category from issue content.

200

201 Categorizes based on Resolution section fields:

202 - "completed": Normal completion with **Action**: fix/implement

203 - "rejected": Explicitly rejected (out of scope, not valid)

204 - "invalid": Invalid reference or spec

205 - "duplicate": Duplicate of existing issue

206 - "deferred": Deferred to future work

207

208 Args:

209 content: Issue file content

210

211 Returns:

212 Resolution category string

213 """

214 # Look for Status field patterns

215 status_match = re.search(r"\*\*Status\*\*:\s*(.+?)(?:\n|$)", content)

216 if status_match:

217 status = status_match.group(1).strip().lower()

218 if "closed" in status:

219 # Check Reason field for specific category

220 reason_match = re.search(r"\*\*Reason\*\*:\s*(.+?)(?:\n|$)", content)

221 if reason_match:

222 reason = reason_match.group(1).strip().lower()

223 if "duplicate" in reason:

224 return "duplicate"

225 if "invalid" in reason:

226 return "invalid"

227 if "deferred" in reason:

228 return "deferred"

229 if "rejected" in reason or "out of scope" in reason:

230 return "rejected"

231 # Generic closed without specific reason

232 return "rejected"

233

234 # Check for Action field (normal completion)

235 action_match = re.search(r"\*\*Action\*\*:\s*(.+?)(?:\n|$)", content)

236 if action_match:

237 return "completed"

238

239 # Default to completed if no resolution section

240 return "completed"

241

242

243 def _detect_processing_agent(content: str, discovered_source: str | None = None) -> str:

244 """Detect which processing agent handled an issue.

245

246 Detection strategy (in priority order):

247 1. Check discovered_source field for 'll-parallel' or 'll-auto'

248 2. Check content for '**Log Type**:' field

249 3. Check content for '**Tool**:' field

250 4. Default to 'manual'

251

252 Args:

253 content: Issue file content

254 discovered_source: Optional discovered_source frontmatter value

255

256 Returns:

257 Agent name: 'll-auto', 'll-parallel', or 'manual'

258 """

259 # Check discovered_source first

260 if discovered_source:

261 source_lower = discovered_source.lower()

262 if "ll-parallel" in source_lower:

263 return "ll-parallel"

264 if "ll-auto" in source_lower:

265 return "ll-auto"

266

267 # Check Log Type field

268 log_type_match = re.search(r"\*\*Log Type\*\*:\s*(.+?)(?:\n|$)", content)

269 if log_type_match:

270 log_type = log_type_match.group(1).strip().lower()

271 if "ll-parallel" in log_type:

272 return "ll-parallel"

273 if "ll-auto" in log_type:

274 return "ll-auto"

275

276 # Check Tool field

277 tool_match = re.search(r"\*\*Tool\*\*:\s*(.+?)(?:\n|$)", content)

278 if tool_match:

279 tool = tool_match.group(1).strip().lower()

280 if "ll-parallel" in tool:

281 return "ll-parallel"

282 if "ll-auto" in tool:

283 return "ll-auto"

284

285 # Default to manual

286 return "manual"

287

288

289 def scan_completed_issues(

290 issues_dir: Path,

291 category_dirs: list[str] | None = None,

292) -> list[CompletedIssue]:

293 """Scan type directories for issues with ``status: done`` frontmatter.

294

295 Files no longer move into a ``completed/`` subdirectory on completion

296 (ENH-1418). Completion is detected by ``status: done`` in the file's

297 YAML frontmatter; files remain in their original type directory

298 (``bugs/``, ``features/``, ``enhancements/``, ``epics/``).

299

300 For backwards compatibility with pre-decoupling repos, a sibling

301 ``completed/`` directory under ``issues_dir`` is also scanned when

302 present so legacy completed issues continue to surface.

303

304 Args:

305 issues_dir: Path to ``.issues/`` (the parent of category dirs).

306 category_dirs: Optional override of category subdirectories to scan.

307 Defaults to ``["bugs", "features", "enhancements", "epics"]``.

308

309 Returns:

310 List of parsed ``CompletedIssue`` objects, sorted by file path.

311 """

312 issues: list[CompletedIssue] = []

313

314 if not issues_dir.exists():

315 return issues

316

317 scan_dirs = category_dirs or ["bugs", "features", "enhancements", "epics"]

318 paths_to_scan: list[Path] = []

319 for category_dir in scan_dirs:

320 category_path = issues_dir / category_dir

321 if not category_path.exists():

322 continue

323 for file_path in category_path.glob("*.md"):

324 try:

325 content = file_path.read_text(encoding="utf-8")

326 fm = parse_frontmatter(content)

327 except Exception as e:

328 logger.warning("Failed to read %s: %s", file_path, e)

329 continue

330 if fm.get("status") != "done":

331 continue

332 paths_to_scan.append(file_path)

333

334 # Legacy completed/ directory (pre-ENH-1418); scan unconditionally

335 # so older repos keep working until ENH-1420 backfills.

336 legacy_completed = issues_dir / "completed"

337 if legacy_completed.exists():

338 paths_to_scan.extend(legacy_completed.glob("*.md"))

339

340 for file_path in sorted(paths_to_scan):

341 try:

342 issue = parse_completed_issue(file_path)

343 issues.append(issue)

344 except Exception as e:

345 logger.warning("Failed to parse %s: %s", file_path, e)

346 continue

347

348 return issues

349

350

351 def scan_completed_issues_from_db(db_path: Path) -> list[CompletedIssue]:

352 """Read completed-issue summary rows from the unified session DB (ENH-1621).

353

354 Queries the v2 ``issue_events`` table for rows with ``transition='done'``

355 and rebuilds :class:`CompletedIssue` dataclasses. Only the summary-relevant

356 fields (path, type, priority, id, completion timestamps) are populated —

357 ``analyze`` / ``export`` paths that need file bodies or git history must

358 continue to use :func:`scan_completed_issues`.

359

360 Returns ``[]`` when the DB is missing or the table has no done rows; the

361 caller treats an empty result as a signal to fall back to file parsing.

362 """

363 from little_loops.session_store import connect

364

365 if not db_path.exists():

366 return []

367 try:

368 conn = connect(db_path)

369 except Exception as exc: # pragma: no cover - defensive

370 logger.warning("Failed to open session DB %s: %s", db_path, exc)

371 return []

372

373 issues: list[CompletedIssue] = []

374 try:

375 try:

376 rows = conn.execute(

377 "SELECT issue_id, issue_type, priority, discovered_by, "

378 "captured_at, completed_at, completed_date "

379 "FROM issue_events WHERE transition = 'done'"

380 ).fetchall()

381 except Exception as exc: # pragma: no cover - schema mismatch / corruption

382 logger.warning("issue_events read failed for %s: %s", db_path, exc)

383 return []

384 finally:

385 conn.close()

386

387 for row in rows:

388 issue_id = row["issue_id"] or "UNKNOWN"

389 issue_type = row["issue_type"] or "UNKNOWN"

390 priority = row["priority"] or "P5"

391 captured_at = _parse_iso_datetime(row["captured_at"])

392 completed_at = _parse_iso_datetime(row["completed_at"])

393 completed_date_val = row["completed_date"]

394 completed_date: date | None = None

395 if isinstance(completed_date_val, str) and completed_date_val:

396 try:

397 completed_date = date.fromisoformat(completed_date_val[:10])

398 except ValueError:

399 completed_date = None

400 if completed_date is None and completed_at is not None:

401 completed_date = completed_at.date()

402 # discovered_date is not stored as a discrete column; derive from

403 # captured_at when present (mirrors `_parse_discovered_date`).

404 discovered_date = captured_at.date() if captured_at is not None else None

405 issues.append(

406 CompletedIssue(

407 path=Path(""), # not tracked in the DB row

408 issue_type=issue_type,

409 priority=priority,

410 issue_id=issue_id,

411 discovered_by=row["discovered_by"],

412 discovered_date=discovered_date,

413 completed_date=completed_date,

414 captured_at=captured_at,

415 completed_at=completed_at,

416 )

417 )

418 return issues

419

420

421 def _parse_discovered_date(fm: dict[str, Any]) -> date | None:

422 """Extract discovered date from parsed frontmatter.

423

424 Prefers ``captured_at`` (ISO datetime, sub-day resolution) when present,

425 coercing via ``.date()`` to preserve the legacy ``date | None`` return type

426 so callers in ``summary.py`` / ``analysis.py`` / ``cli/history.py`` don't

427 need ``.date()`` adjustments. Falls back to ``discovered_date`` on absence

428 or parse failure.

429

430 Args:

431 fm: Parsed frontmatter dictionary

432

433 Returns:

434 Discovered date or None

435 """

436 captured = _parse_captured_at(fm)

437 if captured is not None:

438 return captured.date()

439

440 value = fm.get("discovered_date")

441 if not isinstance(value, str):

442 return None

443 try:

444 return date.fromisoformat(value)

445 except ValueError:

446 return None

447

448

449 def _extract_subsystem(content: str) -> str | None:

450 """Extract primary subsystem/directory from issue content.

451

452 Args:

453 content: Issue file content

454

455 Returns:

456 Directory path (e.g., "scripts/little_loops/") or None

457 """

458 # Look for file paths in Location or common patterns

459 patterns = [

460 r"\*\*File\*\*:\s*`?([^`\n]+/)[^/`\n]+`?", # **File**: path/to/file.py

461 r"`([a-zA-Z_][\w/.-]+/)[^/`]+\.py`", # `path/to/file.py`

462 ]

463

464 for pattern in patterns:

465 match = re.search(pattern, content)

466 if match:

467 return match.group(1)

468

469 return None

470

471

472 def _extract_paths_from_issue(content: str) -> list[str]:

473 """Extract all file paths from issue content.

474

475 Delegates to :func:`~little_loops.text_utils.extract_file_paths`

476 and returns results as a sorted list for backward compatibility.

477

478 Args:

479 content: Issue file content

480

481 Returns:

482 Sorted list of file paths found in content

483 """

484 return sorted(extract_file_paths(content))

485

486

487 def _find_test_file(source_path: str, project_root: Path | None = None) -> str | None:

488 """Find corresponding test file for a source file.

489

490 Checks common test file naming patterns:

491 - tests/test_<name>.py

492 - tests/<path>/test_<name>.py

493 - <path>/test_<name>.py

494 - <path>/<name>_test.py

495 - <path>/tests/test_<name>.py

496

497 Args:

498 source_path: Path to source file (e.g., "src/core/processor.py")

499 project_root: Project root for anchoring existence checks. Defaults to CWD.

500

501 Returns:

502 Path to test file if found, None otherwise

503 """

504 if not source_path.endswith(".py"):

505 return None # Only check Python files for now

506

507 path = Path(source_path)

508 stem = path.stem # filename without extension

509 parent = str(path.parent) if path.parent != Path(".") else ""

510

511 # Generate candidate test file paths

512 candidates: list[str] = [

513 f"tests/test_{stem}.py",

514 f"{parent}/test_{stem}.py" if parent else f"test_{stem}.py",

515 f"{parent}/{stem}_test.py" if parent else f"{stem}_test.py",

516 f"{parent}/tests/test_{stem}.py" if parent else f"tests/test_{stem}.py",

517 ]

518

519 # Add path-aware test locations

520 if parent:

521 candidates.append(f"tests/{parent}/test_{stem}.py")

522

523 # Project-specific pattern for little-loops

524 # e.g., scripts/little_loops/foo.py -> scripts/tests/test_foo.py

525 if source_path.startswith("scripts/little_loops/"):

526 candidates.append(f"scripts/tests/test_{stem}.py")

527

528 for candidate in candidates:

529 if (project_root / candidate).exists() if project_root else Path(candidate).exists():

530 return candidate

531

532 return None

533

534

535 def scan_active_issues(

536 issues_dir: Path,

537 category_dirs: list[str] | None = None,

538) -> list[tuple[Path, str, str, date | None]]:

539 """Scan active issue directories.

540

541 Args:

542 issues_dir: Path to .issues/ directory

543 category_dirs: List of category subdirectory names to scan. When

544 omitted, defaults to ``["bugs", "features", "enhancements"]`` for

545 backward compatibility. Pass ``config.issue_categories`` to

546 include custom project categories.

547

548 Returns:

549 List of (path, issue_type, priority, discovered_date) tuples

550 """

551 results: list[tuple[Path, str, str, date | None]] = []

552

553 for category_dir in category_dirs or ["bugs", "features", "enhancements"]:

554 category_path = issues_dir / category_dir

555 if not category_path.exists():

556 continue

557

558 for file_path in category_path.glob("*.md"):

559 filename = file_path.name

560

561 # Extract priority

562 priority = "P5"

563 priority_match = re.match(r"^(P\d)", filename)

564 if priority_match:

565 priority = priority_match.group(1)

566

567 # Extract type

568 issue_type = "UNKNOWN"

569 type_match = re.search(r"(BUG|ENH|FEAT|EPIC)", filename)

570 if type_match:

571 issue_type = type_match.group(1)

572

573 # Extract discovered date from content

574 discovered_date = None

575 try:

576 content = file_path.read_text(encoding="utf-8")

577 fm = parse_frontmatter(content)

578 discovered_date = _parse_discovered_date(fm)

579 except Exception as e:

580 logger.warning("Failed to parse %s: %s", file_path, e)

581

582 results.append((file_path, issue_type, priority, discovered_date))

583

584 return results