1"""Documentation count verification utilities.

2

3 Provides automated verification that documented counts (commands, agents, skills)

4 match actual file counts in the codebase.

5"""

6

7 import re

8 from dataclasses import dataclass, field

9 from pathlib import Path

10

11 import yaml

12

13 from little_loops.fsm.validation import is_runnable_loop

14

15_DEFAULT_BUDGET_TOKENS = 2000

16_DEFAULT_PER_SKILL_WARN_TOKENS = 200

17

18# Documentation files to check

19 DOC_FILES = [

20 "README.md",

21 "CONTRIBUTING.md",

22 "docs/ARCHITECTURE.md",

23]

24

25# Directories to count

26 COUNT_TARGETS = {

27 "commands": ("commands", "*.md"),

28 "agents": ("agents", "*.md"),

29 "skills": ("skills", "*/SKILL.md"),

30 "loops": ("scripts/little_loops/loops", "*.yaml"),

31}

32

33# Bridge skills are auto-generated from commands/ and should be excluded from the skill count

34 BRIDGE_MARKER = "Bridged from `commands/"

35

36

37@dataclass

38 class CountResult:

39 """Result of counting files in a directory."""

40

41 category: str

42 actual: int

43 documented: int | None = None

44 file: str | None = None

45 line: int | None = None

46 matches: bool = True

47

48

49@dataclass

50 class VerificationResult:

51 """Overall verification result."""

52

53 total_checked: int = 0

54 mismatches: list[CountResult] = field(default_factory=list)

55 all_match: bool = True

56

57 def add_result(self, result: CountResult) -> None:

58 """Add a result and track mismatches."""

59 if not result.matches:

60 self.mismatches.append(result)

61 self.all_match = False

62

63

64@dataclass

65 class FixResult:

66 """Result of fixing counts."""

67

68 fixed_count: int

69 files_modified: list[str]

70

71

72 def count_files(directory: str, pattern: str, base_dir: Path | None = None) -> int:

73 """Count files matching pattern in directory.

74

75 Args:

76 directory: Directory name relative to base_dir

77 pattern: Glob pattern (e.g., "*.md" or "SKILL.md")

78 base_dir: Base directory path (defaults to current working directory)

79

80 Returns:

81 Number of matching files

82 """

83 if base_dir is None:

84 base_dir = Path.cwd()

85 dir_path = base_dir / directory

86 if not dir_path.exists():

87 return 0

88

89 return len(list(dir_path.glob(pattern)))

90

91

92 def extract_count_from_line(line: str, category: str) -> int | None:

93 """Extract count from a documentation line.

94

95 Handles multiple formats:

96 - "34 commands" or "34 slash commands"

97 - "8 agents" or "8 specialized agents"

98 - "6 skills" or "6 skill definitions"

99

100 Args:

101 line: Line text to search

102 category: Category name (commands, agents, skills)

103

104 Returns:

105 Extracted count or None if not found

106 """

107 # For skills, also match singular "skill" (e.g., "skill definitions")

108 # Pattern matches: number followed by optional words and category name

109 # Examples: "34 commands", "8 specialized agents", "6 skill definitions"

110 if category == "skills":

111 # Match both "skills" and "skill" (singular)

112 pattern = r"(\d+)\s+\w*\s*skills?(?!\s+description)"

113 else:

114 pattern = rf"(\d+)\s+\w*\s*{category}"

115

116 match = re.search(pattern, line, re.IGNORECASE)

117 return int(match.group(1)) if match else None

118

119

120 def verify_documentation(

121 base_dir: Path | None = None,

122) -> VerificationResult:

123 """Verify all documented counts against actual file counts.

124

125 Args:

126 base_dir: Base directory path (defaults to current working directory)

127

128 Returns:

129 VerificationResult with all results

130 """

131 if base_dir is None:

132 base_dir = Path.cwd()

133 result = VerificationResult(total_checked=0)

134

135 # Get actual counts

136 actual_counts: dict[str, int] = {}

137 for category, (directory, pattern) in COUNT_TARGETS.items():

138 actual_counts[category] = count_files(directory, pattern, base_dir)

139

140 # Loops live in nested subdirs (e.g. loops/oracles/) and share a directory

141 # with non-runnable library fragments (loops/lib/). Recursively enumerate

142 # and filter to runnable FSM definitions so the verifier stays in sync

143 # with `ll-loop validate`'s notion of "runnable".

144 loops_dir = base_dir / COUNT_TARGETS["loops"][0]

145 if loops_dir.exists():

146 actual_counts["loops"] = sum(1 for p in loops_dir.rglob("*.yaml") if is_runnable_loop(p))

147

148 # Adjust skill count to exclude bridge skills (auto-generated from commands/)

149 skills_dir = base_dir / "skills"

150 if "skills" in actual_counts and skills_dir.exists():

151 actual_counts["skills"] -= sum(

152 1 for p in skills_dir.glob("*/SKILL.md") if BRIDGE_MARKER in p.read_text()

153 )

154

155 # Check each documentation file

156 for doc_file in DOC_FILES:

157 doc_path = base_dir / doc_file

158 if not doc_path.exists():

159 continue

160

161 content = doc_path.read_text()

162 lines = content.splitlines()

163

164 for line_num, line in enumerate(lines, start=1):

165 for category in COUNT_TARGETS:

166 documented = extract_count_from_line(line, category)

167 if documented is not None:

168 actual = actual_counts[category]

169 matches = documented == actual

170

171 count_result = CountResult(

172 category=category,

173 actual=actual,

174 documented=documented,

175 file=str(doc_file),

176 line=line_num,

177 matches=matches,

178 )

179 result.add_result(count_result)

180 result.total_checked += 1

181

182 return result

183

184

185 def format_result_text(result: VerificationResult) -> str:

186 """Format verification result as text.

187

188 Args:

189 result: Verification result

190

191 Returns:

192 Formatted text output

193 """

194 lines = ["Documentation Count Verification", "=" * 40]

195

196 if result.all_match:

197 lines.append(f"✓ All {result.total_checked} count(s) match!")

198 else:

199 lines.append(f"✗ Found {len(result.mismatches)} mismatch(es):")

200 lines.append("")

201

202 for mismatch in result.mismatches:

203 lines.append(

204 f" {mismatch.category}: documented={mismatch.documented}, actual={mismatch.actual}"

205 )

206 lines.append(f" at {mismatch.file}:{mismatch.line}")

207

208 return "\n".join(lines)

209

210

211 def format_result_json(result: VerificationResult) -> str:

212 """Format verification result as JSON.

213

214 Args:

215 result: Verification result

216

217 Returns:

218 JSON string

219 """

220 import json

221

222 data = {

223 "all_match": result.all_match,

224 "total_checked": result.total_checked,

225 "mismatches": [

226 {

227 "category": m.category,

228 "documented": m.documented,

229 "actual": m.actual,

230 "file": m.file,

231 "line": m.line,

232 }

233 for m in result.mismatches

234 ],

235 }

236

237 return json.dumps(data, indent=2)

238

239

240 def format_result_markdown(result: VerificationResult) -> str:

241 """Format verification result as Markdown.

242

243 Args:

244 result: Verification result

245

246 Returns:

247 Markdown formatted string

248 """

249 lines = ["# Documentation Count Verification", ""]

250

251 if result.all_match:

252 lines.append("## ✅ All Counts Match")

253 lines.append(f"\nAll {result.total_checked} documented count(s) are accurate.")

254 else:

255 lines.append("## ❌ Mismatches Found")

256 lines.append("")

257 lines.append("| Category | Documented | Actual | Location |")

258 lines.append("|----------|-----------|--------|----------|")

259

260 for mismatch in result.mismatches:

261 lines.append(

262 f"| {mismatch.category} | {mismatch.documented} | "

263 f"{mismatch.actual} | `{mismatch.file}:{mismatch.line}` |"

264 )

265

266 return "\n".join(lines)

267

268

269@dataclass

270 class SkillBudgetResult:

271 """Result of checking skill description token budget."""

272

273 total_tokens: int

274 threshold_tokens: int

275 under_budget: bool

276 skill_breakdown: list[tuple[Path, str, int]]

277 violations: list[tuple[Path, str, int]]

278

279

280 def _parse_skill_frontmatter(text: str) -> dict[str, str]:

281 """Extract flat key/value pairs from SKILL.md frontmatter.

282

283 Uses yaml.safe_load so YAML block scalars (e.g. ``description: |``)

284 are resolved to their string content instead of the indicator literal.

285 Non-string scalar values are stringified; nested structures are dropped.

286

287 If the frontmatter is not valid YAML (e.g. unquoted colons in values),

288 falls back to a permissive line-based scan that mirrors the historical

289 behaviour — top-level ``key: value`` pairs, block scalars not supported.

290 """

291 if not text.startswith("---"):

292 return {}

293 end = text.find("---", 3)

294 if end == -1:

295 return {}

296 fm_text = text[3:end]

297 try:

298 loaded = yaml.safe_load(fm_text)

299 except yaml.YAMLError:

300 loaded = None

301 if isinstance(loaded, dict):

302 fm: dict[str, str] = {}

303 for key, value in loaded.items():

304 if value is None:

305 fm[str(key)] = ""

306 elif isinstance(value, str):

307 fm[str(key)] = value

308 elif isinstance(value, bool | int | float):

309 fm[str(key)] = str(value).lower() if isinstance(value, bool) else str(value)

310 return fm

311 fm = {}

312 for line in fm_text.splitlines():

313 if line and not line.startswith(" ") and not line.startswith("\t") and ":" in line:

314 key, _, val = line.partition(":")

315 fm[key.strip()] = val.strip()

316 return fm

317

318

319 def check_skill_budget(

320 base_dir: Path | None = None,

321 threshold_tokens: int = _DEFAULT_BUDGET_TOKENS,

322 per_skill_warn_tokens: int = _DEFAULT_PER_SKILL_WARN_TOKENS,

323) -> SkillBudgetResult:

324 """Scan skills/*/SKILL.md description fields, estimate tokens, check budget.

325

326 Skips skills with ``disable-model-invocation: true``. Token estimate uses

327 the character-count approximation ``len(description) // 4``.

328

329 Args:

330 base_dir: Base directory (defaults to cwd)

331 threshold_tokens: Total token budget (default: 2000 = ~1% of 200k context)

332 per_skill_warn_tokens: Per-skill threshold for listing as a violation

333

334 Returns:

335 SkillBudgetResult with total, sorted breakdown, and per-skill violations

336 """

337 if base_dir is None:

338 base_dir = Path.cwd()

339

340 skills_dir = base_dir / "skills"

341 skill_breakdown: list[tuple[Path, str, int]] = []

342

343 if skills_dir.exists():

344 for skill_md in sorted(skills_dir.glob("*/SKILL.md")):

345 try:

346 text = skill_md.read_text()

347 except OSError:

348 continue

349 fm = _parse_skill_frontmatter(text)

350 if fm.get("disable-model-invocation", "").lower() in ("true", "yes", "1"):

351 continue

352 description = fm.get("description", "")

353 tokens = len(description) // 4

354 skill_breakdown.append((skill_md, description, tokens))

355

356 skill_breakdown.sort(key=lambda x: x[2], reverse=True)

357 total_tokens = sum(t for _, _, t in skill_breakdown)

358 violations = [(p, d, t) for p, d, t in skill_breakdown if t >= per_skill_warn_tokens]

359

360 return SkillBudgetResult(

361 total_tokens=total_tokens,

362 threshold_tokens=threshold_tokens,

363 under_budget=total_tokens <= threshold_tokens,

364 skill_breakdown=skill_breakdown,

365 violations=violations,

366 )

367

368

369 def check_skill_sizes(

370 base_dir: Path | None = None,

371 limit: int = 500,

372) -> list[tuple[Path, int]]:

373 """Scan skills/*/SKILL.md files and return those exceeding the line limit.

374

375 Skips skills with ``disable-model-invocation: true``.

376

377 Args:

378 base_dir: Base directory (defaults to cwd)

379 limit: Maximum allowed lines per SKILL.md (default: 500)

380

381 Returns:

382 List of (path, line_count) pairs where line_count > limit

383 """

384 if base_dir is None:

385 base_dir = Path.cwd()

386

387 skills_dir = base_dir / "skills"

388 violations: list[tuple[Path, int]] = []

389

390 if not skills_dir.exists():

391 return violations

392

393 for skill_md in sorted(skills_dir.glob("*/SKILL.md")):

394 try:

395 text = skill_md.read_text()

396 except OSError:

397 continue

398 fm = _parse_skill_frontmatter(text)

399 if fm.get("disable-model-invocation", "").lower() in ("true", "yes", "1"):

400 continue

401 line_count = len(text.splitlines())

402 if line_count > limit:

403 violations.append((skill_md, line_count))

404

405 return violations

406

407

408 def fix_counts(base_dir: Path, result: VerificationResult) -> FixResult:

409 """Fix count mismatches in documentation files.

410

411 Args:

412 base_dir: Base directory path

413 result: Verification result with mismatches

414

415 Returns:

416 FixResult with counts of fixes made

417 """

418 files_modified: set[str] = set()

419 fixed_count = 0

420

421 # Group mismatches by file

422 mismatches_by_file: dict[str, list[CountResult]] = {}

423 for mismatch in result.mismatches:

424 if mismatch.file:

425 mismatches_by_file.setdefault(mismatch.file, []).append(mismatch)

426

427 # Fix each file

428 for file_path, mismatches in mismatches_by_file.items():

429 doc_path = base_dir / file_path

430 content = doc_path.read_text()

431 lines = content.splitlines()

432

433 for mismatch in mismatches:

434 if mismatch.line is not None and 1 <= mismatch.line <= len(lines):

435 line = lines[mismatch.line - 1]

436

437 # Build regex pattern based on category

438 # For skills, also match singular "skill"

439 if mismatch.category == "skills":

440 pattern = r"(\d+)(\s+\w*\s*skills?(?!\s+description))"

441 else:

442 pattern = rf"(\d+)(\s+\w*\s*{re.escape(mismatch.category)})"

443

444 # Replace the count while preserving the rest of the line

445 new_line = re.sub(

446 pattern,

447 str(mismatch.actual) + r"\2",

448 line,

449 count=1, # Only replace first occurrence

450 flags=re.IGNORECASE,

451 )

452

453 if new_line != line:

454 lines[mismatch.line - 1] = new_line

455 fixed_count += 1

456 files_modified.add(file_path)

457

458 # Write back if changes were made

459 if file_path in files_modified:

460 doc_path.write_text("\n".join(lines))

461

462 return FixResult(

463 fixed_count=fixed_count,

464 files_modified=list(files_modified),

465 )