Coverage for little_loops / doc_counts.py: 0%
198 statements
« prev ^ index » next coverage.py v7.12.0, created at 2026-06-04 12:21 -0500
1"""Documentation count verification utilities.
3Provides automated verification that documented counts (commands, agents, skills)
4match actual file counts in the codebase.
5"""
7import re
8from dataclasses import dataclass, field
9from pathlib import Path
11import yaml
13from little_loops.fsm.validation import is_runnable_loop
15_DEFAULT_BUDGET_TOKENS = 2000
16_DEFAULT_PER_SKILL_WARN_TOKENS = 200
18# Documentation files to check
19DOC_FILES = [
20 "README.md",
21 "CONTRIBUTING.md",
22 "docs/ARCHITECTURE.md",
23]
25# Directories to count
26COUNT_TARGETS = {
27 "commands": ("commands", "*.md"),
28 "agents": ("agents", "*.md"),
29 "skills": ("skills", "*/SKILL.md"),
30 "loops": ("scripts/little_loops/loops", "*.yaml"),
31}
33# Bridge skills are auto-generated from commands/ and should be excluded from the skill count
34BRIDGE_MARKER = "Bridged from `commands/"
37@dataclass
38class CountResult:
39 """Result of counting files in a directory."""
41 category: str
42 actual: int
43 documented: int | None = None
44 file: str | None = None
45 line: int | None = None
46 matches: bool = True
49@dataclass
50class VerificationResult:
51 """Overall verification result."""
53 total_checked: int = 0
54 mismatches: list[CountResult] = field(default_factory=list)
55 all_match: bool = True
57 def add_result(self, result: CountResult) -> None:
58 """Add a result and track mismatches."""
59 if not result.matches:
60 self.mismatches.append(result)
61 self.all_match = False
64@dataclass
65class FixResult:
66 """Result of fixing counts."""
68 fixed_count: int
69 files_modified: list[str]
72def count_files(directory: str, pattern: str, base_dir: Path | None = None) -> int:
73 """Count files matching pattern in directory.
75 Args:
76 directory: Directory name relative to base_dir
77 pattern: Glob pattern (e.g., "*.md" or "SKILL.md")
78 base_dir: Base directory path (defaults to current working directory)
80 Returns:
81 Number of matching files
82 """
83 if base_dir is None:
84 base_dir = Path.cwd()
85 dir_path = base_dir / directory
86 if not dir_path.exists():
87 return 0
89 return len(list(dir_path.glob(pattern)))
92def extract_count_from_line(line: str, category: str) -> int | None:
93 """Extract count from a documentation line.
95 Handles multiple formats:
96 - "34 commands" or "34 slash commands"
97 - "8 agents" or "8 specialized agents"
98 - "6 skills" or "6 skill definitions"
100 Args:
101 line: Line text to search
102 category: Category name (commands, agents, skills)
104 Returns:
105 Extracted count or None if not found
106 """
107 # For skills, also match singular "skill" (e.g., "skill definitions")
108 # Pattern matches: number followed by optional words and category name
109 # Examples: "34 commands", "8 specialized agents", "6 skill definitions"
110 if category == "skills":
111 # Match both "skills" and "skill" (singular)
112 pattern = r"(\d+)\s+\w*\s*skills?(?!\s+description)"
113 else:
114 pattern = rf"(\d+)\s+\w*\s*{category}"
116 match = re.search(pattern, line, re.IGNORECASE)
117 return int(match.group(1)) if match else None
120def verify_documentation(
121 base_dir: Path | None = None,
122) -> VerificationResult:
123 """Verify all documented counts against actual file counts.
125 Args:
126 base_dir: Base directory path (defaults to current working directory)
128 Returns:
129 VerificationResult with all results
130 """
131 if base_dir is None:
132 base_dir = Path.cwd()
133 result = VerificationResult(total_checked=0)
135 # Get actual counts
136 actual_counts: dict[str, int] = {}
137 for category, (directory, pattern) in COUNT_TARGETS.items():
138 actual_counts[category] = count_files(directory, pattern, base_dir)
140 # Loops live in nested subdirs (e.g. loops/oracles/) and share a directory
141 # with non-runnable library fragments (loops/lib/). Recursively enumerate
142 # and filter to runnable FSM definitions so the verifier stays in sync
143 # with `ll-loop validate`'s notion of "runnable".
144 loops_dir = base_dir / COUNT_TARGETS["loops"][0]
145 if loops_dir.exists():
146 actual_counts["loops"] = sum(1 for p in loops_dir.rglob("*.yaml") if is_runnable_loop(p))
148 # Adjust skill count to exclude bridge skills (auto-generated from commands/)
149 skills_dir = base_dir / "skills"
150 if "skills" in actual_counts and skills_dir.exists():
151 actual_counts["skills"] -= sum(
152 1 for p in skills_dir.glob("*/SKILL.md") if BRIDGE_MARKER in p.read_text()
153 )
155 # Check each documentation file
156 for doc_file in DOC_FILES:
157 doc_path = base_dir / doc_file
158 if not doc_path.exists():
159 continue
161 content = doc_path.read_text()
162 lines = content.splitlines()
164 for line_num, line in enumerate(lines, start=1):
165 for category in COUNT_TARGETS:
166 documented = extract_count_from_line(line, category)
167 if documented is not None:
168 actual = actual_counts[category]
169 matches = documented == actual
171 count_result = CountResult(
172 category=category,
173 actual=actual,
174 documented=documented,
175 file=str(doc_file),
176 line=line_num,
177 matches=matches,
178 )
179 result.add_result(count_result)
180 result.total_checked += 1
182 return result
185def format_result_text(result: VerificationResult) -> str:
186 """Format verification result as text.
188 Args:
189 result: Verification result
191 Returns:
192 Formatted text output
193 """
194 lines = ["Documentation Count Verification", "=" * 40]
196 if result.all_match:
197 lines.append(f"✓ All {result.total_checked} count(s) match!")
198 else:
199 lines.append(f"✗ Found {len(result.mismatches)} mismatch(es):")
200 lines.append("")
202 for mismatch in result.mismatches:
203 lines.append(
204 f" {mismatch.category}: documented={mismatch.documented}, actual={mismatch.actual}"
205 )
206 lines.append(f" at {mismatch.file}:{mismatch.line}")
208 return "\n".join(lines)
211def format_result_json(result: VerificationResult) -> str:
212 """Format verification result as JSON.
214 Args:
215 result: Verification result
217 Returns:
218 JSON string
219 """
220 import json
222 data = {
223 "all_match": result.all_match,
224 "total_checked": result.total_checked,
225 "mismatches": [
226 {
227 "category": m.category,
228 "documented": m.documented,
229 "actual": m.actual,
230 "file": m.file,
231 "line": m.line,
232 }
233 for m in result.mismatches
234 ],
235 }
237 return json.dumps(data, indent=2)
240def format_result_markdown(result: VerificationResult) -> str:
241 """Format verification result as Markdown.
243 Args:
244 result: Verification result
246 Returns:
247 Markdown formatted string
248 """
249 lines = ["# Documentation Count Verification", ""]
251 if result.all_match:
252 lines.append("## ✅ All Counts Match")
253 lines.append(f"\nAll {result.total_checked} documented count(s) are accurate.")
254 else:
255 lines.append("## ❌ Mismatches Found")
256 lines.append("")
257 lines.append("| Category | Documented | Actual | Location |")
258 lines.append("|----------|-----------|--------|----------|")
260 for mismatch in result.mismatches:
261 lines.append(
262 f"| {mismatch.category} | {mismatch.documented} | "
263 f"{mismatch.actual} | `{mismatch.file}:{mismatch.line}` |"
264 )
266 return "\n".join(lines)
269@dataclass
270class SkillBudgetResult:
271 """Result of checking skill description token budget."""
273 total_tokens: int
274 threshold_tokens: int
275 under_budget: bool
276 skill_breakdown: list[tuple[Path, str, int]]
277 violations: list[tuple[Path, str, int]]
280def _parse_skill_frontmatter(text: str) -> dict[str, str]:
281 """Extract flat key/value pairs from SKILL.md frontmatter.
283 Uses yaml.safe_load so YAML block scalars (e.g. ``description: |``)
284 are resolved to their string content instead of the indicator literal.
285 Non-string scalar values are stringified; nested structures are dropped.
287 If the frontmatter is not valid YAML (e.g. unquoted colons in values),
288 falls back to a permissive line-based scan that mirrors the historical
289 behaviour — top-level ``key: value`` pairs, block scalars not supported.
290 """
291 if not text.startswith("---"):
292 return {}
293 end = text.find("---", 3)
294 if end == -1:
295 return {}
296 fm_text = text[3:end]
297 try:
298 loaded = yaml.safe_load(fm_text)
299 except yaml.YAMLError:
300 loaded = None
301 if isinstance(loaded, dict):
302 fm: dict[str, str] = {}
303 for key, value in loaded.items():
304 if value is None:
305 fm[str(key)] = ""
306 elif isinstance(value, str):
307 fm[str(key)] = value
308 elif isinstance(value, bool | int | float):
309 fm[str(key)] = str(value).lower() if isinstance(value, bool) else str(value)
310 return fm
311 fm = {}
312 for line in fm_text.splitlines():
313 if line and not line.startswith(" ") and not line.startswith("\t") and ":" in line:
314 key, _, val = line.partition(":")
315 fm[key.strip()] = val.strip()
316 return fm
319def check_skill_budget(
320 base_dir: Path | None = None,
321 threshold_tokens: int = _DEFAULT_BUDGET_TOKENS,
322 per_skill_warn_tokens: int = _DEFAULT_PER_SKILL_WARN_TOKENS,
323) -> SkillBudgetResult:
324 """Scan skills/*/SKILL.md description fields, estimate tokens, check budget.
326 Skips skills with ``disable-model-invocation: true``. Token estimate uses
327 the character-count approximation ``len(description) // 4``.
329 Args:
330 base_dir: Base directory (defaults to cwd)
331 threshold_tokens: Total token budget (default: 2000 = ~1% of 200k context)
332 per_skill_warn_tokens: Per-skill threshold for listing as a violation
334 Returns:
335 SkillBudgetResult with total, sorted breakdown, and per-skill violations
336 """
337 if base_dir is None:
338 base_dir = Path.cwd()
340 skills_dir = base_dir / "skills"
341 skill_breakdown: list[tuple[Path, str, int]] = []
343 if skills_dir.exists():
344 for skill_md in sorted(skills_dir.glob("*/SKILL.md")):
345 try:
346 text = skill_md.read_text()
347 except OSError:
348 continue
349 fm = _parse_skill_frontmatter(text)
350 if fm.get("disable-model-invocation", "").lower() in ("true", "yes", "1"):
351 continue
352 description = fm.get("description", "")
353 tokens = len(description) // 4
354 skill_breakdown.append((skill_md, description, tokens))
356 skill_breakdown.sort(key=lambda x: x[2], reverse=True)
357 total_tokens = sum(t for _, _, t in skill_breakdown)
358 violations = [(p, d, t) for p, d, t in skill_breakdown if t >= per_skill_warn_tokens]
360 return SkillBudgetResult(
361 total_tokens=total_tokens,
362 threshold_tokens=threshold_tokens,
363 under_budget=total_tokens <= threshold_tokens,
364 skill_breakdown=skill_breakdown,
365 violations=violations,
366 )
369def check_skill_sizes(
370 base_dir: Path | None = None,
371 limit: int = 500,
372) -> list[tuple[Path, int]]:
373 """Scan skills/*/SKILL.md files and return those exceeding the line limit.
375 Skips skills with ``disable-model-invocation: true``.
377 Args:
378 base_dir: Base directory (defaults to cwd)
379 limit: Maximum allowed lines per SKILL.md (default: 500)
381 Returns:
382 List of (path, line_count) pairs where line_count > limit
383 """
384 if base_dir is None:
385 base_dir = Path.cwd()
387 skills_dir = base_dir / "skills"
388 violations: list[tuple[Path, int]] = []
390 if not skills_dir.exists():
391 return violations
393 for skill_md in sorted(skills_dir.glob("*/SKILL.md")):
394 try:
395 text = skill_md.read_text()
396 except OSError:
397 continue
398 fm = _parse_skill_frontmatter(text)
399 if fm.get("disable-model-invocation", "").lower() in ("true", "yes", "1"):
400 continue
401 line_count = len(text.splitlines())
402 if line_count > limit:
403 violations.append((skill_md, line_count))
405 return violations
408def fix_counts(base_dir: Path, result: VerificationResult) -> FixResult:
409 """Fix count mismatches in documentation files.
411 Args:
412 base_dir: Base directory path
413 result: Verification result with mismatches
415 Returns:
416 FixResult with counts of fixes made
417 """
418 files_modified: set[str] = set()
419 fixed_count = 0
421 # Group mismatches by file
422 mismatches_by_file: dict[str, list[CountResult]] = {}
423 for mismatch in result.mismatches:
424 if mismatch.file:
425 mismatches_by_file.setdefault(mismatch.file, []).append(mismatch)
427 # Fix each file
428 for file_path, mismatches in mismatches_by_file.items():
429 doc_path = base_dir / file_path
430 content = doc_path.read_text()
431 lines = content.splitlines()
433 for mismatch in mismatches:
434 if mismatch.line is not None and 1 <= mismatch.line <= len(lines):
435 line = lines[mismatch.line - 1]
437 # Build regex pattern based on category
438 # For skills, also match singular "skill"
439 if mismatch.category == "skills":
440 pattern = r"(\d+)(\s+\w*\s*skills?(?!\s+description))"
441 else:
442 pattern = rf"(\d+)(\s+\w*\s*{re.escape(mismatch.category)})"
444 # Replace the count while preserving the rest of the line
445 new_line = re.sub(
446 pattern,
447 str(mismatch.actual) + r"\2",
448 line,
449 count=1, # Only replace first occurrence
450 flags=re.IGNORECASE,
451 )
453 if new_line != line:
454 lines[mismatch.line - 1] = new_line
455 fixed_count += 1
456 files_modified.add(file_path)
458 # Write back if changes were made
459 if file_path in files_modified:
460 doc_path.write_text("\n".join(lines))
462 return FixResult(
463 fixed_count=fixed_count,
464 files_modified=list(files_modified),
465 )