Coverage for little_loops / cli / logs.py: 10%
1099 statements
« prev ^ index » next coverage.py v7.12.0, created at 2026-06-08 15:34 -0500
1"""ll-logs: Discover, extract, analyze log entries from ~/.claude/projects/."""
3from __future__ import annotations
5import argparse
6import json
7import re
8import shutil
9import sqlite3
10import sys
11import time
12from collections import Counter, defaultdict
13from dataclasses import dataclass
14from datetime import UTC, datetime, timedelta
15from pathlib import Path
17from little_loops.cli.loop.info import ( # private symbol: cross-module coupling; verify signature on upgrade
18 _format_history_event,
19)
20from little_loops.cli.output import configure_output, print_json, table, use_color_enabled
21from little_loops.cli_args import add_json_arg
22from little_loops.config import BRConfig
23from little_loops.logger import Logger
24from little_loops.session_store import DEFAULT_DB_PATH, cli_event_context, resolve_history_db
25from little_loops.user_messages import get_project_folder
27_COMMAND_NAME_RE = re.compile(r"<command-name>/ll:")
28BRIDGE_MARKER = "Bridged from `commands/"
31def _is_ll_relevant(record: dict) -> bool:
32 """Return True if a JSONL record indicates ll activity.
34 Detects three signal types:
35 (a) queue-operation enqueue with /ll: content
36 (b) user records with <command-name>/ll: pattern in message content
37 """
38 record_type = record.get("type")
40 # (a) queue-operation: only enqueue records with /ll: content signal ll activity
41 if record_type == "queue-operation":
42 return (
43 record.get("operation") == "enqueue"
44 and isinstance(record.get("content"), str)
45 and record["content"].startswith("/ll:")
46 )
48 # (b) user records: check message content for <command-name>/ll: pattern
49 if record_type == "user":
50 message = record.get("message", {})
51 if not isinstance(message, dict):
52 return False
53 content = message.get("content")
54 if isinstance(content, str):
55 return bool(_COMMAND_NAME_RE.search(content))
56 if isinstance(content, list):
57 for block in content:
58 if isinstance(block, dict):
59 text = block.get("text", "")
60 if isinstance(text, str) and _COMMAND_NAME_RE.search(text):
61 return True
63 # (c) assistant records: check for Bash tool-use invoking an ll- command
64 if record_type == "assistant":
65 message = record.get("message", {})
66 content = message.get("content", [])
67 if isinstance(content, list):
68 for block in content:
69 if (
70 isinstance(block, dict)
71 and block.get("type") == "tool_use"
72 and block.get("name") == "Bash"
73 ):
74 cmd = block.get("input", {}).get("command", "")
75 if re.search(r"\bll-\w+", cmd):
76 return True
78 return False
81def _has_ll_activity(project_folder: Path) -> bool:
82 """Return True if any non-agent JSONL file in project_folder has ll activity."""
83 jsonl_files = [f for f in project_folder.glob("*.jsonl") if not f.name.startswith("agent-")]
85 for jsonl_file in jsonl_files:
86 try:
87 with open(jsonl_file, encoding="utf-8") as f:
88 for line in f:
89 line = line.strip()
90 if not line:
91 continue
92 try:
93 record = json.loads(line)
94 except json.JSONDecodeError:
95 continue
96 if _is_ll_relevant(record):
97 return True
98 except OSError:
99 continue
101 return False
104def _extract_cwd_from_project(project_dir: Path) -> Path | None:
105 """Extract the project working directory from cwd fields in JSONL records.
107 Claude Code encodes project paths by replacing '/' with '-', which is
108 lossy for paths containing hyphens. Reading the cwd field from JSONL
109 records gives the canonical path without ambiguity.
110 """
111 jsonl_files = [f for f in project_dir.glob("*.jsonl") if not f.name.startswith("agent-")]
112 for jsonl_file in jsonl_files:
113 try:
114 with open(jsonl_file, encoding="utf-8") as f:
115 for line in f:
116 line = line.strip()
117 if not line:
118 continue
119 try:
120 record = json.loads(line)
121 cwd = record.get("cwd")
122 if isinstance(cwd, str) and cwd:
123 return Path(cwd)
124 except json.JSONDecodeError:
125 continue
126 except OSError:
127 continue
128 return None
131def discover_all_projects(logger: Logger, *, host: str | None = None) -> list[Path]:
132 """Discover all projects with ll activity for the given host.
134 Iterates the host's session directory (e.g. ``~/.claude/projects/`` for
135 Claude Code, ``~/.codex/projects/`` for Codex), resolves each directory
136 name back to an absolute path, checks for ll-relevant JSONL records, and
137 returns a sorted list of paths that exist on disk.
139 Args:
140 logger: Logger instance for warnings.
141 host: Host identifier. If None, auto-detects from ``LL_HOOK_HOST``
142 env var (default ``"claude-code"``).
144 Returns:
145 Sorted list of decoded absolute paths for projects with ll activity.
146 """
147 import os as _os
149 if host is None:
150 host = _os.environ.get("LL_HOOK_HOST", "claude-code")
152 if host == "claude-code":
153 projects_root = Path.home() / ".claude" / "projects"
154 elif host == "codex":
155 projects_root = Path.home() / ".codex" / "projects"
156 elif host == "opencode":
157 projects_root = Path.home() / ".opencode" / "projects"
158 elif host == "pi":
159 projects_root = Path.home() / ".pi" / "projects"
160 else:
161 return []
163 if not projects_root.exists():
164 return []
166 results: list[Path] = []
168 for project_dir in projects_root.iterdir():
169 if not project_dir.is_dir():
170 continue
172 # Prefer cwd field from JSONL records; fall back to lossy decode.
173 # The lossy decode ("-Users-foo-bar" -> "/Users/foo/bar") breaks for
174 # paths that contain hyphens (e.g. "little-loops", macOS per-user
175 # temp dirs like /tmp/claude-501/).
176 decoded_path = _extract_cwd_from_project(project_dir) or Path(
177 project_dir.name.replace("-", "/")
178 )
180 if not decoded_path.exists():
181 logger.warning(f"Decoded path does not exist: {decoded_path}")
182 continue
184 if _has_ll_activity(project_dir):
185 results.append(decoded_path)
187 return sorted(results)
190def _cmd_matches(record: dict, cmd: str) -> bool:
191 """Return True if record contains a Bash tool-use whose command includes cmd."""
192 message = record.get("message", {})
193 content = message.get("content", [])
194 if isinstance(content, list):
195 for block in content:
196 if (
197 isinstance(block, dict)
198 and block.get("type") == "tool_use"
199 and block.get("name") == "Bash"
200 ):
201 command = block.get("input", {}).get("command", "")
202 if cmd in command:
203 return True
204 return False
207_LL_BASH_RE = re.compile(r"\b(ll-[\w-]+)")
208_QUEUE_SKILL_RE = re.compile(r"^/ll:(\S+)")
209_COMMAND_NAME_SKILL_RE = re.compile(r"<command-name>/ll:(\S+)")
212@dataclass
213class InvocationEvent:
214 """A single ll invocation event extracted from a JSONL record."""
216 tool_name: str
217 timestamp: str
218 session_id: str
221def _extract_ll_event_streams(
222 project_folder: Path, *, window_days: int | None = None
223) -> dict[str, list[InvocationEvent]]:
224 """Extract per-session ordered ll-invocation event streams from JSONL files.
226 Walks JSONL files (skipping ``agent-*``), filters to records with ll activity,
227 extracts the tool/skill name, and returns a dict mapping ``sessionId`` to a
228 timestamp-sorted list of ``InvocationEvent``.
230 Args:
231 project_folder: Path to the claude project session directory.
232 window_days: If set, filter records to within N days of the latest record.
234 Returns:
235 Dict of ``{session_id: [InvocationEvent, ...]}`` with events sorted by timestamp.
236 """
237 events_by_session: dict[str, list[InvocationEvent]] = {}
239 jsonl_files = [f for f in project_folder.glob("*.jsonl") if not f.name.startswith("agent-")]
240 if not jsonl_files:
241 return events_by_session
243 # Collect all raw events first for window-days filtering
244 all_events: list[InvocationEvent] = []
245 latest_ts: str | None = None
247 for jsonl_file in jsonl_files:
248 try:
249 with open(jsonl_file, encoding="utf-8") as f:
250 for line in f:
251 line = line.strip()
252 if not line:
253 continue
254 try:
255 record = json.loads(line)
256 except json.JSONDecodeError:
257 continue
259 tool_name = _extract_tool_name(record)
260 if tool_name is None:
261 continue
263 ts = record.get("timestamp", "")
264 sid = record.get("sessionId", "")
266 evt = InvocationEvent(tool_name=tool_name, timestamp=ts, session_id=sid)
267 all_events.append(evt)
269 if ts and (latest_ts is None or ts > latest_ts):
270 latest_ts = ts
271 except OSError:
272 continue
274 # Apply window-days filter
275 if window_days is not None and latest_ts:
276 cutoff = _parse_iso_timestamp(latest_ts) - timedelta(days=window_days)
277 all_events = [e for e in all_events if _parse_iso_timestamp(e.timestamp) >= cutoff]
279 # Bucket by session and sort
280 for evt in all_events:
281 events_by_session.setdefault(evt.session_id, []).append(evt)
283 for session_id in events_by_session:
284 events_by_session[session_id].sort(key=lambda e: e.timestamp)
286 return events_by_session
289def _extract_tool_name(record: dict) -> str | None:
290 """Extract the ll tool/skill name from a JSONL record.
292 Detects three signal types (matching ``_is_ll_relevant``):
293 (a) queue-operation enqueue with ``/ll:<name>`` → skill name
294 (b) user records with ``<command-name>/ll:<name>`` → skill name
295 (c) assistant Bash tool-use invoking ``ll-<tool>`` → CLI tool name
296 """
297 record_type = record.get("type")
299 # (a) queue-operation enqueue
300 if record_type == "queue-operation" and record.get("operation") == "enqueue":
301 content = record.get("content", "")
302 if isinstance(content, str) and content.startswith("/ll:"):
303 m = _QUEUE_SKILL_RE.match(content)
304 if m:
305 return m.group(1)
307 # (b) user records with <command-name>/ll: pattern
308 if record_type == "user":
309 message = record.get("message", {})
310 if not isinstance(message, dict):
311 return None
312 content = message.get("content")
313 text = ""
314 if isinstance(content, str):
315 text = content
316 elif isinstance(content, list):
317 for block in content:
318 if isinstance(block, dict):
319 text = block.get("text", "")
320 if text:
321 break
322 if text:
323 m = _COMMAND_NAME_SKILL_RE.search(text)
324 if m:
325 # Extract skill name, stripping trailing </command-name> if present
326 name = m.group(1)
327 if name.endswith("</command-name>"):
328 name = name[: -len("</command-name>")]
329 return name
331 # (c) assistant Bash tool-use invoking ll-<tool>
332 if record_type == "assistant":
333 message = record.get("message", {})
334 content = message.get("content", [])
335 if isinstance(content, list):
336 for block in content:
337 if (
338 isinstance(block, dict)
339 and block.get("type") == "tool_use"
340 and block.get("name") == "Bash"
341 ):
342 cmd = block.get("input", {}).get("command", "")
343 m = _LL_BASH_RE.search(cmd)
344 if m:
345 return m.group(1)
347 return None
350def _parse_iso_timestamp(ts: str) -> datetime:
351 """Parse an ISO 8601 timestamp string to a timezone-aware datetime.
353 Handles both ``Z``-suffixed and ``+00:00`` offset formats. Returns
354 ``datetime.min`` with UTC tzinfo for unparseable input.
355 """
356 if not ts:
357 return datetime.min.replace(tzinfo=UTC)
358 try:
359 # Handle Z suffix
360 if ts.endswith("Z"):
361 ts = ts[:-1] + "+00:00"
362 dt = datetime.fromisoformat(ts)
363 if dt.tzinfo is None:
364 dt = dt.replace(tzinfo=UTC)
365 return dt
366 except (ValueError, TypeError):
367 return datetime.min.replace(tzinfo=UTC)
370@dataclass
371class Edge:
372 """A transition edge within an n-gram chain."""
374 from_: str
375 to: str
376 freq: float
379@dataclass
380class ChainResult:
381 """An n-gram chain with occurrence count and per-edge transition frequencies."""
383 chain: list[str]
384 count: int
385 edges: list[Edge]
387 def to_dict(self) -> dict:
388 return {
389 "chain": self.chain,
390 "count": self.count,
391 "edges": [{"from": e.from_, "to": e.to, "freq": e.freq} for e in self.edges],
392 }
395def _count_ngrams(
396 events_by_session: dict[str, list[InvocationEvent]],
397 min_len: int = 2,
398) -> Counter:
399 """Count n-grams across per-session event streams.
401 Args:
402 events_by_session: Per-session ordered event streams.
403 min_len: Minimum n-gram length (window size).
405 Returns:
406 Counter mapping ``(tool_1, tool_2, ...)`` tuples to occurrence counts.
407 """
408 counter: Counter = Counter()
409 for events in events_by_session.values():
410 names = [e.tool_name for e in events]
411 # For chains shorter than min_len, still consider them as single n-grams
412 # but note: min_len is the minimum, so we use range(min_len, len(names)+1)
413 for n in range(min_len, len(names) + 1):
414 for i in range(len(names) - n + 1):
415 ngram = tuple(names[i : i + n])
416 counter[ngram] += 1
417 return counter
420def _build_chain_results(
421 counter: Counter,
422 min_count: int = 1,
423 top: int | None = None,
424) -> list[ChainResult]:
425 """Build ranked ``ChainResult`` list from n-gram counter.
427 Args:
428 counter: n-gram counter from ``_count_ngrams``.
429 min_count: Minimum occurrence count to include.
430 top: If set, limit to top N chains by frequency.
432 Returns:
433 List of ``ChainResult`` sorted by count descending.
434 """
435 results: list[ChainResult] = []
436 for ngram, count in counter.most_common():
437 if count < min_count:
438 continue
439 edges = _compute_edges(ngram, counter)
440 results.append(ChainResult(chain=list(ngram), count=count, edges=edges))
442 if top is not None:
443 results = results[:top]
445 return results
448def _compute_edges(ngram: tuple[str, ...], counter: Counter) -> list[Edge]:
449 """Compute per-edge transition frequencies for an n-gram chain.
451 For each adjacent pair ``(from, to)`` in the chain, computes the frequency
452 as the proportion of times ``from → to`` appears out of all transitions
453 originating from ``from`` across the entire corpus.
454 """
455 edges: list[Edge] = []
456 # Build a transition counter for all pairs in the corpus
457 all_transitions: Counter = Counter()
458 out_degree: Counter = Counter()
459 for ngram_key, count in counter.items():
460 for i in range(len(ngram_key) - 1):
461 pair = (ngram_key[i], ngram_key[i + 1])
462 all_transitions[pair] += count
463 out_degree[ngram_key[i]] += count
465 for i in range(len(ngram) - 1):
466 from_ = ngram[i]
467 to = ngram[i + 1]
468 pair = (from_, to)
469 total_out = out_degree.get(from_, 0)
470 freq = all_transitions.get(pair, 0) / total_out if total_out > 0 else 0.0
471 edges.append(Edge(from_=from_, to=to, freq=round(freq, 4)))
473 return edges
476def _cmd_sequences(args: argparse.Namespace, logger: Logger) -> int:
477 """Extract n-grams of ll invocations from JSONL log files."""
478 if args.project:
479 cwd_path: Path = args.project
480 project_folder = get_project_folder(cwd_path)
481 if project_folder is None:
482 logger.error(f"No session project folder found for: {cwd_path}")
483 return 1
484 project_items = [(cwd_path, project_folder)]
485 else:
486 decoded_paths = discover_all_projects(logger)
487 project_items = []
488 for decoded_path in decoded_paths:
489 folder = get_project_folder(decoded_path)
490 if folder is not None:
491 project_items.append((decoded_path, folder))
493 # Aggregate events across all projects
494 all_events: dict[str, list[InvocationEvent]] = {}
495 for _cwd_path, project_folder in project_items:
496 events = _extract_ll_event_streams(project_folder, window_days=args.window_days)
497 for sid, evt_list in events.items():
498 all_events.setdefault(sid, []).extend(evt_list)
500 # Sort each session's events by timestamp
501 for sid in all_events:
502 all_events[sid].sort(key=lambda e: e.timestamp)
504 # Count n-grams
505 counter = _count_ngrams(all_events, min_len=args.min_len)
506 results = _build_chain_results(counter, min_count=args.min_count, top=args.top)
508 if args.json:
509 print_json([r.to_dict() for r in results])
510 else:
511 if not results:
512 print("No sequences found.")
513 return 0
515 # Print ranked table
516 for rank, r in enumerate(results, 1):
517 chain_str = " → ".join(r.chain)
518 print(f"{rank}. [{r.count}] {chain_str}")
519 for edge in r.edges:
520 print(f" {edge.from_} → {edge.to}: {edge.freq:.4f}")
522 return 0
525def generate_index(logs_dir: Path) -> None:
526 """Generate logs/index.md summarising extracted projects."""
527 rows = []
529 if logs_dir.exists():
530 for subdir in sorted(logs_dir.iterdir()):
531 if not subdir.is_dir():
532 continue
534 jsonl_files = [f for f in subdir.glob("*.jsonl") if not f.name.startswith("agent-")]
535 if not jsonl_files:
536 continue
538 timestamps: list[str] = []
539 for jsonl_file in jsonl_files:
540 try:
541 with open(jsonl_file, encoding="utf-8") as f:
542 for line in f:
543 line = line.strip()
544 if not line:
545 continue
546 try:
547 record = json.loads(line)
548 except json.JSONDecodeError:
549 continue
550 ts = record.get("timestamp")
551 if ts:
552 timestamps.append(ts)
553 except OSError:
554 continue
556 if timestamps:
557 earliest = min(timestamps)[:10]
558 latest = max(timestamps)[:10]
559 date_range = f"{earliest} – {latest}" if earliest != latest else earliest
560 else:
561 date_range = ""
563 rows.append((subdir.name, len(jsonl_files), date_range))
565 lines = ["# Logs Index", ""]
566 if rows:
567 lines.append("| Project | Sessions | Date Range |")
568 lines.append("|---------|----------|------------|")
569 for name, count, date_range in rows:
570 lines.append(f"| {name} | {count} | {date_range} |")
571 else:
572 lines.append("*No projects extracted yet.*")
573 lines.append("")
575 logs_dir.mkdir(parents=True, exist_ok=True)
576 (logs_dir / "index.md").write_text("\n".join(lines), encoding="utf-8")
579def _cmd_extract(args: argparse.Namespace, logger: Logger) -> int:
580 """Extract ll-relevant JSONL records to logs/<slug>/<session-id>.jsonl."""
581 if args.project:
582 cwd_path: Path = args.project
583 project_folder = get_project_folder(cwd_path)
584 if project_folder is None:
585 logger.error(f"No session project folder found for: {cwd_path}")
586 return 1
587 project_items = [(cwd_path, project_folder)]
588 else:
589 decoded_paths = discover_all_projects(logger)
590 project_items = []
591 for decoded_path in decoded_paths:
592 folder = get_project_folder(decoded_path)
593 if folder is not None:
594 project_items.append((decoded_path, folder))
596 for cwd_path, project_folder in project_items:
597 slug = cwd_path.resolve().name
598 buckets: dict[str, list[dict]] = {}
600 jsonl_files = [f for f in project_folder.glob("*.jsonl") if not f.name.startswith("agent-")]
601 for jsonl_file in jsonl_files:
602 try:
603 with open(jsonl_file, encoding="utf-8") as f:
604 for line in f:
605 line = line.strip()
606 if not line:
607 continue
608 try:
609 record = json.loads(line)
610 except json.JSONDecodeError:
611 continue
612 if _is_ll_relevant(record):
613 session_id = record.get("sessionId", "")
614 buckets.setdefault(session_id, []).append(record)
615 except OSError:
616 continue
618 if args.cmd:
619 filtered: dict[str, list[dict]] = {}
620 for session_id, records in buckets.items():
621 matching = [r for r in records if _cmd_matches(r, args.cmd)]
622 if matching:
623 filtered[session_id] = matching
624 buckets = filtered
626 out_base = Path.cwd() / "logs" / slug
627 for session_id, records in buckets.items():
628 out_file = out_base / f"{session_id}.jsonl"
629 out_file.parent.mkdir(parents=True, exist_ok=True)
630 with open(out_file, "w", encoding="utf-8") as f:
631 for record in records:
632 f.write(json.dumps(record) + "\n")
634 generate_index(Path.cwd() / "logs")
635 return 0
638def _cmd_tail(args: argparse.Namespace, loops_dir: Path) -> int:
639 """Stream live events from an active loop session."""
640 events_file = loops_dir / ".running" / f"{args.loop}.events.jsonl"
642 if not events_file.exists():
643 print(f"No active session for loop '{args.loop}'", file=sys.stderr)
644 return 1
646 width = shutil.get_terminal_size().columns
647 try:
648 with open(events_file, encoding="utf-8") as f:
649 f.seek(0, 2)
650 while True:
651 line = f.readline()
652 if line:
653 line = line.strip()
654 if line:
655 try:
656 event = json.loads(line)
657 except json.JSONDecodeError:
658 continue
659 formatted = _format_history_event(event, verbose=False, width=width)
660 if formatted is not None:
661 print(formatted)
662 else:
663 time.sleep(0.1)
664 except KeyboardInterrupt:
665 return 0
667 return 0
670_CORRECTION_WINDOW_SEC = 30
673def _aggregate_skill_stats(
674 db_path: Path,
675 *,
676 window_days: int | None = None,
677) -> dict[str, dict[str, int]] | None:
678 """Aggregate per-skill invocation and correction counts from history.db.
680 Returns None when the database is absent, or an empty dict when the database
681 has no skill_events rows. Corrections are attributed to the most recent skill
682 event in the same session within _CORRECTION_WINDOW_SEC seconds.
683 """
684 if not db_path.exists():
685 return None
687 conn = sqlite3.connect(str(db_path))
688 conn.row_factory = sqlite3.Row
689 try:
690 try:
691 skill_rows = conn.execute(
692 "SELECT ts, session_id, skill_name FROM skill_events ORDER BY ts"
693 ).fetchall()
694 except sqlite3.OperationalError:
695 return None
697 if not skill_rows:
698 return {}
700 if window_days is not None:
701 latest_ts = skill_rows[-1]["ts"] or ""
702 cutoff = _parse_iso_timestamp(latest_ts) - timedelta(days=window_days)
703 skill_rows = [r for r in skill_rows if _parse_iso_timestamp(r["ts"] or "") >= cutoff]
705 stats: dict[str, dict[str, int]] = defaultdict(lambda: {"invocations": 0, "corrections": 0})
706 for row in skill_rows:
707 stats[row["skill_name"] or "unknown"]["invocations"] += 1
709 session_skills: dict[str, list[tuple[str, str]]] = defaultdict(list)
710 for row in skill_rows:
711 sid = row["session_id"] or ""
712 session_skills[sid].append((row["ts"] or "", row["skill_name"] or "unknown"))
714 try:
715 corr_rows = conn.execute(
716 "SELECT ts, session_id FROM user_corrections ORDER BY ts"
717 ).fetchall()
718 except sqlite3.OperationalError:
719 corr_rows = []
721 for corr in corr_rows:
722 c_ts = corr["ts"] or ""
723 sid = corr["session_id"] or ""
724 candidates = session_skills.get(sid, [])
725 best_skill: str | None = None
726 best_ts: str = ""
727 for s_ts, s_name in candidates:
728 if s_ts <= c_ts and s_ts >= best_ts:
729 best_ts = s_ts
730 best_skill = s_name
731 if best_skill is not None:
732 elapsed = (
733 _parse_iso_timestamp(c_ts) - _parse_iso_timestamp(best_ts)
734 ).total_seconds()
735 if 0 <= elapsed <= _CORRECTION_WINDOW_SEC:
736 stats[best_skill]["corrections"] += 1
738 return dict(stats)
739 finally:
740 conn.close()
743def _load_catalog_names(root_dir: Path) -> set[str]:
744 """Load normalized skill/command names from skills/ and commands/ under root_dir.
746 Excludes bridge skills (containing BRIDGE_MARKER) and skills/commands with
747 disable-model-invocation: true. Normalizes names by stripping the "ll-" prefix
748 so catalog names match the skill_events.skill_name recording convention.
749 """
750 import yaml
752 names: set[str] = set()
754 skills_dir = root_dir / "skills"
755 if skills_dir.is_dir():
756 for skill_md in sorted(skills_dir.glob("*/SKILL.md")):
757 try:
758 text = skill_md.read_text()
759 except OSError:
760 continue
761 if BRIDGE_MARKER in text:
762 continue
763 name: str = skill_md.parent.name
764 if text.startswith("---"):
765 end = text.find("---", 3)
766 if end != -1:
767 try:
768 fm = yaml.safe_load(text[3:end]) or {}
769 except yaml.YAMLError:
770 fm = {}
771 if isinstance(fm, dict):
772 dmi = fm.get("disable-model-invocation")
773 if (isinstance(dmi, bool) and dmi) or (
774 isinstance(dmi, str) and dmi.strip().lower() in ("true", "yes", "1")
775 ):
776 continue
777 name = str(fm.get("name") or name)
778 if name.startswith("ll-"):
779 name = name[3:]
780 if name:
781 names.add(name)
783 commands_dir = root_dir / "commands"
784 if commands_dir.is_dir():
785 for cmd_md in sorted(commands_dir.glob("*.md")):
786 stem = cmd_md.stem
787 try:
788 text = cmd_md.read_text()
789 except OSError:
790 names.add(stem)
791 continue
792 if text.startswith("---"):
793 end = text.find("---", 3)
794 if end != -1:
795 try:
796 fm = yaml.safe_load(text[3:end]) or {}
797 except yaml.YAMLError:
798 fm = {}
799 if isinstance(fm, dict):
800 dmi = fm.get("disable-model-invocation")
801 if (isinstance(dmi, bool) and dmi) or (
802 isinstance(dmi, str) and dmi.strip().lower() in ("true", "yes", "1")
803 ):
804 continue
805 names.add(stem)
807 return names
810def _cmd_dead_skills(args: argparse.Namespace, logger: Logger) -> int:
811 """List catalog skills/commands never or rarely invoked within the window."""
812 if args.project:
813 db_paths = [Path(args.project) / ".ll" / "history.db"]
814 catalog_root = Path(args.project)
815 else:
816 decoded_paths = discover_all_projects(logger)
817 db_paths = [p / ".ll" / "history.db" for p in decoded_paths]
818 catalog_root = Path.cwd()
820 merged: dict[str, int] = defaultdict(int)
821 for db_path in db_paths:
822 result = _aggregate_skill_stats(db_path, window_days=args.window_days)
823 if result is None:
824 continue
825 for skill, counts in result.items():
826 merged[skill] += counts["invocations"]
828 catalog_names = _load_catalog_names(catalog_root)
829 if not catalog_names:
830 logger.warning(
831 "No catalog skills found — run from an ll project root with skills/ directory."
832 )
833 return 0
835 threshold = args.threshold
836 rows = []
837 for name in sorted(catalog_names):
838 count = merged.get(name, 0)
839 if count == 0:
840 rows.append({"skill": name, "invocations": 0, "tier": "never"})
841 elif count <= threshold:
842 rows.append({"skill": name, "invocations": count, "tier": "rarely"})
844 if args.json:
845 print_json(rows)
846 return 0
848 if not rows:
849 print("No dead or rarely-invoked skills found.")
850 return 0
852 headers = ["Skill", "Invocations", "Tier"]
853 table_rows = [[str(r["skill"]), str(r["invocations"]), str(r["tier"])] for r in rows]
854 print(table(headers, table_rows))
855 return 0
858_STACK_FRAME_RE = re.compile(r'\s*File "[^"]+", line \d+[^\n]*')
859_ABS_PATH_RE = re.compile(r"/(?:[^\s,;\"']+/)+[^\s,;\"']+")
860_LINE_NUM_RE = re.compile(r"\bline \d+\b")
861_LL_VERIFY_RE = re.compile(r"^ll-verify-\w+")
864def _normalize_error_sig(text: str) -> str:
865 """Strip volatile parts (paths, line numbers, stack frames) from error text.
867 Returns a stable string suitable as a cluster key.
868 """
869 text = _STACK_FRAME_RE.sub("", text)
870 text = _ABS_PATH_RE.sub("<path>", text)
871 text = _LINE_NUM_RE.sub("line N", text)
872 return re.sub(r"\s+", " ", text).strip()[:300]
875def _extract_error_text(content: object) -> str:
876 """Extract plain text from a tool_result content field (string or list of text blocks)."""
877 if isinstance(content, str):
878 return content
879 if isinstance(content, list):
880 parts: list[str] = []
881 for item in content:
882 if isinstance(item, dict):
883 text = item.get("text", "")
884 if isinstance(text, str):
885 parts.append(text)
886 return "\n".join(parts)
887 return ""
890@dataclass
891class _FailureCluster:
892 """Aggregated failure cluster keyed on (tool_name, normalized_sig)."""
894 tool_name: str
895 normalized_sig: str
896 count: int
897 sample_error: str
898 session_ids: list[str]
901def _cmd_scan_failures(args: argparse.Namespace, logger: Logger) -> int:
902 """Mine failed ll-* Bash calls from interactive session JSONL logs."""
903 from little_loops.issue_lifecycle import FailureType, classify_failure
905 if args.project:
906 cwd_path: Path = args.project
907 project_folder = get_project_folder(cwd_path)
908 if project_folder is None:
909 logger.error(f"No session project folder found for: {cwd_path}")
910 return 1
911 project_items = [(cwd_path, project_folder)]
912 else:
913 decoded_paths = discover_all_projects(logger)
914 project_items = []
915 for decoded_path in decoded_paths:
916 folder = get_project_folder(decoded_path)
917 if folder is not None:
918 project_items.append((decoded_path, folder))
920 # raw_clusters maps (tool_name, normalized_sig) -> (count, sample_error, session_ids, latest_ts)
921 raw_clusters: dict[tuple[str, str], tuple[int, str, list[str], str]] = {}
922 latest_ts_overall: str = ""
924 for _cwd_path, project_folder in project_items:
925 jsonl_files = [f for f in project_folder.glob("*.jsonl") if not f.name.startswith("agent-")]
927 for jsonl_file in jsonl_files:
928 try:
929 with open(jsonl_file, encoding="utf-8") as f:
930 lines = f.readlines()
931 except OSError:
932 continue
934 # pending maps tool_use_id -> (ll_tool_name, timestamp)
935 pending: dict[str, tuple[str, str]] = {}
937 for line in lines:
938 line = line.strip()
939 if not line:
940 continue
941 try:
942 record = json.loads(line)
943 except json.JSONDecodeError:
944 continue
946 record_type = record.get("type")
947 ts = record.get("timestamp", "")
948 session_id = record.get("sessionId", "")
950 if ts and ts > latest_ts_overall:
951 latest_ts_overall = ts
953 if record_type == "assistant":
954 message = record.get("message", {})
955 content = message.get("content", [])
956 if not isinstance(content, list):
957 continue
958 for block in content:
959 if not isinstance(block, dict):
960 continue
961 if block.get("type") != "tool_use" or block.get("name") != "Bash":
962 continue
963 cmd = block.get("input", {}).get("command", "")
964 m = _LL_BASH_RE.search(cmd)
965 if not m:
966 continue
967 tool_name = m.group(1)
968 block_id = block.get("id", "")
969 if block_id:
970 pending[block_id] = (tool_name, ts)
972 elif record_type == "user":
973 message = record.get("message", {})
974 content = message.get("content", [])
975 if not isinstance(content, list) or not content:
976 continue
977 for block in content:
978 if not isinstance(block, dict):
979 continue
980 if block.get("type") != "tool_result":
981 continue
982 tool_use_id = block.get("tool_use_id", "")
983 if tool_use_id not in pending:
984 continue
985 tool_name, _invoke_ts = pending.pop(tool_use_id)
987 # Skip ll-verify-* tools — exit 1 is expected gate behavior
988 if _LL_VERIFY_RE.match(tool_name):
989 continue
991 is_error_flag = block.get("is_error") is True
992 raw_content = block.get("content", "")
993 error_text = _extract_error_text(raw_content)
994 has_traceback = "Traceback (most recent call last)" in error_text
996 if not (is_error_flag or has_traceback):
997 continue
999 returncode = 1 if is_error_flag else 0
1000 failure_type, _reason = classify_failure(error_text, returncode)
1001 if failure_type == FailureType.TRANSIENT:
1002 continue
1004 normalized_sig = _normalize_error_sig(error_text)
1005 key = (tool_name, normalized_sig)
1007 if key in raw_clusters:
1008 cnt, sample, sids, _lts = raw_clusters[key]
1009 if session_id not in sids:
1010 sids.append(session_id)
1011 raw_clusters[key] = (cnt + 1, sample, sids, ts)
1012 else:
1013 raw_clusters[key] = (1, error_text[:500], [session_id], ts)
1015 # Apply window-days filter using the latest seen timestamp as anchor
1016 if args.window_days is not None and latest_ts_overall:
1017 cutoff = _parse_iso_timestamp(latest_ts_overall) - timedelta(days=args.window_days)
1018 raw_clusters = {
1019 k: v for k, v in raw_clusters.items() if _parse_iso_timestamp(v[3]) >= cutoff
1020 }
1022 clusters: list[_FailureCluster] = sorted(
1023 [
1024 _FailureCluster(
1025 tool_name=k[0],
1026 normalized_sig=k[1],
1027 count=v[0],
1028 sample_error=v[1],
1029 session_ids=v[2],
1030 )
1031 for k, v in raw_clusters.items()
1032 ],
1033 key=lambda c: c.count,
1034 reverse=True,
1035 )
1037 if not clusters:
1038 if not args.json:
1039 print("No ll-* failures found.")
1040 else:
1041 print_json([])
1042 return 0
1044 if args.capture:
1045 return _capture_failure_clusters(clusters, logger)
1047 if args.json:
1048 print_json(
1049 [
1050 {
1051 "tool": c.tool_name,
1052 "count": c.count,
1053 "normalized_sig": c.normalized_sig,
1054 "sample_error": c.sample_error,
1055 "session_ids": c.session_ids,
1056 }
1057 for c in clusters
1058 ]
1059 )
1060 return 0
1062 for c in clusters:
1063 print(f"[{c.count}x] {c.tool_name}")
1064 print(f" Sessions: {', '.join(c.session_ids[:5])}")
1065 for sl in c.sample_error.splitlines()[:5]:
1066 print(f" {sl}")
1067 print()
1069 return 0
1072def _capture_failure_clusters(clusters: list[_FailureCluster], logger: Logger) -> int:
1073 """Create bug issue files for each distinct failure cluster (--capture mode)."""
1074 from little_loops.issue_lifecycle import create_issue_from_failure
1075 from little_loops.issue_parser import IssueInfo
1077 config = BRConfig(Path.cwd())
1078 created = 0
1080 for c in clusters:
1081 stub_info = IssueInfo(
1082 path=Path(f"cli/{c.tool_name}"),
1083 issue_type="bugs",
1084 priority="P1",
1085 issue_id=c.tool_name,
1086 title=f"Tool failure in {c.tool_name}",
1087 )
1088 result = create_issue_from_failure(c.sample_error, stub_info, config, logger)
1089 if result is not None:
1090 logger.info(f"Created: {result.name}")
1091 created += 1
1093 logger.info(f"Captured {created} failure cluster(s) as bug issues.")
1094 return 0
1097def _cmd_stats(args: argparse.Namespace, logger: Logger) -> int:
1098 """Aggregate skill invocation frequency and correction rate from history.db."""
1099 if args.project:
1100 db_paths = [Path(args.project) / ".ll" / "history.db"]
1101 else:
1102 decoded_paths = discover_all_projects(logger)
1103 db_paths = [p / ".ll" / "history.db" for p in decoded_paths]
1105 merged: dict[str, dict[str, int]] = defaultdict(lambda: {"invocations": 0, "corrections": 0})
1106 found_any_db = False
1107 for db_path in db_paths:
1108 result = _aggregate_skill_stats(db_path, window_days=args.window_days)
1109 if result is None:
1110 continue
1111 found_any_db = True
1112 for skill, counts in result.items():
1113 merged[skill]["invocations"] += counts["invocations"]
1114 merged[skill]["corrections"] += counts["corrections"]
1116 if not merged:
1117 if not found_any_db:
1118 logger.warning("No history.db found — run with an active ll project.")
1119 else:
1120 print("No skill events recorded yet.")
1121 return 0
1123 sort_key = getattr(args, "sort", "freq")
1124 if sort_key == "corrections":
1125 ranked = sorted(merged.items(), key=lambda kv: kv[1]["corrections"], reverse=True)
1126 else:
1127 ranked = sorted(merged.items(), key=lambda kv: kv[1]["invocations"], reverse=True)
1129 if args.json:
1130 rows_json = [
1131 {
1132 "skill": skill,
1133 "invocations": counts["invocations"],
1134 "corrections": counts["corrections"],
1135 "correction_rate": (
1136 round(counts["corrections"] / counts["invocations"], 4)
1137 if counts["invocations"] > 0
1138 else 0.0
1139 ),
1140 "errors": None,
1141 "error_rate": None,
1142 }
1143 for skill, counts in ranked
1144 ]
1145 print_json(rows_json)
1146 return 0
1148 headers = ["Skill", "Invocations", "Corrections", "Corr%", "Errors"]
1149 rows = []
1150 for skill, counts in ranked:
1151 inv = counts["invocations"]
1152 corr = counts["corrections"]
1153 corr_pct = f"{corr / inv * 100:.1f}%" if inv > 0 else "0.0%"
1154 rows.append([skill, str(inv), str(corr), corr_pct, "N/A"])
1156 print(table(headers, rows))
1157 return 0
1160def _resolve_session_log(session_ref: str, db_path: Path) -> Path | None:
1161 """Resolve a session reference (session ID or JSONL path) to a JSONL file path.
1163 Tries in order:
1164 1. Direct file path if the ref resolves to an existing ``.jsonl`` file
1165 2. DB lookup of ``session_id → jsonl_path`` in the sessions table
1166 Returns None if unresolvable.
1167 """
1168 candidate = Path(session_ref)
1169 if candidate.suffix == ".jsonl" and candidate.exists():
1170 return candidate
1172 if db_path.exists():
1173 conn = sqlite3.connect(str(db_path))
1174 conn.row_factory = sqlite3.Row
1175 try:
1176 row = conn.execute(
1177 "SELECT jsonl_path FROM sessions WHERE session_id = ?",
1178 (session_ref,),
1179 ).fetchone()
1180 if row and row["jsonl_path"]:
1181 return Path(row["jsonl_path"])
1182 except sqlite3.OperationalError:
1183 pass
1184 finally:
1185 conn.close()
1187 return None
1190def _events_from_jsonl(jsonl_path: Path) -> list[InvocationEvent]:
1191 """Extract ll invocation events from a single JSONL file, sorted by timestamp."""
1192 events: list[InvocationEvent] = []
1193 try:
1194 with open(jsonl_path, encoding="utf-8") as f:
1195 for line in f:
1196 line = line.strip()
1197 if not line:
1198 continue
1199 try:
1200 record = json.loads(line)
1201 except json.JSONDecodeError:
1202 continue
1203 tool_name = _extract_tool_name(record)
1204 if tool_name is None:
1205 continue
1206 ts = record.get("timestamp", "")
1207 sid = record.get("sessionId", "")
1208 events.append(InvocationEvent(tool_name=tool_name, timestamp=ts, session_id=sid))
1209 except OSError:
1210 pass
1211 events.sort(key=lambda e: e.timestamp)
1212 return events
1215@dataclass
1216class SessionDiff:
1217 """Behavioral diff between two ll sessions."""
1219 session_a: str
1220 session_b: str
1221 skills_added: list[str]
1222 skills_removed: list[str]
1223 count_deltas: dict[str, dict[str, int]]
1224 sequence_diff: list[str]
1226 def to_dict(self) -> dict:
1227 return {
1228 "session_a": self.session_a,
1229 "session_b": self.session_b,
1230 "skills_added": self.skills_added,
1231 "skills_removed": self.skills_removed,
1232 "count_deltas": self.count_deltas,
1233 "sequence_diff": self.sequence_diff,
1234 }
1237def _compute_session_diff(
1238 session_a: str,
1239 events_a: list[InvocationEvent],
1240 session_b: str,
1241 events_b: list[InvocationEvent],
1242) -> SessionDiff:
1243 """Compute the behavioral diff between two session event streams."""
1244 import difflib
1245 from collections import Counter as _Counter
1247 names_a = [e.tool_name for e in events_a]
1248 names_b = [e.tool_name for e in events_b]
1250 set_a = set(names_a)
1251 set_b = set(names_b)
1252 skills_added = sorted(set_b - set_a)
1253 skills_removed = sorted(set_a - set_b)
1255 counter_a: Counter = _Counter(names_a)
1256 counter_b: Counter = _Counter(names_b)
1257 count_deltas: dict[str, dict[str, int]] = {}
1258 for skill in sorted(set_a | set_b):
1259 ca = counter_a.get(skill, 0)
1260 cb = counter_b.get(skill, 0)
1261 if ca != cb:
1262 count_deltas[skill] = {"a": ca, "b": cb, "delta": cb - ca}
1264 label_a = f"session_a ({session_a[:8]})" if len(session_a) > 8 else f"session_a ({session_a})"
1265 label_b = f"session_b ({session_b[:8]})" if len(session_b) > 8 else f"session_b ({session_b})"
1266 sequence_diff = list(
1267 difflib.unified_diff(names_a, names_b, fromfile=label_a, tofile=label_b, lineterm="")
1268 )
1270 return SessionDiff(
1271 session_a=session_a,
1272 session_b=session_b,
1273 skills_added=skills_added,
1274 skills_removed=skills_removed,
1275 count_deltas=count_deltas,
1276 sequence_diff=sequence_diff,
1277 )
1280def _cmd_diff(args: argparse.Namespace, logger: Logger) -> int:
1281 """Compare two sessions' ll-invocation behavior."""
1282 db_path = resolve_history_db()
1284 path_a = _resolve_session_log(args.session_a, db_path)
1285 if path_a is None:
1286 logger.error(f"Cannot resolve session: {args.session_a}")
1287 return 1
1289 path_b = _resolve_session_log(args.session_b, db_path)
1290 if path_b is None:
1291 logger.error(f"Cannot resolve session: {args.session_b}")
1292 return 1
1294 events_a = _events_from_jsonl(path_a)
1295 events_b = _events_from_jsonl(path_b)
1297 diff = _compute_session_diff(args.session_a, events_a, args.session_b, events_b)
1299 if args.json:
1300 print_json(diff.to_dict())
1301 return 0
1303 if (
1304 not diff.skills_added
1305 and not diff.skills_removed
1306 and not diff.count_deltas
1307 and not diff.sequence_diff
1308 ):
1309 print("No behavioral differences found.")
1310 return 0
1312 if diff.skills_added:
1313 print(f"Skills added ({len(diff.skills_added)}):")
1314 for s in diff.skills_added:
1315 print(f" + {s}")
1317 if diff.skills_removed:
1318 if diff.skills_added:
1319 print()
1320 print(f"Skills removed ({len(diff.skills_removed)}):")
1321 for s in diff.skills_removed:
1322 print(f" - {s}")
1324 if diff.count_deltas:
1325 print()
1326 print("Invocation count changes:")
1327 for skill, counts in sorted(diff.count_deltas.items()):
1328 delta_str = f"+{counts['delta']}" if counts["delta"] > 0 else str(counts["delta"])
1329 print(f" {skill}: {counts['a']} → {counts['b']} ({delta_str})")
1331 if diff.sequence_diff:
1332 print()
1333 print("Sequence diff:")
1334 for line in diff.sequence_diff:
1335 print(f" {line}")
1337 return 0
1340_ISSUE_ID_RE = re.compile(r"\b[A-Z]+-\d+\b")
1343@dataclass
1344class _EvalInvocation:
1345 """A reconstructed ll-harness invocation extracted from a JSONL record.
1347 Carries the runner kind and raw (un-redacted) input-context text that the
1348 EvalFixture export needs but that ``InvocationEvent`` discards.
1349 """
1351 runner: str # "skill" | "cmd"
1352 target: str # skill name (runner==skill) or full shell command (runner==cmd)
1353 session_id: str
1354 timestamp: str
1355 input_context: str # raw user-message text; "" when none (e.g. Bash invocations)
1358def _extract_eval_invocation(record: dict) -> _EvalInvocation | None:
1359 """Reconstruct a single ll-harness invocation from a JSONL record.
1361 Mirrors ``_extract_tool_name``'s three signals but also captures the runner
1362 kind and raw input-context text needed for EvalFixture export:
1363 (a) queue enqueue ``/ll:<name>`` -> runner=skill, target=name
1364 (b) user ``<command-name>/ll:<name>`` -> runner=skill, target=name
1365 (c) assistant Bash ``ll-<tool>`` -> runner=cmd, target=<full command>
1367 Returns None for records that carry no ll invocation signal.
1368 """
1369 record_type = record.get("type")
1370 sid = record.get("sessionId", "")
1371 ts = record.get("timestamp", "")
1373 # (a) queue-operation enqueue
1374 if record_type == "queue-operation" and record.get("operation") == "enqueue":
1375 content = record.get("content", "")
1376 if isinstance(content, str) and content.startswith("/ll:"):
1377 m = _QUEUE_SKILL_RE.match(content)
1378 if m:
1379 return _EvalInvocation("skill", m.group(1), sid, ts, content)
1381 # (b) user records with <command-name>/ll: pattern
1382 if record_type == "user":
1383 message = record.get("message", {})
1384 if not isinstance(message, dict):
1385 return None
1386 content = message.get("content")
1387 text = ""
1388 if isinstance(content, str):
1389 text = content
1390 elif isinstance(content, list):
1391 for block in content:
1392 if isinstance(block, dict):
1393 text = block.get("text", "")
1394 if text:
1395 break
1396 if text:
1397 m = _COMMAND_NAME_SKILL_RE.search(text)
1398 if m:
1399 name = m.group(1)
1400 if name.endswith("</command-name>"):
1401 name = name[: -len("</command-name>")]
1402 return _EvalInvocation("skill", name, sid, ts, text)
1404 # (c) assistant Bash tool-use invoking ll-<tool>
1405 if record_type == "assistant":
1406 message = record.get("message", {})
1407 content = message.get("content", [])
1408 if isinstance(content, list):
1409 for block in content:
1410 if (
1411 isinstance(block, dict)
1412 and block.get("type") == "tool_use"
1413 and block.get("name") == "Bash"
1414 ):
1415 cmd = block.get("input", {}).get("command", "")
1416 if _LL_BASH_RE.search(cmd):
1417 return _EvalInvocation("cmd", cmd, sid, ts, "")
1419 return None
1422def _record_has_error(record: dict) -> bool:
1423 """True if a JSONL record is a ``tool_result`` flagged ``is_error``.
1425 Used as the session-level ``failed`` outcome signal: session logs expose no
1426 output-quality judgment, only execution evidence (ARCHITECTURE-017).
1427 """
1428 if record.get("type") != "user":
1429 return False
1430 message = record.get("message", {})
1431 if not isinstance(message, dict):
1432 return False
1433 content = message.get("content")
1434 if isinstance(content, list):
1435 for block in content:
1436 if (
1437 isinstance(block, dict)
1438 and block.get("type") == "tool_result"
1439 and block.get("is_error")
1440 ):
1441 return True
1442 return False
1445def _classify_outcome(metadata: dict, *, has_error: bool) -> str:
1446 """Map session metadata + error signal to an EvalFixture execution outcome.
1448 Precedence ``failed`` > ``corrected`` > ``accepted``; ``unknown`` when the DB
1449 returned no metadata. Per ARCHITECTURE-017 the taxonomy is EXECUTION, not
1450 output quality. ``metadata`` is the dict from
1451 ``history_reader.lookup_session_metadata`` (``{}`` when the session is absent).
1452 """
1453 if has_error:
1454 return "failed"
1455 if not metadata:
1456 return "unknown"
1457 if metadata.get("has_corrections"):
1458 return "corrected"
1459 return "accepted"
1462def _redact_input_context(text: str) -> tuple[str | None, bool]:
1463 """Best-effort, non-blocking redaction of user-message text.
1465 Applies ``pii.redact_pii`` (email/phone/SSN) then ``_ABS_PATH_RE`` (absolute
1466 paths). Returns ``(redacted_text_or_None, pii_detected)`` where ``pii_detected``
1467 is True when either pass altered the text. Never raises and never drops a
1468 record for unredactable content (ARCHITECTURE-017).
1469 """
1470 if not text:
1471 return None, False
1472 from little_loops.pii import redact_pii
1474 redacted = redact_pii(text)
1475 redacted = _ABS_PATH_RE.sub("<path>", redacted)
1476 return redacted, redacted != text
1479def _build_eval_fixture(inv: _EvalInvocation, outcome: str) -> dict:
1480 """Map a reconstructed invocation + outcome to an EvalFixture v1 record.
1482 Pure function (no I/O) — the mapping core covered by unit tests. Schema and
1483 field semantics per decision ARCHITECTURE-017 in ``.ll/decisions.yaml``: the
1484 fixture replays into ``ll-harness <runner> <target> [runner_args...]
1485 [--exit-code N] [--semantic TEXT] [--timeout S]`` (ll-harness has no loader).
1486 """
1487 input_context, pii_detected = _redact_input_context(inv.input_context)
1488 issue_match = _ISSUE_ID_RE.search(inv.input_context) if inv.input_context else None
1489 issue_id = issue_match.group(0) if issue_match else None
1490 skill_name = inv.target if inv.runner == "skill" else None
1491 return {
1492 "runner": inv.runner,
1493 "target": inv.target,
1494 "session_id": inv.session_id,
1495 "timestamp": inv.timestamp,
1496 "outcome": outcome,
1497 "runner_args": [],
1498 "exit_code": None,
1499 "semantic": None,
1500 "timeout": 120,
1501 "input_context": input_context,
1502 "issue_id": issue_id,
1503 "skill_name": skill_name,
1504 "pii_detected": pii_detected,
1505 }
1508def _fixture_to_harness_argv(fixture: dict) -> list[str]:
1509 """Serialize an EvalFixture record back into an ``ll-harness`` argv.
1511 ll-harness has no fixture loader (ARCHITECTURE-017); a fixture replays by
1512 serializing its fields into the harness CLI arg surface. Used by the
1513 round-trip test to prove every exported fixture is a valid harness invocation.
1514 """
1515 argv: list[str] = [fixture["runner"], fixture["target"]]
1516 argv.extend(fixture.get("runner_args") or [])
1517 if fixture.get("exit_code") is not None:
1518 argv.extend(["--exit-code", str(fixture["exit_code"])])
1519 if fixture.get("semantic") is not None:
1520 argv.extend(["--semantic", str(fixture["semantic"])])
1521 timeout = fixture.get("timeout")
1522 if timeout is not None and timeout != 120:
1523 argv.extend(["--timeout", str(timeout)])
1524 return argv
1527def _cmd_eval_export(args: argparse.Namespace) -> int:
1528 """Export ll-harness eval fixtures reconstructed from session logs (FEAT-1971).
1530 Walks the current project's JSONL logs, reconstructs each ll invocation, sources
1531 an execution outcome from ``history_reader.lookup_session_metadata``, redacts the
1532 input context, and writes EvalFixture v1 records (YAML default, JSON with
1533 ``--json``). Schema + outcome taxonomy: decision ARCHITECTURE-017 in
1534 ``.ll/decisions.yaml``.
1535 """
1536 from little_loops.history_reader import lookup_session_metadata
1538 cwd_path = Path(args.project) if args.project else Path.cwd()
1539 project_folder = get_project_folder(cwd_path)
1540 if project_folder is None:
1541 print(f"No session project folder found for: {cwd_path}", file=sys.stderr)
1542 return 1
1543 from little_loops.session_store import resolve_history_db
1545 db_path = resolve_history_db(cwd_path / ".ll" / "history.db")
1547 # Single JSONL pass: collect raw invocations + per-session error flags together
1548 # (avoids the double-parse the decision warns against).
1549 invocations: list[_EvalInvocation] = []
1550 session_has_error: dict[str, bool] = {}
1551 jsonl_files = [f for f in project_folder.glob("*.jsonl") if not f.name.startswith("agent-")]
1552 for jsonl_file in jsonl_files:
1553 try:
1554 with open(jsonl_file, encoding="utf-8") as f:
1555 for line in f:
1556 line = line.strip()
1557 if not line:
1558 continue
1559 try:
1560 record = json.loads(line)
1561 except json.JSONDecodeError:
1562 continue
1563 if _record_has_error(record):
1564 sid = record.get("sessionId", "")
1565 if sid:
1566 session_has_error[sid] = True
1567 inv = _extract_eval_invocation(record)
1568 if inv is not None:
1569 invocations.append(inv)
1570 except OSError:
1571 continue
1573 # Stable, deterministic order: by timestamp then session.
1574 invocations.sort(key=lambda e: (e.timestamp, e.session_id))
1576 metadata_cache: dict[str, dict] = {}
1577 fixtures: list[dict] = []
1578 skipped = 0
1579 for inv in invocations:
1580 # --skill: keep only skill-runner invocations of the named target.
1581 if args.skill and not (inv.runner == "skill" and inv.target == args.skill):
1582 continue
1584 if inv.session_id not in metadata_cache:
1585 metadata_cache[inv.session_id] = lookup_session_metadata(inv.session_id, db=db_path)
1586 outcome = _classify_outcome(
1587 metadata_cache[inv.session_id],
1588 has_error=session_has_error.get(inv.session_id, False),
1589 )
1590 # No extractable execution outcome -> skip with a logged count.
1591 if outcome == "unknown":
1592 skipped += 1
1593 continue
1595 fixture = _build_eval_fixture(inv, outcome)
1597 # --issue: match the extracted issue_id or a literal occurrence in target.
1598 if args.issue and args.issue != fixture["issue_id"] and args.issue not in inv.target:
1599 continue
1601 fixtures.append(fixture)
1602 if args.limit and len(fixtures) >= args.limit:
1603 break
1605 if args.json:
1606 output = json.dumps(fixtures, indent=2)
1607 else:
1608 import yaml
1610 output = yaml.safe_dump(fixtures, sort_keys=False, default_flow_style=False)
1612 if args.out:
1613 out_path = Path(args.out)
1614 try:
1615 out_path.parent.mkdir(parents=True, exist_ok=True)
1616 out_path.write_text(output, encoding="utf-8")
1617 except OSError as exc:
1618 print(f"Failed to write {out_path}: {exc}", file=sys.stderr)
1619 return 1
1620 print(f"Wrote {len(fixtures)} fixture(s) to {out_path}", file=sys.stderr)
1621 else:
1622 print(output, end="" if output.endswith("\n") else "\n")
1624 if skipped:
1625 print(
1626 f"Skipped {skipped} invocation(s) with no extractable outcome",
1627 file=sys.stderr,
1628 )
1630 return 0
1633def _build_parser() -> argparse.ArgumentParser:
1634 """Build the argument parser for ll-logs."""
1635 parser = argparse.ArgumentParser(
1636 prog="ll-logs",
1637 description="Discover and extract ll-relevant JSONL entries from Claude Code logs",
1638 formatter_class=argparse.RawDescriptionHelpFormatter,
1639 epilog="""
1640Examples:
1641 %(prog)s discover # List all projects with ll activity
1642 %(prog)s tail --loop <name> # Stream live events from an active loop session
1643 %(prog)s extract --all # Extract all projects to logs/
1644 %(prog)s extract --project /path # Extract one project to logs/<slug>/
1645 %(prog)s extract --all --cmd ll-history # Filter to ll-history invocations
1646""",
1647 )
1649 subparsers = parser.add_subparsers(dest="command", help="Available commands")
1650 discover_parser = subparsers.add_parser(
1651 "discover",
1652 help="List all Claude projects with ll activity (one path per line, sorted)",
1653 )
1654 add_json_arg(discover_parser)
1656 tail_parser = subparsers.add_parser(
1657 "tail",
1658 help="Stream live events from an active loop session",
1659 )
1660 tail_parser.add_argument("--loop", required=True, metavar="NAME", help="Loop name to tail")
1662 extract_parser = subparsers.add_parser(
1663 "extract",
1664 help="Extract ll-relevant JSONL records to logs/<slug>/<session-id>.jsonl",
1665 )
1666 target_group = extract_parser.add_mutually_exclusive_group(required=True)
1667 target_group.add_argument(
1668 "--project",
1669 type=Path,
1670 metavar="DIR",
1671 help="Working directory of the target project",
1672 )
1673 target_group.add_argument(
1674 "--all",
1675 action="store_true",
1676 help="Extract all projects with ll activity",
1677 )
1678 extract_parser.add_argument(
1679 "--cmd",
1680 metavar="TOOL",
1681 help="Filter to records containing this ll- tool name (e.g. ll-history)",
1682 )
1684 sequences_parser = subparsers.add_parser(
1685 "sequences",
1686 help="Extract tool-chain n-grams of ll invocations from JSONL logs",
1687 )
1688 sequences_target = sequences_parser.add_mutually_exclusive_group(required=True)
1689 sequences_target.add_argument(
1690 "--project",
1691 type=Path,
1692 metavar="DIR",
1693 help="Working directory of the target project",
1694 )
1695 sequences_target.add_argument(
1696 "--all",
1697 action="store_true",
1698 help="Analyze all projects with ll activity",
1699 )
1700 sequences_parser.add_argument(
1701 "--min-len",
1702 type=int,
1703 default=2,
1704 metavar="N",
1705 help="Minimum n-gram length (default: 2)",
1706 )
1707 sequences_parser.add_argument(
1708 "--min-count",
1709 type=int,
1710 default=1,
1711 metavar="M",
1712 help="Minimum occurrence count to include (default: 1)",
1713 )
1714 sequences_parser.add_argument(
1715 "--top",
1716 type=int,
1717 default=None,
1718 metavar="N",
1719 help="Limit output to top N chains by frequency",
1720 )
1721 sequences_parser.add_argument(
1722 "--window-days",
1723 type=int,
1724 default=None,
1725 metavar="D",
1726 help="Only consider records within D days of latest record",
1727 )
1728 add_json_arg(sequences_parser)
1730 stats_parser = subparsers.add_parser(
1731 "stats",
1732 help="Aggregate skill invocation frequency and correction rate from history.db",
1733 )
1734 stats_target = stats_parser.add_mutually_exclusive_group(required=True)
1735 stats_target.add_argument(
1736 "--project",
1737 type=Path,
1738 metavar="DIR",
1739 help="Working directory of the target project",
1740 )
1741 stats_target.add_argument(
1742 "--all",
1743 action="store_true",
1744 help="Aggregate across all projects with ll activity",
1745 )
1746 stats_parser.add_argument(
1747 "--window-days",
1748 type=int,
1749 default=None,
1750 metavar="D",
1751 help="Only consider records within D days of latest record",
1752 )
1753 stats_parser.add_argument(
1754 "--sort",
1755 choices=["freq", "corrections"],
1756 default="freq",
1757 help="Sort output by invocation frequency or correction count (default: freq)",
1758 )
1759 add_json_arg(stats_parser)
1761 scan_failures_parser = subparsers.add_parser(
1762 "scan-failures",
1763 help="Mine failed ll-* calls from interactive session logs and propose bug issues",
1764 )
1765 scan_failures_target = scan_failures_parser.add_mutually_exclusive_group(required=True)
1766 scan_failures_target.add_argument(
1767 "--project",
1768 type=Path,
1769 metavar="DIR",
1770 help="Working directory of the target project",
1771 )
1772 scan_failures_target.add_argument(
1773 "--all",
1774 action="store_true",
1775 help="Scan all projects with ll activity",
1776 )
1777 scan_failures_parser.add_argument(
1778 "--window-days",
1779 type=int,
1780 default=None,
1781 metavar="D",
1782 help="Only consider records within D days of latest record",
1783 )
1784 scan_failures_parser.add_argument(
1785 "--capture",
1786 action="store_true",
1787 help="Create bug issue files for each failure cluster (one per tool+error signature)",
1788 )
1789 add_json_arg(scan_failures_parser)
1791 dead_skills_parser = subparsers.add_parser(
1792 "dead-skills",
1793 help="List catalog skills/commands with zero or low invocations across the corpus",
1794 )
1795 dead_skills_target = dead_skills_parser.add_mutually_exclusive_group(required=True)
1796 dead_skills_target.add_argument(
1797 "--project",
1798 type=Path,
1799 metavar="DIR",
1800 help="Working directory of the target project (also used as catalog root)",
1801 )
1802 dead_skills_target.add_argument(
1803 "--all",
1804 action="store_true",
1805 help="Aggregate across all projects; catalog loaded from current directory",
1806 )
1807 dead_skills_parser.add_argument(
1808 "--window-days",
1809 type=int,
1810 default=None,
1811 metavar="D",
1812 help="Only consider records within D days of latest record",
1813 )
1814 dead_skills_parser.add_argument(
1815 "--threshold",
1816 type=int,
1817 default=3,
1818 metavar="N",
1819 help="Skills with invocations <= N are 'rarely' invoked (default: 3)",
1820 )
1821 add_json_arg(dead_skills_parser)
1823 diff_parser = subparsers.add_parser(
1824 "diff",
1825 help="Compare two sessions' ll-invocation behavior (skills, sequences, counts)",
1826 )
1827 diff_parser.add_argument(
1828 "session_a", metavar="SESSION_A", help="First session ID or JSONL file path"
1829 )
1830 diff_parser.add_argument(
1831 "session_b", metavar="SESSION_B", help="Second session ID or JSONL file path"
1832 )
1833 add_json_arg(diff_parser)
1835 eval_export_parser = subparsers.add_parser(
1836 "eval-export",
1837 help="Export eval fixtures from ll-harness session logs",
1838 )
1839 eval_export_parser.add_argument(
1840 "--project",
1841 type=Path,
1842 metavar="DIR",
1843 help="Project working directory (default: current directory)",
1844 )
1845 eval_export_parser.add_argument(
1846 "--skill",
1847 metavar="NAME",
1848 help="Filter by skill name",
1849 )
1850 eval_export_parser.add_argument(
1851 "--issue",
1852 metavar="ID",
1853 help="Filter by issue ID in session context",
1854 )
1855 eval_export_parser.add_argument(
1856 "--limit",
1857 type=int,
1858 default=0,
1859 metavar="N",
1860 help="Cap output records (0 = unlimited)",
1861 )
1862 eval_export_parser.add_argument(
1863 "--out",
1864 metavar="PATH",
1865 help="Write output to file (default: stdout)",
1866 )
1867 eval_export_parser.add_argument(
1868 "--json",
1869 action="store_true",
1870 help="JSON output instead of YAML (default: YAML)",
1871 )
1873 return parser
1876def _parse_args() -> argparse.Namespace:
1877 """Parse command-line arguments. Exposed for testing."""
1878 return _build_parser().parse_args()
1881def main_logs() -> int:
1882 """Entry point for ll-logs command.
1884 Returns:
1885 0 on success, 1 when no subcommand given or on error.
1886 """
1887 with cli_event_context(DEFAULT_DB_PATH, "ll-logs", sys.argv[1:]):
1888 configure_output()
1889 logger = Logger(use_color=use_color_enabled())
1891 parser = _build_parser()
1892 args = parser.parse_args()
1894 if not args.command:
1895 parser.print_help()
1896 return 1
1898 if args.command == "discover":
1899 projects = discover_all_projects(logger)
1900 if args.json:
1901 print_json({"paths": [str(p) for p in projects]})
1902 else:
1903 for path in projects:
1904 print(path)
1905 return 0
1907 if args.command == "tail":
1908 config = BRConfig(Path.cwd())
1909 loops_dir = Path(config.loops.loops_dir)
1910 return _cmd_tail(args, loops_dir)
1912 if args.command == "extract":
1913 return _cmd_extract(args, logger)
1915 if args.command == "sequences":
1916 return _cmd_sequences(args, logger)
1918 if args.command == "stats":
1919 return _cmd_stats(args, logger)
1921 if args.command == "scan-failures":
1922 return _cmd_scan_failures(args, logger)
1924 if args.command == "dead-skills":
1925 return _cmd_dead_skills(args, logger)
1927 if args.command == "diff":
1928 return _cmd_diff(args, logger)
1930 if args.command == "eval-export":
1931 return _cmd_eval_export(args)
1933 return 1