Coverage for little_loops / cli / messages.py: 8%
130 statements
« prev ^ index » next coverage.py v7.12.0, created at 2026-06-04 12:21 -0500
1"""ll-messages: Extract user messages from Claude Code session logs."""
3from __future__ import annotations
5import argparse
6import sys
7from pathlib import Path
9from little_loops.logger import Logger
10from little_loops.session_store import DEFAULT_DB_PATH, cli_event_context
13def main_messages() -> int:
14 """Entry point for ll-messages command.
16 Extract user messages from Claude Code session logs.
18 Returns:
19 Exit code (0 = success)
20 """
21 with cli_event_context(DEFAULT_DB_PATH, "ll-messages", sys.argv[1:]):
22 import json
23 from datetime import datetime
25 from little_loops.user_messages import (
26 CommandRecord,
27 UserMessage,
28 build_examples,
29 extract_commands,
30 extract_user_messages,
31 get_project_folder,
32 )
34 parser = argparse.ArgumentParser(
35 description="Extract user messages from Claude Code logs",
36 formatter_class=argparse.RawDescriptionHelpFormatter,
37 epilog="""
38Examples:
39 %(prog)s # Last 100 messages to file
40 %(prog)s -n 50 # Last 50 messages
41 %(prog)s --since 2026-01-01 # Messages since date
42 %(prog)s -o output.jsonl # Custom output path
43 %(prog)s --stdout # Print to terminal
44 %(prog)s --include-response-context # Include response metadata
45 %(prog)s --skip-cli # Exclude CLI commands from output
46 %(prog)s --commands-only # Extract only CLI commands
47 %(prog)s --skill capture-issue # Filter to sessions where skill was invoked
48 %(prog)s --skill capture-issue --examples-format # Output (input, output) training pairs
49 %(prog)s --skill refine-issue --examples-format --context-window 5 --stdout
50 %(prog)s --sft-format chatml --stdout
51 %(prog)s --sft-format sharegpt --context-window 3 --since 2026-05-01 --stdout
52 %(prog)s --sft-format alpaca --output data/sft/raw.jsonl
54Pipeline with ll-workflows (use the conventional path so ll-workflows finds it automatically):
55 %(prog)s --output .ll/workflow-analysis/step1-patterns.jsonl
56 ll-workflows analyze --patterns .ll/workflow-analysis/step1-patterns.yaml
57""",
58 )
59 parser.add_argument(
60 "-n",
61 "--limit",
62 type=int,
63 default=100,
64 help="Maximum number of messages to extract (default: 100)",
65 )
66 parser.add_argument(
67 "--since",
68 "-S",
69 type=str,
70 help="Only include messages after this date (YYYY-MM-DD or ISO format)",
71 )
72 parser.add_argument(
73 "-o",
74 "--output",
75 type=Path,
76 help="Output file path (default: .ll/user-messages-{timestamp}.jsonl)",
77 )
78 parser.add_argument(
79 "--cwd",
80 type=Path,
81 help="Working directory to use (default: current directory)",
82 )
83 parser.add_argument(
84 "--exclude-agents",
85 action="store_true",
86 help="Exclude agent session files (agent-*.jsonl)",
87 )
88 parser.add_argument(
89 "--stdout",
90 action="store_true",
91 help="Print messages to stdout instead of writing to file",
92 )
93 parser.add_argument(
94 "-v",
95 "--verbose",
96 action="store_true",
97 help="Print verbose progress information",
98 )
99 parser.add_argument(
100 "--include-response-context",
101 action="store_true",
102 help="Include metadata from assistant responses (tools used, files modified)",
103 )
104 parser.add_argument(
105 "--skip-cli",
106 action="store_true",
107 help="Exclude CLI commands from output (included by default)",
108 )
109 parser.add_argument(
110 "--commands-only",
111 action="store_true",
112 help="Extract only CLI commands, no user messages",
113 )
114 parser.add_argument(
115 "--tools",
116 type=str,
117 default="Bash",
118 help="Comma-separated list of tools to extract commands from (default: Bash)",
119 )
120 parser.add_argument(
121 "--skill",
122 type=str,
123 help="Filter to sessions where this skill was invoked (e.g. capture-issue)",
124 )
125 format_group = parser.add_mutually_exclusive_group()
126 format_group.add_argument(
127 "--examples-format",
128 action="store_true",
129 help="Output (input, output) training pairs for prompt optimization instead of raw messages",
130 )
131 format_group.add_argument(
132 "--sft-format",
133 choices=["chatml", "alpaca", "sharegpt"],
134 help="Output conversation turns in SFT training format as JSON-lines (chatml, alpaca, sharegpt)",
135 )
136 parser.add_argument(
137 "--context-window",
138 type=int,
139 default=3,
140 help="Number of preceding messages to include as context in --examples-format or --sft-format (default: 3)",
141 )
143 args = parser.parse_args()
145 logger = Logger(verbose=args.verbose)
147 # Parse since date if provided
148 since = None
149 if args.since:
150 try:
151 # Try ISO format first
152 since = datetime.fromisoformat(args.since.replace("Z", "+00:00"))
153 except ValueError:
154 try:
155 # Try YYYY-MM-DD format
156 since = datetime.strptime(args.since, "%Y-%m-%d")
157 except ValueError:
158 logger.error(f"Invalid date format: {args.since}")
159 logger.error("Use YYYY-MM-DD or ISO format")
160 return 1
162 # Get project folder
163 cwd = args.cwd or Path.cwd()
164 project_folder = get_project_folder(cwd)
166 if project_folder is None:
167 logger.error(f"No Claude project folder found for: {cwd}")
168 logger.error(f"Expected: ~/.claude/projects/{str(cwd).replace('/', '-')}")
169 return 1
171 logger.info(f"Project folder: {project_folder}")
172 logger.info(f"Limit: {args.limit}")
173 if since:
174 logger.info(f"Since: {since}")
176 # Parse tools list
177 tools_list = [t.strip() for t in args.tools.split(",")]
179 # Extract data based on flags
180 messages: list[UserMessage] = []
181 commands: list[CommandRecord] = []
183 if not args.commands_only:
184 messages = extract_user_messages(
185 project_folder=project_folder,
186 limit=None, # Apply limit after merging
187 since=since,
188 include_agent_sessions=not args.exclude_agents,
189 include_response_context=args.include_response_context or args.examples_format,
190 )
192 if not args.skip_cli or args.commands_only:
193 commands = extract_commands(
194 project_folder=project_folder,
195 limit=None, # Apply limit after merging
196 since=since,
197 include_agent_sessions=not args.exclude_agents,
198 tools=tools_list,
199 )
201 # Apply skill filter (session-level): keep only sessions where skill was invoked
202 if args.skill:
203 import re
205 skill_pattern = re.compile(rf"<command-name>/ll:{re.escape(args.skill)}</command-name>")
206 matching_sessions = {
207 msg.session_id for msg in messages if skill_pattern.search(msg.content)
208 }
209 messages = [msg for msg in messages if msg.session_id in matching_sessions]
210 commands = [c for c in commands if c.session_id in matching_sessions]
212 if not messages and not commands:
213 logger.warning("No user messages or commands found")
214 return 0
216 # When --examples-format is set, reshape to ExampleRecord output and return early
217 if args.examples_format:
218 if not args.skill:
219 logger.error("--examples-format requires --skill to be specified")
220 return 1
221 examples = build_examples(messages, args.skill, args.context_window)
222 examples.sort(key=lambda x: x.timestamp, reverse=True)
223 if args.limit is not None:
224 examples = examples[: args.limit]
225 logger.info(f"Found {len(examples)} examples")
226 if args.stdout:
227 for item in examples:
228 print(json.dumps(item.to_dict()))
229 else:
230 output_path = _save_combined(examples, args.output)
231 logger.success(f"Saved {len(examples)} examples to: {output_path}")
232 return 0
234 # When --sft-format is set, emit conversation turns in SFT training format
235 if args.sft_format:
236 from little_loops.sft_formatter import to_alpaca, to_chatml, to_sharegpt
237 from little_loops.user_messages import extract_conversation_turns
239 formatter = {"chatml": to_chatml, "alpaca": to_alpaca, "sharegpt": to_sharegpt}[
240 args.sft_format
241 ]
242 windows = extract_conversation_turns(
243 project_folder=project_folder,
244 since=since,
245 context_window=args.context_window,
246 include_agent_sessions=not args.exclude_agents,
247 )
248 if args.limit is not None:
249 windows = windows[: args.limit]
250 logger.info(f"Found {len(windows)} conversation windows")
251 if args.stdout:
252 for window in windows:
253 print(json.dumps(formatter(window)))
254 else:
255 output_path = _save_combined([_SFTItem(formatter(w)) for w in windows], args.output)
256 logger.success(f"Saved {len(windows)} windows to: {output_path}")
257 return 0
259 # Merge and sort by timestamp
260 combined: list[UserMessage | CommandRecord] = []
261 combined.extend(messages)
262 combined.extend(commands)
263 combined.sort(key=lambda x: x.timestamp, reverse=True)
265 # Apply limit
266 if args.limit is not None:
267 combined = combined[: args.limit]
269 msg_count = len([x for x in combined if isinstance(x, UserMessage)])
270 cmd_count = len([x for x in combined if isinstance(x, CommandRecord)])
271 logger.info(f"Found {msg_count} messages, {cmd_count} commands")
273 # Output
274 if args.stdout:
275 for record in combined:
276 print(json.dumps(record.to_dict()))
277 else:
278 output_path = _save_combined(combined, args.output)
279 logger.success(f"Saved {len(combined)} records to: {output_path}")
281 return 0
284def _save_combined(
285 items: list,
286 output_path: Path | None = None,
287) -> Path:
288 """Save combined messages and commands to JSONL file."""
289 import json
290 from datetime import datetime
292 if output_path is None:
293 timestamp = datetime.now().strftime("%Y%m%d-%H%M%S")
294 output_dir = Path.cwd() / ".claude"
295 output_dir.mkdir(parents=True, exist_ok=True)
296 output_path = output_dir / f"user-messages-{timestamp}.jsonl"
298 output_path = Path(output_path)
299 output_path.parent.mkdir(parents=True, exist_ok=True)
301 with open(output_path, "w", encoding="utf-8") as f:
302 for item in items:
303 f.write(json.dumps(item.to_dict()) + "\n")
305 return output_path
308class _SFTItem:
309 """Thin wrapper to make an SFT dict compatible with _save_combined()."""
311 def __init__(self, data: dict) -> None:
312 self._data = data
314 def to_dict(self) -> dict:
315 return self._data