Coverage for agentos/marketplace/importer.py: 0%
244 statements
« prev ^ index » next coverage.py v7.14.3, created at 2026-07-02 09:59 +0800
1"""
2Marketplace Importer — Import skills from external sources (OpenClaw, HuggingFace, GitHub).
4OpenClaw Community: https://github.com/openclaw/skills
5 - skill.yaml → SkillManifest (openclaw format)
6 - Auto-detect format, convert, register
8Usage:
9 from agentos.marketplace.importer import OpenClawImporter
10 importer = OpenClawImporter(registry)
11 skill = await importer.import_skill("pdf-tools")
12"""
14from __future__ import annotations
16import asyncio
17import json
18import tempfile
19import zipfile
20from dataclasses import dataclass, field
21from pathlib import Path
22from typing import Optional, Callable, Any
24from agentos.marketplace.manifest import SkillManifest, SkillFormat, ToolDef
25from agentos.marketplace.registry import SkillRegistry, SearchResult, InstallResult
28# ── OpenClaw Importer ──
30OPENCLAW_SKILLS_REPO = "https://github.com/nicepkg/openclaw-skill-store"
31OPENCLAW_RAW_BASE = "https://raw.githubusercontent.com/nicepkg/openclaw-skill-store/main"
32OPENCLAW_API = "https://api.github.com/repos/nicepkg/openclaw-skill-store"
35@dataclass
36class RemoteSkill:
37 """Skill metadata discovered from a remote source."""
38 name: str
39 path: str # Relative path in the repo
40 description: str = ""
41 author: str = ""
42 version: str = "0.1.0"
43 tags: list[str] = field(default_factory=list)
44 download_url: str = ""
45 raw_url: str = ""
46 source: str = "openclaw"
49class OpenClawImporter:
50 """Import skills from the OpenClaw community skill store.
52 Flow:
53 1. list_available() — fetch skill catalog from GitHub API
54 2. import_skill(name) — download skill.yaml → parse → register
55 3. import_all() — batch import all available skills
56 """
58 def __init__(self, registry: SkillRegistry, cache_dir: str = ""):
59 self._registry = registry
60 self._cache_dir = Path(cache_dir) if cache_dir else Path.home() / ".agentos" / "marketplace" / "openclaw"
61 self._cache_dir.mkdir(parents=True, exist_ok=True)
63 self._catalog: list[RemoteSkill] = []
64 self._fetch_fn: Optional[Callable[[str], str]] = None # Injection point for testing
66 # ── Catalog ──
68 async def list_available(self, refresh: bool = False) -> list[RemoteSkill]:
69 """List all available OpenClaw community skills.
71 Returns cached catalog unless refresh=True.
72 """
73 if self._catalog and not refresh:
74 return self._catalog
76 skills = []
78 # Try fetching directory listing from the raw GitHub API
79 try:
80 import aiohttp
81 except ImportError:
82 return await self._list_fallback()
84 try:
85 async with aiohttp.ClientSession() as session:
86 # Fetch the top-level directory listing
87 url = f"{OPENCLAW_API}/contents/skills"
88 async with session.get(url, headers={"Accept": "application/vnd.github.v3+json"}) as resp:
89 if resp.status != 200:
90 return await self._list_fallback()
92 entries = await resp.json()
93 for entry in entries:
94 if entry.get("type") != "dir":
95 continue
97 skill_name = entry["name"]
98 skill_path = f"skills/{skill_name}"
99 raw_url = f"{OPENCLAW_RAW_BASE}/{skill_path}/skill.yaml"
100 download_url = entry.get("url", "")
102 # Try to read skill.yaml for metadata
103 meta = await self._fetch_skill_meta(session, skill_path)
104 skills.append(RemoteSkill(
105 name=meta.get("name", skill_name),
106 path=skill_path,
107 description=meta.get("description", ""),
108 author=meta.get("author", ""),
109 version=meta.get("version", "0.1.0"),
110 tags=meta.get("tags", []),
111 raw_url=raw_url,
112 download_url=download_url,
113 ))
115 except Exception:
116 return await self._list_fallback()
118 self._catalog = skills
119 return skills
121 async def _list_fallback(self) -> list[RemoteSkill]:
122 """Fallback: return a curated list of known OpenClaw skills (60+)."""
123 known_skills = [
124 # ── Meta & Creator (3) ──
125 RemoteSkill(name="skill-creator", path="skills/skill-creator",
126 description="Guide for creating effective skills", tags=["meta", "creator"]),
127 RemoteSkill(name="mcp-builder", path="skills/mcp-builder",
128 description="Guide for creating MCP servers and tools", tags=["mcp", "infra"]),
129 RemoteSkill(name="coding-agent", path="skills/coding-agent",
130 description="Autonomous coding agent for complex software tasks", tags=["dev", "agent"]),
132 # ── Office & Documents (8) ──
133 RemoteSkill(name="docx", path="skills/docx",
134 description="Create and edit .docx documents", tags=["document", "office"]),
135 RemoteSkill(name="pdf", path="skills/pdf",
136 description="PDF manipulation toolkit: merge, split, extract, annotate", tags=["document", "pdf"]),
137 RemoteSkill(name="pptx", path="skills/pptx",
138 description="Create and edit .pptx presentations", tags=["presentation", "office"]),
139 RemoteSkill(name="xlsx", path="skills/xlsx",
140 description="Create and edit .xlsx spreadsheets with formulas and charts", tags=["spreadsheet", "office"]),
141 RemoteSkill(name="nano-pdf", path="skills/nano-pdf",
142 description="Lightweight PDF reading and text extraction", tags=["document", "pdf"]),
143 RemoteSkill(name="notion", path="skills/notion",
144 description="Notion integration: pages, databases, blocks CRUD", tags=["productivity", "notion"]),
145 RemoteSkill(name="obsidian", path="skills/obsidian",
146 description="Obsidian vault integration: read/write notes, backlinks", tags=["knowledge", "obsidian"]),
147 RemoteSkill(name="bear-notes", path="skills/bear-notes",
148 description="Bear notes app integration for Apple ecosystem", tags=["notes", "apple"]),
150 # ── Design & Creative (6) ──
151 RemoteSkill(name="brand-guidelines", path="skills/brand-guidelines",
152 description="Applies brand colors and typography to any artifact", tags=["design", "brand"]),
153 RemoteSkill(name="canvas-design", path="skills/canvas-design",
154 description="Create beautiful visual art in .png and .pdf", tags=["art", "design"]),
155 RemoteSkill(name="algorithmic-art", path="skills/algorithmic-art",
156 description="Creating algorithmic art using p5.js", tags=["art", "creative"]),
157 RemoteSkill(name="theme-factory", path="skills/theme-factory",
158 description="Apply visual themes: color schemes, typography, spacing", tags=["design", "theme"]),
159 RemoteSkill(name="slack-gif-creator", path="skills/slack-gif-creator",
160 description="Create animated GIFs optimized for Slack", tags=["media", "slack"]),
161 RemoteSkill(name="openai-image-gen", path="skills/openai-image-gen",
162 description="Generate images using DALL-E / OpenAI image API", tags=["ai", "image"]),
164 # ── Web & Frontend (5) ──
165 RemoteSkill(name="frontend-design", path="skills/frontend-design",
166 description="Create distinctive production-grade frontend interfaces", tags=["web", "frontend"]),
167 RemoteSkill(name="web-artifacts-builder", path="skills/web-artifacts-builder",
168 description="Build complex multi-file HTML artifacts with CSS/JS", tags=["web", "html"]),
169 RemoteSkill(name="web-search", path="skills/web-search",
170 description="Web search with multiple engines and result parsing", tags=["web", "search"]),
171 RemoteSkill(name="blogwatcher", path="skills/blogwatcher",
172 description="Monitor blogs and RSS feeds for updates", tags=["web", "monitoring"]),
173 RemoteSkill(name="wikipedia", path="skills/wikipedia",
174 description="Search and extract content from Wikipedia", tags=["web", "knowledge"]),
176 # ── Developer Tools (10) ──
177 RemoteSkill(name="github", path="skills/github",
178 description="GitHub API: repos, issues, PRs, actions, gists", tags=["dev", "github"]),
179 RemoteSkill(name="gh-issues", path="skills/gh-issues",
180 description="Deep GitHub issues management and triage", tags=["dev", "github"]),
181 RemoteSkill(name="git", path="skills/git",
182 description="Git version control: commit, branch, merge, rebase", tags=["dev", "vcs"]),
183 RemoteSkill(name="docker", path="skills/docker",
184 description="Docker container management: build, run, compose", tags=["dev", "infra"]),
185 RemoteSkill(name="code-review", path="skills/code-review",
186 description="Automated code review with best-practice suggestions", tags=["dev", "quality"]),
187 RemoteSkill(name="database", path="skills/database",
188 description="SQL/NoSQL database query and schema management", tags=["dev", "data"]),
189 RemoteSkill(name="api-tester", path="skills/api-tester",
190 description="REST/GraphQL API testing and documentation", tags=["dev", "api"]),
191 RemoteSkill(name="tmux", path="skills/tmux",
192 description="Tmux session management and automation", tags=["dev", "terminal"]),
193 RemoteSkill(name="node-connect", path="skills/node-connect",
194 description="Node.js runtime integration and package management", tags=["dev", "node"]),
195 RemoteSkill(name="model-usage", path="skills/model-usage",
196 description="Track and optimize AI model usage and costs", tags=["dev", "ai"]),
198 # ── Communication & Messaging (6) ──
199 RemoteSkill(name="internal-comms", path="skills/internal-comms",
200 description="Internal communications: announcements, memos, updates", tags=["writing", "business"]),
201 RemoteSkill(name="slack", path="skills/slack",
202 description="Slack integration: messages, channels, reactions", tags=["communication", "slack"]),
203 RemoteSkill(name="discord", path="skills/discord",
204 description="Discord bot integration for servers and DMs", tags=["communication", "discord"]),
205 RemoteSkill(name="email", path="skills/email",
206 description="Email composition, sending, and inbox management", tags=["communication", "email"]),
207 RemoteSkill(name="telegram", path="skills/telegram",
208 description="Telegram bot API: messages, channels, inline queries", tags=["communication", "telegram"]),
209 RemoteSkill(name="imsg", path="skills/imsg",
210 description="iMessage integration for Apple ecosystem", tags=["communication", "apple"]),
212 # ── Productivity (8) ──
213 RemoteSkill(name="calendar", path="skills/calendar",
214 description="Calendar management: events, reminders, scheduling", tags=["productivity", "time"]),
215 RemoteSkill(name="task-manager", path="skills/task-manager",
216 description="Task and to-do list management with priorities", tags=["productivity", "tasks"]),
217 RemoteSkill(name="notes", path="skills/notes",
218 description="Quick note-taking with search and organization", tags=["productivity", "notes"]),
219 RemoteSkill(name="apple-notes", path="skills/apple-notes",
220 description="Apple Notes app integration", tags=["productivity", "apple"]),
221 RemoteSkill(name="apple-reminders", path="skills/apple-reminders",
222 description="Apple Reminders app integration", tags=["productivity", "apple"]),
223 RemoteSkill(name="things-mac", path="skills/things-mac",
224 description="Things 3 task manager integration for macOS", tags=["productivity", "mac"]),
225 RemoteSkill(name="trello", path="skills/trello",
226 description="Trello board management: cards, lists, boards", tags=["productivity", "pm"]),
227 RemoteSkill(name="summarize", path="skills/summarize",
228 description="Intelligent text summarization with configurable depth", tags=["productivity", "text"]),
230 # ── Data & Analysis (5) ──
231 RemoteSkill(name="data-analysis", path="skills/data-analysis",
232 description="Statistical analysis, visualization, and reporting", tags=["data", "analytics"]),
233 RemoteSkill(name="spreadsheet", path="skills/spreadsheet",
234 description="Advanced spreadsheet operations and formulas", tags=["data", "office"]),
235 RemoteSkill(name="csv-toolkit", path="skills/csv-toolkit",
236 description="CSV parsing, transformation, and export toolkit", tags=["data", "csv"]),
237 RemoteSkill(name="json-toolkit", path="skills/json-toolkit",
238 description="JSON manipulation, validation, and transformation", tags=["data", "json"]),
239 RemoteSkill(name="markdown-toolkit", path="skills/markdown-toolkit",
240 description="Markdown rendering, conversion, and templating", tags=["writing", "markdown"]),
242 # ── Media & Multimedia (5) ──
243 RemoteSkill(name="video-frames", path="skills/video-frames",
244 description="Extract and analyze frames from video files", tags=["media", "video"]),
245 RemoteSkill(name="audio-transcribe", path="skills/audio-transcribe",
246 description="Speech-to-text transcription with Whisper API", tags=["media", "audio"]),
247 RemoteSkill(name="openai-whisper", path="skills/openai-whisper",
248 description="OpenAI Whisper speech recognition integration", tags=["media", "audio"]),
249 RemoteSkill(name="openai-whisper-api", path="skills/openai-whisper-api",
250 description="OpenAI Whisper API with batch processing", tags=["media", "audio"]),
251 RemoteSkill(name="sherpa-onnx-tts", path="skills/sherpa-onnx-tts",
252 description="Text-to-speech with Sherpa-ONNX engine", tags=["media", "tts"]),
254 # ── System & Automation (5) ──
255 RemoteSkill(name="automation", path="skills/automation",
256 description="Workflow automation: triggers, actions, scheduling", tags=["automation", "workflow"]),
257 RemoteSkill(name="file-organizer", path="skills/file-organizer",
258 description="Smart file organization: sort, rename, deduplicate", tags=["system", "files"]),
259 RemoteSkill(name="backup", path="skills/backup",
260 description="Automated backup and restore for files and configs", tags=["system", "backup"]),
261 RemoteSkill(name="weather", path="skills/weather",
262 description="Weather forecasts, alerts, and historical data", tags=["utility", "weather"]),
263 RemoteSkill(name="healthcheck", path="skills/healthcheck",
264 description="System health monitoring and diagnostics", tags=["system", "monitoring"]),
266 # ── Security & Privacy (3) ──
267 RemoteSkill(name="1password", path="skills/1password",
268 description="1Password vault integration for secrets management", tags=["security", "password"]),
269 RemoteSkill(name="encryption", path="skills/encryption",
270 description="File encryption/decryption with multiple algorithms", tags=["security", "crypto"]),
271 RemoteSkill(name="session-logs", path="skills/session-logs",
272 description="Audit and analyze agent session logs", tags=["security", "audit"]),
273 ]
274 self._catalog = known_skills
275 return known_skills
277 async def _fetch_skill_meta(self, session, skill_path: str) -> dict:
278 """Fetch skill.yaml metadata for a single skill."""
279 url = f"{OPENCLAW_API}/contents/{skill_path}/skill.yaml"
280 try:
281 async with session.get(url, headers={"Accept": "application/vnd.github.v3.raw"}) as resp:
282 if resp.status == 200:
283 text = await resp.text()
284 import yaml
285 return yaml.safe_load(text) or {}
286 except Exception:
287 pass
288 return {}
290 # ── Import ──
292 async def import_skill(self, name: str, force: bool = False) -> Optional[InstallResult]:
293 """Import a single skill from OpenClaw by name.
295 Pipeline:
296 1. Find in catalog
297 2. Fetch skill.yaml from raw GitHub
298 3. Parse as OpenClaw format → SkillManifest
299 4. Register in SkillRegistry
300 """
301 # Ensure catalog is loaded
302 if not self._catalog:
303 await self.list_available()
305 # Find skill
306 skill_ref = None
307 for s in self._catalog:
308 if s.name == name:
309 skill_ref = s
310 break
312 if not skill_ref:
313 return None
315 # Fetch skill.yaml
316 yaml_url = f"{OPENCLAW_RAW_BASE}/{skill_ref.path}/skill.yaml"
317 yaml_text = await self._fetch_url(yaml_url)
319 if not yaml_text:
320 return None
322 # Parse as OpenClaw format
323 import yaml
324 try:
325 raw = yaml.safe_load(yaml_text)
326 except yaml.YAMLError:
327 return None
329 if not raw:
330 return None
332 # Convert to SkillManifest
333 raw["format"] = "openclaw"
334 manifest = SkillManifest.from_dict(
335 raw,
336 source=f"openclaw:{name}",
337 install_path=str(self._cache_dir / name),
338 )
340 # Register
341 return self._registry.register(manifest, force=force)
343 async def import_all(self, max_skills: int = 50) -> list[InstallResult]:
344 """Import all available OpenClaw skills."""
345 if not self._catalog:
346 await self.list_available()
348 results = []
349 semaphore = asyncio.Semaphore(5) # Limit concurrent fetches
351 async def _import_one(skill: RemoteSkill):
352 async with semaphore:
353 return await self.import_skill(skill.name)
355 tasks = [_import_one(s) for s in self._catalog[:max_skills]]
356 raw_results = await asyncio.gather(*tasks, return_exceptions=True)
358 for r in raw_results:
359 if isinstance(r, Exception):
360 pass
361 elif r is not None:
362 results.append(r)
364 return results
366 async def search(self, query: str) -> list[RemoteSkill]:
367 """Search the catalog by name/description/tag."""
368 if not self._catalog:
369 await self.list_available()
371 q = query.lower()
372 results = []
373 for s in self._catalog:
374 if (q in s.name.lower() or
375 q in s.description.lower() or
376 any(q in t.lower() for t in s.tags)):
377 results.append(s)
378 return results
380 # ── Internal ──
382 async def _fetch_url(self, url: str) -> str:
383 """Fetch URL content (supports GitHub API + raw)."""
384 if self._fetch_fn:
385 return self._fetch_fn(url)
387 try:
388 import aiohttp
389 async with aiohttp.ClientSession() as session:
390 async with session.get(url, headers={"Accept": "application/vnd.github.v3.raw"}) as resp:
391 if resp.status == 200:
392 return await resp.text()
393 except Exception:
394 pass
396 # Fallback: urllib
397 try:
398 import urllib.request
399 req = urllib.request.Request(url, headers={"Accept": "application/vnd.github.v3.raw"})
400 with urllib.request.urlopen(req, timeout=10) as resp:
401 return resp.read().decode("utf-8")
402 except Exception:
403 pass
405 return ""
408# ── HuggingFace Importer ──
410class HuggingFaceImporter:
411 """Import skills from HuggingFace.co skill repositories.
413 Flow:
414 hf://username/skill-repo → download → parse skill.yaml → register
415 """
417 def __init__(self, registry: SkillRegistry, cache_dir: str = ""):
418 self._registry = registry
419 self._cache_dir = Path(cache_dir) if cache_dir else Path.home() / ".agentos" / "marketplace" / "huggingface"
420 self._cache_dir.mkdir(parents=True, exist_ok=True)
422 async def import_from_hf(self, repo_id: str, force: bool = False) -> Optional[InstallResult]:
423 """Import a skill from HuggingFace repo.
425 Args:
426 repo_id: e.g. 'username/agentos-skill-translator'
427 """
428 import aiohttp
430 # Try fetching skill.yaml from main branch
431 yaml_url = f"https://huggingface.co/{repo_id}/resolve/main/skill.yaml"
432 yaml_text = ""
433 try:
434 async with aiohttp.ClientSession() as session:
435 async with session.get(yaml_url, timeout=10) as resp:
436 if resp.status == 200:
437 yaml_text = await resp.text()
438 except Exception:
439 pass
441 if not yaml_text:
442 yaml_url = f"https://huggingface.co/{repo_id}/resolve/main/agentos.yaml"
443 try:
444 async with aiohttp.ClientSession() as session:
445 async with session.get(yaml_url, timeout=10) as resp:
446 if resp.status == 200:
447 yaml_text = await resp.text()
448 except Exception:
449 pass
451 if not yaml_text:
452 return None
454 import yaml
455 try:
456 raw = yaml.safe_load(yaml_text)
457 except yaml.YAMLError:
458 return None
460 manifest = SkillManifest.from_dict(
461 raw,
462 source=f"huggingface:{repo_id}",
463 install_path=str(self._cache_dir / repo_id.replace("/", "_")),
464 )
466 return self._registry.register(manifest, force=force)
469# ── GitHub Importer ──
471class GitHubImporter:
472 """Import skills from arbitrary GitHub repositories.
474 Flow:
475 github://user/repo/path → download skill.yaml → parse → register
476 """
478 def __init__(self, registry: SkillRegistry, cache_dir: str = ""):
479 self._registry = registry
480 self._cache_dir = Path(cache_dir) if cache_dir else Path.home() / ".agentos" / "marketplace" / "github"
481 self._cache_dir.mkdir(parents=True, exist_ok=True)
483 async def import_from_github(
484 self, repo: str, path: str = "", ref: str = "main", force: bool = False,
485 ) -> Optional[InstallResult]:
486 """Import a skill from a GitHub repo.
488 Args:
489 repo: 'user/repo'
490 path: subdirectory containing skill.yaml (e.g. 'skills/my-skill')
491 ref: branch/tag (default 'main')
492 """
493 raw_base = f"https://raw.githubusercontent.com/{repo}/{ref}"
494 manifest_path = f"{raw_base}/{path}/skill.yaml" if path else f"{raw_base}/skill.yaml"
496 yaml_text = ""
497 import aiohttp
498 try:
499 async with aiohttp.ClientSession() as session:
500 async with session.get(manifest_path, timeout=10) as resp:
501 if resp.status == 200:
502 yaml_text = await resp.text()
503 except Exception:
504 pass
506 if not yaml_text:
507 return None
509 import yaml
510 try:
511 raw = yaml.safe_load(yaml_text)
512 except yaml.YAMLError:
513 return None
515 safe_name = repo.replace("/", "_")
516 manifest = SkillManifest.from_dict(
517 raw,
518 source=f"github:{repo}/{path}" if path else f"github:{repo}",
519 install_path=str(self._cache_dir / safe_name),
520 )
522 return self._registry.register(manifest, force=force)
524 async def import_release(
525 self, repo: str, tag: str = "latest", force: bool = False,
526 ) -> Optional[InstallResult]:
527 """Import from a tagged GitHub release."""
528 if tag == "latest":
529 import aiohttp
530 url = f"https://api.github.com/repos/{repo}/releases/latest"
531 try:
532 async with aiohttp.ClientSession() as session:
533 async with session.get(url, timeout=10) as resp:
534 if resp.status == 200:
535 data = await resp.json()
536 tag = data.get("tag_name", "main")
537 except Exception:
538 tag = "main"
540 return await self.import_from_github(repo, ref=tag, force=force)
543# ── Unified Importer ──
545class UnifiedImporter:
546 """Single entry point for importing skills from any supported source.
548 Usage:
549 importer = UnifiedImporter(registry)
551 # From OpenClaw community
552 skill = await importer.import_from("openclaw:pdf-tools")
554 # From HuggingFace
555 skill = await importer.import_from("hf://username/repo")
557 # From arbitrary GitHub
558 skill = await importer.import_from("github://user/repo/skills/my-skill")
559 """
561 _PROTOCOLS = {
562 "openclaw": "openclaw",
563 "hf": "huggingface",
564 "huggingface": "huggingface",
565 "github": "github",
566 "gh": "github",
567 }
569 def __init__(self, registry: SkillRegistry, cache_dir: str = ""):
570 self._registry = registry
571 self._cache_dir = cache_dir
572 self._openclaw = OpenClawImporter(registry, cache_dir)
573 self._huggingface = HuggingFaceImporter(registry, cache_dir)
574 self._github = GitHubImporter(registry, cache_dir)
576 async def import_from(self, uri: str, force: bool = False) -> Optional[InstallResult]:
577 """Import a skill from a URI.
579 URI formats:
580 - 'openclaw:skill-name' OpenClaw community skill
581 - 'hf://user/repo' HuggingFace repo
582 - 'github://user/repo[/path]' GitHub repo
583 - 'skill-name' Default: try OpenClaw first
584 """
585 # Parse protocol
586 if "://" in uri:
587 protocol, rest = uri.split("://", 1)
588 elif ":" in uri and uri.split(":")[0] in self._PROTOCOLS:
589 protocol, rest = uri.split(":", 1)
590 else:
591 # Default: try OpenClaw
592 return await self._openclaw.import_skill(uri, force=force)
594 protocol = self._PROTOCOLS.get(protocol, protocol)
596 if protocol == "openclaw":
597 return await self._openclaw.import_skill(rest, force=force)
599 elif protocol == "huggingface":
600 return await self._huggingface.import_from_hf(rest, force=force)
602 elif protocol == "github":
603 parts = rest.split("/")
604 if len(parts) >= 2:
605 repo = f"{parts[0]}/{parts[1]}"
606 subpath = "/".join(parts[2:]) if len(parts) > 2 else ""
607 return await self._github.import_from_github(repo, subpath, force=force)
609 return None
611 async def list_openclaw(self, refresh: bool = False) -> list[RemoteSkill]:
612 """List all available OpenClaw community skills."""
613 return await self._openclaw.list_available(refresh=refresh)