Coverage for agentos/marketplace/importer.py: 0%
241 statements
« prev ^ index » next coverage.py v7.14.3, created at 2026-07-06 10:59 +0800
« prev ^ index » next coverage.py v7.14.3, created at 2026-07-06 10:59 +0800
1"""
2Marketplace Importer — Import skills from external sources (OpenClaw, HuggingFace, GitHub).
4OpenClaw Community: https://github.com/openclaw/skills
5 - skill.yaml → SkillManifest (openclaw format)
6 - Auto-detect format, convert, register
8Usage:
9 from agentos.marketplace.importer import OpenClawImporter
10 importer = OpenClawImporter(registry)
11 skill = await importer.import_skill("pdf-tools")
12"""
14from __future__ import annotations
16import asyncio
17from dataclasses import dataclass, field
18from pathlib import Path
19from typing import Optional, Callable
21from agentos.marketplace.manifest import SkillManifest
22from agentos.marketplace.registry import SkillRegistry, InstallResult
25# ── OpenClaw Importer ──
27OPENCLAW_SKILLS_REPO = "https://github.com/nicepkg/openclaw-skill-store"
28OPENCLAW_RAW_BASE = "https://raw.githubusercontent.com/nicepkg/openclaw-skill-store/main"
29OPENCLAW_API = "https://api.github.com/repos/nicepkg/openclaw-skill-store"
32@dataclass
33class RemoteSkill:
34 """Skill metadata discovered from a remote source."""
35 name: str
36 path: str # Relative path in the repo
37 description: str = ""
38 author: str = ""
39 version: str = "0.1.0"
40 tags: list[str] = field(default_factory=list)
41 download_url: str = ""
42 raw_url: str = ""
43 source: str = "openclaw"
46class OpenClawImporter:
47 """Import skills from the OpenClaw community skill store.
49 Flow:
50 1. list_available() — fetch skill catalog from GitHub API
51 2. import_skill(name) — download skill.yaml → parse → register
52 3. import_all() — batch import all available skills
53 """
55 def __init__(self, registry: SkillRegistry, cache_dir: str = ""):
56 self._registry = registry
57 self._cache_dir = Path(cache_dir) if cache_dir else Path.home() / ".agentos" / "marketplace" / "openclaw"
58 self._cache_dir.mkdir(parents=True, exist_ok=True)
60 self._catalog: list[RemoteSkill] = []
61 self._fetch_fn: Optional[Callable[[str], str]] = None # Injection point for testing
63 # ── Catalog ──
65 async def list_available(self, refresh: bool = False) -> list[RemoteSkill]:
66 """List all available OpenClaw community skills.
68 Returns cached catalog unless refresh=True.
69 """
70 if self._catalog and not refresh:
71 return self._catalog
73 skills = []
75 # Try fetching directory listing from the raw GitHub API
76 try:
77 import aiohttp
78 except ImportError:
79 return await self._list_fallback()
81 try:
82 async with aiohttp.ClientSession() as session:
83 # Fetch the top-level directory listing
84 url = f"{OPENCLAW_API}/contents/skills"
85 async with session.get(url, headers={"Accept": "application/vnd.github.v3+json"}) as resp:
86 if resp.status != 200:
87 return await self._list_fallback()
89 entries = await resp.json()
90 for entry in entries:
91 if entry.get("type") != "dir":
92 continue
94 skill_name = entry["name"]
95 skill_path = f"skills/{skill_name}"
96 raw_url = f"{OPENCLAW_RAW_BASE}/{skill_path}/skill.yaml"
97 download_url = entry.get("url", "")
99 # Try to read skill.yaml for metadata
100 meta = await self._fetch_skill_meta(session, skill_path)
101 skills.append(RemoteSkill(
102 name=meta.get("name", skill_name),
103 path=skill_path,
104 description=meta.get("description", ""),
105 author=meta.get("author", ""),
106 version=meta.get("version", "0.1.0"),
107 tags=meta.get("tags", []),
108 raw_url=raw_url,
109 download_url=download_url,
110 ))
112 except Exception:
113 return await self._list_fallback()
115 self._catalog = skills
116 return skills
118 async def _list_fallback(self) -> list[RemoteSkill]:
119 """Fallback: return a curated list of known OpenClaw skills (60+)."""
120 known_skills = [
121 # ── Meta & Creator (3) ──
122 RemoteSkill(name="skill-creator", path="skills/skill-creator",
123 description="Guide for creating effective skills", tags=["meta", "creator"]),
124 RemoteSkill(name="mcp-builder", path="skills/mcp-builder",
125 description="Guide for creating MCP servers and tools", tags=["mcp", "infra"]),
126 RemoteSkill(name="coding-agent", path="skills/coding-agent",
127 description="Autonomous coding agent for complex software tasks", tags=["dev", "agent"]),
129 # ── Office & Documents (8) ──
130 RemoteSkill(name="docx", path="skills/docx",
131 description="Create and edit .docx documents", tags=["document", "office"]),
132 RemoteSkill(name="pdf", path="skills/pdf",
133 description="PDF manipulation toolkit: merge, split, extract, annotate", tags=["document", "pdf"]),
134 RemoteSkill(name="pptx", path="skills/pptx",
135 description="Create and edit .pptx presentations", tags=["presentation", "office"]),
136 RemoteSkill(name="xlsx", path="skills/xlsx",
137 description="Create and edit .xlsx spreadsheets with formulas and charts", tags=["spreadsheet", "office"]),
138 RemoteSkill(name="nano-pdf", path="skills/nano-pdf",
139 description="Lightweight PDF reading and text extraction", tags=["document", "pdf"]),
140 RemoteSkill(name="notion", path="skills/notion",
141 description="Notion integration: pages, databases, blocks CRUD", tags=["productivity", "notion"]),
142 RemoteSkill(name="obsidian", path="skills/obsidian",
143 description="Obsidian vault integration: read/write notes, backlinks", tags=["knowledge", "obsidian"]),
144 RemoteSkill(name="bear-notes", path="skills/bear-notes",
145 description="Bear notes app integration for Apple ecosystem", tags=["notes", "apple"]),
147 # ── Design & Creative (6) ──
148 RemoteSkill(name="brand-guidelines", path="skills/brand-guidelines",
149 description="Applies brand colors and typography to any artifact", tags=["design", "brand"]),
150 RemoteSkill(name="canvas-design", path="skills/canvas-design",
151 description="Create beautiful visual art in .png and .pdf", tags=["art", "design"]),
152 RemoteSkill(name="algorithmic-art", path="skills/algorithmic-art",
153 description="Creating algorithmic art using p5.js", tags=["art", "creative"]),
154 RemoteSkill(name="theme-factory", path="skills/theme-factory",
155 description="Apply visual themes: color schemes, typography, spacing", tags=["design", "theme"]),
156 RemoteSkill(name="slack-gif-creator", path="skills/slack-gif-creator",
157 description="Create animated GIFs optimized for Slack", tags=["media", "slack"]),
158 RemoteSkill(name="openai-image-gen", path="skills/openai-image-gen",
159 description="Generate images using DALL-E / OpenAI image API", tags=["ai", "image"]),
161 # ── Web & Frontend (5) ──
162 RemoteSkill(name="frontend-design", path="skills/frontend-design",
163 description="Create distinctive production-grade frontend interfaces", tags=["web", "frontend"]),
164 RemoteSkill(name="web-artifacts-builder", path="skills/web-artifacts-builder",
165 description="Build complex multi-file HTML artifacts with CSS/JS", tags=["web", "html"]),
166 RemoteSkill(name="web-search", path="skills/web-search",
167 description="Web search with multiple engines and result parsing", tags=["web", "search"]),
168 RemoteSkill(name="blogwatcher", path="skills/blogwatcher",
169 description="Monitor blogs and RSS feeds for updates", tags=["web", "monitoring"]),
170 RemoteSkill(name="wikipedia", path="skills/wikipedia",
171 description="Search and extract content from Wikipedia", tags=["web", "knowledge"]),
173 # ── Developer Tools (10) ──
174 RemoteSkill(name="github", path="skills/github",
175 description="GitHub API: repos, issues, PRs, actions, gists", tags=["dev", "github"]),
176 RemoteSkill(name="gh-issues", path="skills/gh-issues",
177 description="Deep GitHub issues management and triage", tags=["dev", "github"]),
178 RemoteSkill(name="git", path="skills/git",
179 description="Git version control: commit, branch, merge, rebase", tags=["dev", "vcs"]),
180 RemoteSkill(name="docker", path="skills/docker",
181 description="Docker container management: build, run, compose", tags=["dev", "infra"]),
182 RemoteSkill(name="code-review", path="skills/code-review",
183 description="Automated code review with best-practice suggestions", tags=["dev", "quality"]),
184 RemoteSkill(name="database", path="skills/database",
185 description="SQL/NoSQL database query and schema management", tags=["dev", "data"]),
186 RemoteSkill(name="api-tester", path="skills/api-tester",
187 description="REST/GraphQL API testing and documentation", tags=["dev", "api"]),
188 RemoteSkill(name="tmux", path="skills/tmux",
189 description="Tmux session management and automation", tags=["dev", "terminal"]),
190 RemoteSkill(name="node-connect", path="skills/node-connect",
191 description="Node.js runtime integration and package management", tags=["dev", "node"]),
192 RemoteSkill(name="model-usage", path="skills/model-usage",
193 description="Track and optimize AI model usage and costs", tags=["dev", "ai"]),
195 # ── Communication & Messaging (6) ──
196 RemoteSkill(name="internal-comms", path="skills/internal-comms",
197 description="Internal communications: announcements, memos, updates", tags=["writing", "business"]),
198 RemoteSkill(name="slack", path="skills/slack",
199 description="Slack integration: messages, channels, reactions", tags=["communication", "slack"]),
200 RemoteSkill(name="discord", path="skills/discord",
201 description="Discord bot integration for servers and DMs", tags=["communication", "discord"]),
202 RemoteSkill(name="email", path="skills/email",
203 description="Email composition, sending, and inbox management", tags=["communication", "email"]),
204 RemoteSkill(name="telegram", path="skills/telegram",
205 description="Telegram bot API: messages, channels, inline queries", tags=["communication", "telegram"]),
206 RemoteSkill(name="imsg", path="skills/imsg",
207 description="iMessage integration for Apple ecosystem", tags=["communication", "apple"]),
209 # ── Productivity (8) ──
210 RemoteSkill(name="calendar", path="skills/calendar",
211 description="Calendar management: events, reminders, scheduling", tags=["productivity", "time"]),
212 RemoteSkill(name="task-manager", path="skills/task-manager",
213 description="Task and to-do list management with priorities", tags=["productivity", "tasks"]),
214 RemoteSkill(name="notes", path="skills/notes",
215 description="Quick note-taking with search and organization", tags=["productivity", "notes"]),
216 RemoteSkill(name="apple-notes", path="skills/apple-notes",
217 description="Apple Notes app integration", tags=["productivity", "apple"]),
218 RemoteSkill(name="apple-reminders", path="skills/apple-reminders",
219 description="Apple Reminders app integration", tags=["productivity", "apple"]),
220 RemoteSkill(name="things-mac", path="skills/things-mac",
221 description="Things 3 task manager integration for macOS", tags=["productivity", "mac"]),
222 RemoteSkill(name="trello", path="skills/trello",
223 description="Trello board management: cards, lists, boards", tags=["productivity", "pm"]),
224 RemoteSkill(name="summarize", path="skills/summarize",
225 description="Intelligent text summarization with configurable depth", tags=["productivity", "text"]),
227 # ── Data & Analysis (5) ──
228 RemoteSkill(name="data-analysis", path="skills/data-analysis",
229 description="Statistical analysis, visualization, and reporting", tags=["data", "analytics"]),
230 RemoteSkill(name="spreadsheet", path="skills/spreadsheet",
231 description="Advanced spreadsheet operations and formulas", tags=["data", "office"]),
232 RemoteSkill(name="csv-toolkit", path="skills/csv-toolkit",
233 description="CSV parsing, transformation, and export toolkit", tags=["data", "csv"]),
234 RemoteSkill(name="json-toolkit", path="skills/json-toolkit",
235 description="JSON manipulation, validation, and transformation", tags=["data", "json"]),
236 RemoteSkill(name="markdown-toolkit", path="skills/markdown-toolkit",
237 description="Markdown rendering, conversion, and templating", tags=["writing", "markdown"]),
239 # ── Media & Multimedia (5) ──
240 RemoteSkill(name="video-frames", path="skills/video-frames",
241 description="Extract and analyze frames from video files", tags=["media", "video"]),
242 RemoteSkill(name="audio-transcribe", path="skills/audio-transcribe",
243 description="Speech-to-text transcription with Whisper API", tags=["media", "audio"]),
244 RemoteSkill(name="openai-whisper", path="skills/openai-whisper",
245 description="OpenAI Whisper speech recognition integration", tags=["media", "audio"]),
246 RemoteSkill(name="openai-whisper-api", path="skills/openai-whisper-api",
247 description="OpenAI Whisper API with batch processing", tags=["media", "audio"]),
248 RemoteSkill(name="sherpa-onnx-tts", path="skills/sherpa-onnx-tts",
249 description="Text-to-speech with Sherpa-ONNX engine", tags=["media", "tts"]),
251 # ── System & Automation (5) ──
252 RemoteSkill(name="automation", path="skills/automation",
253 description="Workflow automation: triggers, actions, scheduling", tags=["automation", "workflow"]),
254 RemoteSkill(name="file-organizer", path="skills/file-organizer",
255 description="Smart file organization: sort, rename, deduplicate", tags=["system", "files"]),
256 RemoteSkill(name="backup", path="skills/backup",
257 description="Automated backup and restore for files and configs", tags=["system", "backup"]),
258 RemoteSkill(name="weather", path="skills/weather",
259 description="Weather forecasts, alerts, and historical data", tags=["utility", "weather"]),
260 RemoteSkill(name="healthcheck", path="skills/healthcheck",
261 description="System health monitoring and diagnostics", tags=["system", "monitoring"]),
263 # ── Security & Privacy (3) ──
264 RemoteSkill(name="1password", path="skills/1password",
265 description="1Password vault integration for secrets management", tags=["security", "password"]),
266 RemoteSkill(name="encryption", path="skills/encryption",
267 description="File encryption/decryption with multiple algorithms", tags=["security", "crypto"]),
268 RemoteSkill(name="session-logs", path="skills/session-logs",
269 description="Audit and analyze agent session logs", tags=["security", "audit"]),
270 ]
271 self._catalog = known_skills
272 return known_skills
274 async def _fetch_skill_meta(self, session, skill_path: str) -> dict:
275 """Fetch skill.yaml metadata for a single skill."""
276 url = f"{OPENCLAW_API}/contents/{skill_path}/skill.yaml"
277 try:
278 async with session.get(url, headers={"Accept": "application/vnd.github.v3.raw"}) as resp:
279 if resp.status == 200:
280 text = await resp.text()
281 import yaml
282 return yaml.safe_load(text) or {}
283 except Exception:
284 pass
285 return {}
287 # ── Import ──
289 async def import_skill(self, name: str, force: bool = False) -> Optional[InstallResult]:
290 """Import a single skill from OpenClaw by name.
292 Pipeline:
293 1. Find in catalog
294 2. Fetch skill.yaml from raw GitHub
295 3. Parse as OpenClaw format → SkillManifest
296 4. Register in SkillRegistry
297 """
298 # Ensure catalog is loaded
299 if not self._catalog:
300 await self.list_available()
302 # Find skill
303 skill_ref = None
304 for s in self._catalog:
305 if s.name == name:
306 skill_ref = s
307 break
309 if not skill_ref:
310 return None
312 # Fetch skill.yaml
313 yaml_url = f"{OPENCLAW_RAW_BASE}/{skill_ref.path}/skill.yaml"
314 yaml_text = await self._fetch_url(yaml_url)
316 if not yaml_text:
317 return None
319 # Parse as OpenClaw format
320 import yaml
321 try:
322 raw = yaml.safe_load(yaml_text)
323 except yaml.YAMLError:
324 return None
326 if not raw:
327 return None
329 # Convert to SkillManifest
330 raw["format"] = "openclaw"
331 manifest = SkillManifest.from_dict(
332 raw,
333 source=f"openclaw:{name}",
334 install_path=str(self._cache_dir / name),
335 )
337 # Register
338 return self._registry.register(manifest, force=force)
340 async def import_all(self, max_skills: int = 50) -> list[InstallResult]:
341 """Import all available OpenClaw skills."""
342 if not self._catalog:
343 await self.list_available()
345 results = []
346 semaphore = asyncio.Semaphore(5) # Limit concurrent fetches
348 async def _import_one(skill: RemoteSkill):
349 async with semaphore:
350 return await self.import_skill(skill.name)
352 tasks = [_import_one(s) for s in self._catalog[:max_skills]]
353 raw_results = await asyncio.gather(*tasks, return_exceptions=True)
355 for r in raw_results:
356 if isinstance(r, Exception):
357 pass
358 elif r is not None:
359 results.append(r)
361 return results
363 async def search(self, query: str) -> list[RemoteSkill]:
364 """Search the catalog by name/description/tag."""
365 if not self._catalog:
366 await self.list_available()
368 q = query.lower()
369 results = []
370 for s in self._catalog:
371 if (q in s.name.lower() or
372 q in s.description.lower() or
373 any(q in t.lower() for t in s.tags)):
374 results.append(s)
375 return results
377 # ── Internal ──
379 async def _fetch_url(self, url: str) -> str:
380 """Fetch URL content (supports GitHub API + raw)."""
381 if self._fetch_fn:
382 return self._fetch_fn(url)
384 try:
385 import aiohttp
386 async with aiohttp.ClientSession() as session:
387 async with session.get(url, headers={"Accept": "application/vnd.github.v3.raw"}) as resp:
388 if resp.status == 200:
389 return await resp.text()
390 except Exception:
391 pass
393 # Fallback: urllib
394 try:
395 import urllib.request
396 req = urllib.request.Request(url, headers={"Accept": "application/vnd.github.v3.raw"})
397 with urllib.request.urlopen(req, timeout=10) as resp:
398 return resp.read().decode("utf-8")
399 except Exception:
400 pass
402 return ""
405# ── HuggingFace Importer ──
407class HuggingFaceImporter:
408 """Import skills from HuggingFace.co skill repositories.
410 Flow:
411 hf://username/skill-repo → download → parse skill.yaml → register
412 """
414 def __init__(self, registry: SkillRegistry, cache_dir: str = ""):
415 self._registry = registry
416 self._cache_dir = Path(cache_dir) if cache_dir else Path.home() / ".agentos" / "marketplace" / "huggingface"
417 self._cache_dir.mkdir(parents=True, exist_ok=True)
419 async def import_from_hf(self, repo_id: str, force: bool = False) -> Optional[InstallResult]:
420 """Import a skill from HuggingFace repo.
422 Args:
423 repo_id: e.g. 'username/agentos-skill-translator'
424 """
425 import aiohttp
427 # Try fetching skill.yaml from main branch
428 yaml_url = f"https://huggingface.co/{repo_id}/resolve/main/skill.yaml"
429 yaml_text = ""
430 try:
431 async with aiohttp.ClientSession() as session:
432 async with session.get(yaml_url, timeout=10) as resp:
433 if resp.status == 200:
434 yaml_text = await resp.text()
435 except Exception:
436 pass
438 if not yaml_text:
439 yaml_url = f"https://huggingface.co/{repo_id}/resolve/main/agentos.yaml"
440 try:
441 async with aiohttp.ClientSession() as session:
442 async with session.get(yaml_url, timeout=10) as resp:
443 if resp.status == 200:
444 yaml_text = await resp.text()
445 except Exception:
446 pass
448 if not yaml_text:
449 return None
451 import yaml
452 try:
453 raw = yaml.safe_load(yaml_text)
454 except yaml.YAMLError:
455 return None
457 manifest = SkillManifest.from_dict(
458 raw,
459 source=f"huggingface:{repo_id}",
460 install_path=str(self._cache_dir / repo_id.replace("/", "_")),
461 )
463 return self._registry.register(manifest, force=force)
466# ── GitHub Importer ──
468class GitHubImporter:
469 """Import skills from arbitrary GitHub repositories.
471 Flow:
472 github://user/repo/path → download skill.yaml → parse → register
473 """
475 def __init__(self, registry: SkillRegistry, cache_dir: str = ""):
476 self._registry = registry
477 self._cache_dir = Path(cache_dir) if cache_dir else Path.home() / ".agentos" / "marketplace" / "github"
478 self._cache_dir.mkdir(parents=True, exist_ok=True)
480 async def import_from_github(
481 self, repo: str, path: str = "", ref: str = "main", force: bool = False,
482 ) -> Optional[InstallResult]:
483 """Import a skill from a GitHub repo.
485 Args:
486 repo: 'user/repo'
487 path: subdirectory containing skill.yaml (e.g. 'skills/my-skill')
488 ref: branch/tag (default 'main')
489 """
490 raw_base = f"https://raw.githubusercontent.com/{repo}/{ref}"
491 manifest_path = f"{raw_base}/{path}/skill.yaml" if path else f"{raw_base}/skill.yaml"
493 yaml_text = ""
494 import aiohttp
495 try:
496 async with aiohttp.ClientSession() as session:
497 async with session.get(manifest_path, timeout=10) as resp:
498 if resp.status == 200:
499 yaml_text = await resp.text()
500 except Exception:
501 pass
503 if not yaml_text:
504 return None
506 import yaml
507 try:
508 raw = yaml.safe_load(yaml_text)
509 except yaml.YAMLError:
510 return None
512 safe_name = repo.replace("/", "_")
513 manifest = SkillManifest.from_dict(
514 raw,
515 source=f"github:{repo}/{path}" if path else f"github:{repo}",
516 install_path=str(self._cache_dir / safe_name),
517 )
519 return self._registry.register(manifest, force=force)
521 async def import_release(
522 self, repo: str, tag: str = "latest", force: bool = False,
523 ) -> Optional[InstallResult]:
524 """Import from a tagged GitHub release."""
525 if tag == "latest":
526 import aiohttp
527 url = f"https://api.github.com/repos/{repo}/releases/latest"
528 try:
529 async with aiohttp.ClientSession() as session:
530 async with session.get(url, timeout=10) as resp:
531 if resp.status == 200:
532 data = await resp.json()
533 tag = data.get("tag_name", "main")
534 except Exception:
535 tag = "main"
537 return await self.import_from_github(repo, ref=tag, force=force)
540# ── Unified Importer ──
542class UnifiedImporter:
543 """Single entry point for importing skills from any supported source.
545 Usage:
546 importer = UnifiedImporter(registry)
548 # From OpenClaw community
549 skill = await importer.import_from("openclaw:pdf-tools")
551 # From HuggingFace
552 skill = await importer.import_from("hf://username/repo")
554 # From arbitrary GitHub
555 skill = await importer.import_from("github://user/repo/skills/my-skill")
556 """
558 _PROTOCOLS = {
559 "openclaw": "openclaw",
560 "hf": "huggingface",
561 "huggingface": "huggingface",
562 "github": "github",
563 "gh": "github",
564 }
566 def __init__(self, registry: SkillRegistry, cache_dir: str = ""):
567 self._registry = registry
568 self._cache_dir = cache_dir
569 self._openclaw = OpenClawImporter(registry, cache_dir)
570 self._huggingface = HuggingFaceImporter(registry, cache_dir)
571 self._github = GitHubImporter(registry, cache_dir)
573 async def import_from(self, uri: str, force: bool = False) -> Optional[InstallResult]:
574 """Import a skill from a URI.
576 URI formats:
577 - 'openclaw:skill-name' OpenClaw community skill
578 - 'hf://user/repo' HuggingFace repo
579 - 'github://user/repo[/path]' GitHub repo
580 - 'skill-name' Default: try OpenClaw first
581 """
582 # Parse protocol
583 if "://" in uri:
584 protocol, rest = uri.split("://", 1)
585 elif ":" in uri and uri.split(":")[0] in self._PROTOCOLS:
586 protocol, rest = uri.split(":", 1)
587 else:
588 # Default: try OpenClaw
589 return await self._openclaw.import_skill(uri, force=force)
591 protocol = self._PROTOCOLS.get(protocol, protocol)
593 if protocol == "openclaw":
594 return await self._openclaw.import_skill(rest, force=force)
596 elif protocol == "huggingface":
597 return await self._huggingface.import_from_hf(rest, force=force)
599 elif protocol == "github":
600 parts = rest.split("/")
601 if len(parts) >= 2:
602 repo = f"{parts[0]}/{parts[1]}"
603 subpath = "/".join(parts[2:]) if len(parts) > 2 else ""
604 return await self._github.import_from_github(repo, subpath, force=force)
606 return None
608 async def list_openclaw(self, refresh: bool = False) -> list[RemoteSkill]:
609 """List all available OpenClaw community skills."""
610 return await self._openclaw.list_available(refresh=refresh)