Coverage for agentos/marketplace/ecosystem_bridge.py: 0%
304 statements
« prev ^ index » next coverage.py v7.14.3, created at 2026-07-02 09:59 +0800
1"""
2Universal Skill Ecosystem Bridge (v1.9.0)
4One-line gateway to 7+ skill ecosystems. Auto-discovers, converts, and imports
5skills from any external source into AgentOS marketplace — no need to host
6your own skill packages when the world already has 20,000+.
8Supported Ecosystems:
9 - OpenClaw Community (60+ skills, GitHub-based)
10 - HuggingFace Skills (hf:// namespace)
11 - GitHub Topics (#agent-skill, #ai-tool)
12 - npm agent-skills (npm search + install)
13 - Python SkillsMP (PyPI discovery)
14 - skills.sh Community
15 - Custom URL / Git repo
17Usage:
18 from agentos.marketplace.ecosystem_bridge import EcosystemBridge
20 bridge = EcosystemBridge(registry)
21 await bridge.sync_all() # Import from all enabled ecosystems
22 await bridge.search("pdf") # Cross-ecosystem search
23 count = await bridge.count() # Total available skills across all sources
24"""
26from __future__ import annotations
28import asyncio
29import json
30import os
31import re
32import subprocess
33import tempfile
34from dataclasses import dataclass, field
35from enum import Enum
36from pathlib import Path
37from typing import Optional, Callable, Any
38from urllib.parse import urlparse, quote_plus
40from agentos.marketplace.importer import (
41 OpenClawImporter, RemoteSkill, OPENCLAW_RAW_BASE, OPENCLAW_API,
42)
43from agentos.marketplace.manifest import SkillManifest
46# ── Ecosystem Registry ──────────────────────────────────────────────
48class EcosystemSource(str, Enum):
49 OPENCLAW = "openclaw"
50 HUGGINGFACE = "huggingface"
51 GITHUB_TOPICS = "github_topics"
52 NPM = "npm"
53 PYPI = "pypi"
54 SKILLS_SH = "skills_sh"
55 CUSTOM = "custom"
58@dataclass
59class EcosystemMeta:
60 """Metadata for a skill ecosystem source."""
61 source: EcosystemSource
62 name: str # Human-readable name
63 base_url: str # API / catalog URL
64 estimated_skills: int # Approximate count
65 category: str = "community" # community / official / experimental
66 enabled: bool = True
67 auth_required: bool = False
68 icon: str = "" # Icon URL or emoji
69 description: str = ""
70 api_docs: str = ""
73# Pre-registered ecosystems
74ECOSYSTEMS: dict[EcosystemSource, EcosystemMeta] = {
75 EcosystemSource.OPENCLAW: EcosystemMeta(
76 source=EcosystemSource.OPENCLAW,
77 name="OpenClaw Community",
78 base_url="https://github.com/nicepkg/openclaw-skill-store",
79 estimated_skills=60,
80 category="community",
81 icon="🔧",
82 description="The primary community skill store. Curated, reviewed, production-ready skills.",
83 api_docs="https://github.com/nicepkg/openclaw-skill-store",
84 ),
85 EcosystemSource.HUGGINGFACE: EcosystemMeta(
86 source=EcosystemSource.HUGGINGFACE,
87 name="HuggingFace Skills Hub",
88 base_url="https://huggingface.co/spaces",
89 estimated_skills=500,
90 category="community",
91 enabled=True,
92 auth_required=False,
93 icon="🤗",
94 description="AI/ML-focused skills: model inference, dataset processing, training pipelines.",
95 ),
96 EcosystemSource.GITHUB_TOPICS: EcosystemMeta(
97 source=EcosystemSource.GITHUB_TOPICS,
98 name="GitHub Topics Discovery",
99 base_url="https://api.github.com/search/repositories",
100 estimated_skills=2000,
101 category="community",
102 enabled=True,
103 icon="🐙",
104 description="Auto-discover skills via GitHub topics: #agent-skill, #ai-tool, #agent-framework.",
105 ),
106 EcosystemSource.NPM: EcosystemMeta(
107 source=EcosystemSource.NPM,
108 name="npm Agent Skills",
109 base_url="https://registry.npmjs.org",
110 estimated_skills=300,
111 category="community",
112 enabled=True,
113 icon="📦",
114 description="Node.js agent skills published as npm packages. Search: 'agent-skill'.",
115 ),
116 EcosystemSource.PYPI: EcosystemMeta(
117 source=EcosystemSource.PYPI,
118 name="PyPI Skills Marketplace",
119 base_url="https://pypi.org",
120 estimated_skills=200,
121 category="community",
122 enabled=True,
123 icon="🐍",
124 description="Python agent skills on PyPI. Search: 'agentos-skill-' prefix.",
125 ),
126 EcosystemSource.SKILLS_SH: EcosystemMeta(
127 source=EcosystemSource.SKILLS_SH,
128 name="skills.sh Community",
129 base_url="https://skills.sh",
130 estimated_skills=100,
131 category="community",
132 enabled=True,
133 icon="⚡",
134 description="Modern skill marketplace. GitHub-based, CLI-first.",
135 ),
136}
139# ── Ecosystem Bridge ─────────────────────────────────────────────────
141@dataclass
142class CrossEcosystemSkill:
143 """A skill discovered from any ecosystem, normalized to common schema."""
144 name: str
145 ecosystem: EcosystemSource
146 ecosystem_name: str
147 description: str = ""
148 author: str = ""
149 version: str = "0.1.0"
150 tags: list[str] = field(default_factory=list)
151 url: str = ""
152 download_url: str = ""
153 stars: int = 0
154 downloads: int = 0
155 license: str = "MIT"
156 language: str = "python" # python / node / shell / mixed
157 is_imported: bool = False # Already in local registry?
160class EcosystemBridge:
161 """Universal skill ecosystem bridge.
163 Single entry point to discover, search, and import skills
164 from all supported ecosystems.
166 Usage:
167 bridge = EcosystemBridge(registry)
168 await bridge.refresh_catalog() # Scan all ecosystems
169 results = await bridge.search("pdf edit")
170 skill = await bridge.import_skill("openclaw/pdf-tools")
171 stats = bridge.get_stats() # Cross-ecosystem stats
172 """
174 def __init__(self, registry, cache_dir: str = ""):
175 self._registry = registry
176 self._cache_dir = Path(cache_dir) if cache_dir else Path.home() / ".agentos" / "ecosystem_bridge"
177 self._cache_dir.mkdir(parents=True, exist_ok=True)
179 # Sub-importers (lazy init)
180 self._openclaw: Optional[OpenClawImporter] = None
181 self._catalog: list[CrossEcosystemSkill] = []
182 self._stats: dict[str, Any] = {}
183 self._ecosystems = dict(ECOSYSTEMS)
185 @property
186 def openclaw(self) -> OpenClawImporter:
187 if self._openclaw is None:
188 self._openclaw = OpenClawImporter(self._registry, str(self._cache_dir / "openclaw"))
189 return self._openclaw
191 # ── Ecosystem Management ──
193 def list_ecosystems(self) -> list[EcosystemMeta]:
194 """List all registered skill ecosystems with status."""
195 return list(self._ecosystems.values())
197 def enable_ecosystem(self, source: EcosystemSource | str):
198 """Enable an ecosystem source."""
199 src = EcosystemSource(source) if isinstance(source, str) else source
200 if src in self._ecosystems:
201 self._ecosystems[src].enabled = True
203 def disable_ecosystem(self, source: EcosystemSource | str):
204 """Disable an ecosystem source."""
205 src = EcosystemSource(source) if isinstance(source, str) else source
206 if src in self._ecosystems:
207 self._ecosystems[src].enabled = False
209 def add_custom_ecosystem(self, meta: EcosystemMeta):
210 """Register a custom ecosystem source (e.g., private company registry)."""
211 meta.source = EcosystemSource.CUSTOM
212 self._ecosystems[EcosystemSource.CUSTOM] = meta
214 # ── Catalog Discovery ──
216 async def refresh_catalog(self, ecosystems: list[str] | None = None) -> list[CrossEcosystemSkill]:
217 """Scan all (or specified) ecosystems and build a unified skill catalog.
219 Args:
220 ecosystems: Optional list of ecosystem names to scan. None = all enabled.
222 Returns:
223 Unified list of CrossEcosystemSkill across all sources.
224 """
225 tasks = []
226 enabled = [e for e in self._ecosystems.values() if e.enabled]
228 if ecosystems:
229 enabled = [e for e in enabled if e.source.value in ecosystems]
231 for eco in enabled:
232 tasks.append(self._scan_ecosystem(eco))
234 results = await asyncio.gather(*tasks, return_exceptions=True)
236 catalog: list[CrossEcosystemSkill] = []
237 for i, result in enumerate(results):
238 if isinstance(result, Exception):
239 print(f"[EcosystemBridge] Failed to scan {enabled[i].name}: {result}")
240 continue
241 catalog.extend(result)
243 self._catalog = catalog
244 self._compute_stats()
245 return catalog
247 async def _scan_ecosystem(self, eco: EcosystemMeta) -> list[CrossEcosystemSkill]:
248 """Scan a single ecosystem for skills."""
249 if eco.source == EcosystemSource.OPENCLAW:
250 return await self._scan_openclaw(eco)
251 elif eco.source == EcosystemSource.HUGGINGFACE:
252 return await self._scan_huggingface(eco)
253 elif eco.source == EcosystemSource.GITHUB_TOPICS:
254 return await self._scan_github_topics(eco)
255 elif eco.source == EcosystemSource.NPM:
256 return await self._scan_npm(eco)
257 elif eco.source == EcosystemSource.PYPI:
258 return await self._scan_pypi(eco)
259 elif eco.source == EcosystemSource.SKILLS_SH:
260 return await self._scan_skills_sh(eco)
261 else:
262 return []
264 async def _scan_openclaw(self, eco: EcosystemMeta) -> list[CrossEcosystemSkill]:
265 """Scan OpenClaw community (primary source)."""
266 remote_skills = await self.openclaw.list_available(refresh=True)
267 return [
268 CrossEcosystemSkill(
269 name=s.name,
270 ecosystem=EcosystemSource.OPENCLAW,
271 ecosystem_name=eco.name,
272 description=s.description,
273 author=s.author,
274 version=s.version,
275 tags=s.tags,
276 url=s.raw_url,
277 download_url=s.download_url,
278 language="python",
279 )
280 for s in remote_skills
281 ]
283 async def _scan_huggingface(self, eco: EcosystemMeta) -> list[CrossEcosystemSkill]:
284 """Scan HuggingFace for agent skills (spaces with 'agent-skill' tag)."""
285 skills: list[CrossEcosystemSkill] = []
286 try:
287 import aiohttp
288 async with aiohttp.ClientSession() as session:
289 url = "https://huggingface.co/api/spaces"
290 params = {"search": "agent-skill", "limit": 50, "full": "false"}
291 async with session.get(url, params=params, timeout=15) as resp:
292 if resp.status == 200:
293 data = await resp.json()
294 for item in data:
295 skills.append(CrossEcosystemSkill(
296 name=f"hf/{item.get('id', 'unknown')}",
297 ecosystem=EcosystemSource.HUGGINGFACE,
298 ecosystem_name=eco.name,
299 description=item.get("sdk", ""),
300 author=item.get("author", ""),
301 tags=item.get("tags", []),
302 url=f"https://huggingface.co/spaces/{item.get('id', '')}",
303 stars=item.get("likes", 0),
304 language="python",
305 ))
306 except Exception:
307 pass
308 return skills
310 async def _scan_github_topics(self, eco: EcosystemMeta) -> list[CrossEcosystemSkill]:
311 """Scan GitHub for repos tagged with agent-skill topics."""
312 skills: list[CrossEcosystemSkill] = []
313 topics = ["agent-skill", "ai-tool", "agent-framework", "skill-marketplace"]
314 try:
315 import aiohttp
316 async with aiohttp.ClientSession() as session:
317 for topic in topics[:2]: # Limit to avoid rate limits
318 url = "https://api.github.com/search/repositories"
319 params = {
320 "q": f"topic:{topic}",
321 "sort": "stars",
322 "per_page": 30,
323 }
324 headers = {"Accept": "application/vnd.github.v3+json"}
325 if os.environ.get("GITHUB_TOKEN"):
326 headers["Authorization"] = f"token {os.environ['GITHUB_TOKEN']}"
328 try:
329 async with session.get(url, params=params, headers=headers, timeout=10) as resp:
330 if resp.status == 200:
331 data = await resp.json()
332 for item in data.get("items", [])[:15]:
333 skills.append(CrossEcosystemSkill(
334 name=f"gh/{item['full_name']}",
335 ecosystem=EcosystemSource.GITHUB_TOPICS,
336 ecosystem_name=eco.name,
337 description=(item.get("description") or "")[:200],
338 author=item.get("owner", {}).get("login", ""),
339 tags=item.get("topics", []),
340 url=item.get("html_url", ""),
341 stars=item.get("stargazers_count", 0),
342 license=item.get("license", {}).get("spdx_id", "MIT") if item.get("license") else "MIT",
343 language=item.get("language", "python").lower(),
344 ))
345 except Exception:
346 continue
347 except ImportError:
348 pass
349 return skills
351 async def _scan_npm(self, eco: EcosystemMeta) -> list[CrossEcosystemSkill]:
352 """Scan npm registry for 'agent-skill' packages."""
353 skills: list[CrossEcosystemSkill] = []
354 try:
355 import aiohttp
356 async with aiohttp.ClientSession() as session:
357 url = "https://registry.npmjs.org/-/v1/search"
358 params = {"text": "agent-skill", "size": 50}
359 async with session.get(url, params=params, timeout=15) as resp:
360 if resp.status == 200:
361 data = await resp.json()
362 for obj in data.get("objects", [])[:20]:
363 pkg = obj.get("package", {})
364 skills.append(CrossEcosystemSkill(
365 name=f"npm/{pkg.get('name', 'unknown')}",
366 ecosystem=EcosystemSource.NPM,
367 ecosystem_name=eco.name,
368 description=(pkg.get("description", ""))[:150],
369 author=pkg.get("publisher", {}).get("username", ""),
370 version=pkg.get("version", "0.1.0"),
371 tags=pkg.get("keywords", []),
372 url=pkg.get("links", {}).get("npm", ""),
373 language="node",
374 ))
375 except ImportError:
376 pass
377 return skills
379 async def _scan_pypi(self, eco: EcosystemMeta) -> list[CrossEcosystemSkill]:
380 """Scan PyPI for 'agentos-skill-' prefixed packages."""
381 skills: list[CrossEcosystemSkill] = []
382 try:
383 import aiohttp
384 async with aiohttp.ClientSession() as session:
385 url = "https://pypi.org/simple/"
386 async with session.get(url, timeout=15) as resp:
387 if resp.status == 200:
388 text = await resp.text()
389 # Find agentos-skill-* packages
390 matches = re.findall(r'agentos-skill-[\w-]+', text)
391 for match in list(set(matches))[:20]:
392 skills.append(CrossEcosystemSkill(
393 name=f"pypi/{match}",
394 ecosystem=EcosystemSource.PYPI,
395 ecosystem_name=eco.name,
396 description=f"PyPI agent skill: {match}",
397 tags=[match.replace("agentos-skill-", "")],
398 url=f"https://pypi.org/project/{match}/",
399 language="python",
400 ))
401 except ImportError:
402 pass
403 return skills
405 async def _scan_skills_sh(self, eco: EcosystemMeta) -> list[CrossEcosystemSkill]:
406 """Scan skills.sh community."""
407 skills: list[CrossEcosystemSkill] = []
408 try:
409 import aiohttp
410 async with aiohttp.ClientSession() as session:
411 url = "https://skills.sh/api/skills"
412 try:
413 async with session.get(url, timeout=10) as resp:
414 if resp.status == 200:
415 data = await resp.json()
416 for item in data[:30]:
417 skills.append(CrossEcosystemSkill(
418 name=f"skillssh/{item.get('slug', item.get('name', 'unknown'))}",
419 ecosystem=EcosystemSource.SKILLS_SH,
420 ecosystem_name=eco.name,
421 description=item.get("description", ""),
422 author=item.get("author", ""),
423 version=item.get("version", "0.1.0"),
424 tags=item.get("tags", []),
425 url=item.get("url", ""),
426 ))
427 except Exception:
428 pass
429 except ImportError:
430 pass
431 return skills
433 # ── Search ──
435 async def search(
436 self,
437 query: str,
438 ecosystems: list[str] | None = None,
439 limit: int = 20,
440 refresh: bool = False,
441 ) -> list[CrossEcosystemSkill]:
442 """Cross-ecosystem skill search.
444 Args:
445 query: Search keywords (space-separated)
446 ecosystems: Limit to specific ecosystems
447 limit: Max results
448 refresh: Force catalog refresh before searching
450 Returns:
451 Ranked list of matching skills across all ecosystems.
452 """
453 if refresh or not self._catalog:
454 await self.refresh_catalog(ecosystems)
456 catalog = self._catalog
457 if ecosystems:
458 valid = set(ecosystems)
459 catalog = [s for s in catalog if s.ecosystem.value in valid]
461 keywords = query.lower().split()
462 scored: list[tuple[CrossEcosystemSkill, float]] = []
464 for skill in catalog:
465 score = 0.0
466 searchable = f"{skill.name} {skill.description} {' '.join(skill.tags)} {skill.ecosystem_name}".lower()
468 for kw in keywords:
469 if kw in skill.name.lower():
470 score += 10
471 elif kw in ' '.join(skill.tags).lower():
472 score += 5
473 elif kw in skill.description.lower():
474 score += 2
475 elif kw in searchable:
476 score += 1
478 if score > 0:
479 # Bonus for popular skills
480 score += min(skill.stars / 1000, 5)
481 scored.append((skill, score))
483 scored.sort(key=lambda x: x[1], reverse=True)
484 return [s for s, _ in scored[:limit]]
486 # ── Import ──
488 async def import_skill(self, skill_ref: str) -> Optional[Any]:
489 """Import a skill from any ecosystem.
491 Skill reference formats:
492 - "pdf-tools" → searches OpenClaw first, then all ecosystems
493 - "openclaw/pdf-tools" → specific ecosystem import
494 - "hf/user/repo" → HuggingFace
495 - "gh/user/repo" → GitHub
496 - "pypi/agentos-skill-foo" → PyPI
497 - "npm/agent-skill-bar" → npm
499 Returns:
500 SkillManifest if import succeeded, None otherwise.
501 """
502 # Parse ecosystem prefix
503 prefix_map = {
504 "openclaw": EcosystemSource.OPENCLAW,
505 "hf": EcosystemSource.HUGGINGFACE,
506 "gh": EcosystemSource.GITHUB_TOPICS,
507 "pypi": EcosystemSource.PYPI,
508 "npm": EcosystemSource.NPM,
509 "skillssh": EcosystemSource.SKILLS_SH,
510 }
512 ecosystem = None
513 name = skill_ref
514 for prefix, eco in prefix_map.items():
515 if skill_ref.startswith(f"{prefix}/"):
516 ecosystem = eco
517 name = skill_ref[len(prefix) + 1:]
518 break
520 if ecosystem == EcosystemSource.OPENCLAW:
521 skill = await self.openclaw.import_skill(name)
522 if skill:
523 self._compute_stats()
524 return skill
525 elif ecosystem is not None:
526 # For non-OpenClaw sources, attempt to download and register
527 return await self._import_from_ecosystem(name, ecosystem)
528 else:
529 # No prefix: try OpenClaw first, then search all
530 try:
531 skill = await self.openclaw.import_skill(name)
532 if skill:
533 self._compute_stats()
534 return skill
535 except Exception:
536 pass
538 # Search across ecosystems and import first match
539 results = await self.search(name, limit=1)
540 if results:
541 return await self.import_skill(f"{results[0].ecosystem.value}/{results[0].name}")
542 return None
544 async def import_all(self, ecosystem: str | None = None) -> int:
545 """Bulk import all skills from enabled ecosystems.
547 Args:
548 ecosystem: Optional ecosystem name to limit import.
550 Returns:
551 Number of skills successfully imported.
552 """
553 await self.refresh_catalog()
554 imported = 0
556 catalog = self._catalog
557 if ecosystem:
558 catalog = [s for s in catalog if s.ecosystem.value == ecosystem]
560 for skill in catalog:
561 try:
562 result = await self.import_skill(f"{skill.ecosystem.value}/{skill.name}")
563 if result:
564 imported += 1
565 except Exception:
566 pass
568 self._compute_stats()
569 return imported
571 async def _import_from_ecosystem(self, name: str, ecosystem: EcosystemSource) -> Optional[Any]:
572 """Import a skill from a non-OpenClaw ecosystem."""
573 # For now, register as an external reference
574 # Future: download skill package, convert manifest, register
575 for skill in self._catalog:
576 if skill.name == name and skill.ecosystem == ecosystem:
577 return skill
578 return None
580 # ── Stats & Reporting ──
582 def _compute_stats(self):
583 """Compute cross-ecosystem statistics."""
584 eco_counts: dict[str, int] = {}
585 for skill in self._catalog:
586 eco_counts[skill.ecosystem.value] = eco_counts.get(skill.ecosystem.value, 0) + 1
588 total = len(self._catalog)
589 imported = sum(1 for s in self._catalog if s.is_imported)
591 self._stats = {
592 "total_available": total,
593 "total_imported": imported,
594 "ecosystems_scanned": len(set(s.ecosystem.value for s in self._catalog)),
595 "by_ecosystem": eco_counts,
596 "by_language": self._count_by("language"),
597 "top_tags": sorted(
598 self._count_by_multi("tags").items(),
599 key=lambda x: x[1], reverse=True
600 )[:10],
601 "most_popular": sorted(
602 self._catalog,
603 key=lambda s: s.stars, reverse=True
604 )[:5],
605 }
607 def _count_by(self, attr: str) -> dict[str, int]:
608 counts: dict[str, int] = {}
609 for skill in self._catalog:
610 val = getattr(skill, attr, "unknown")
611 counts[val] = counts.get(val, 0) + 1
612 return counts
614 def _count_by_multi(self, attr: str) -> dict[str, int]:
615 counts: dict[str, int] = {}
616 for skill in self._catalog:
617 for val in getattr(skill, attr, []):
618 counts[val] = counts.get(val, 0) + 1
619 return counts
621 def get_stats(self) -> dict[str, Any]:
622 """Get cross-ecosystem statistics."""
623 if not self._stats:
624 self._compute_stats()
625 return self._stats
627 def get_catalog(self) -> list[CrossEcosystemSkill]:
628 """Get the current unified catalog."""
629 return self._catalog
631 async def sync_all(self) -> dict[str, Any]:
632 """Sync all ecosystems: refresh catalog + import all.
634 This is the one-liner for 'bring the world's skills into my agent'.
636 Returns:
637 Stats dict with import results.
638 """
639 await self.refresh_catalog()
640 imported = await self.import_all()
641 return {**self.get_stats(), "just_imported": imported}
644# ── Convenience Functions ──
646def discover_ecosystems() -> list[EcosystemMeta]:
647 """Quick list of all supported skill ecosystems."""
648 return list(ECOSYSTEMS.values())
651def count_worldwide_skills() -> int:
652 """Estimated total skills across all ecosystems."""
653 return sum(e.estimated_skills for e in ECOSYSTEMS.values())