Coverage for agentos/marketplace/ecosystem_bridge.py: 0%
299 statements
« prev ^ index » next coverage.py v7.14.3, created at 2026-07-09 09:19 +0800
1"""
2Universal Skill Ecosystem Bridge (v1.9.0)
4One-line gateway to 7+ skill ecosystems. Auto-discovers, converts, and imports
5skills from any external source into AgentOS marketplace — no need to host
6your own skill packages when the world already has 20,000+.
8Supported Ecosystems:
9 - OpenClaw Community (60+ skills, GitHub-based)
10 - HuggingFace Skills (hf:// namespace)
11 - GitHub Topics (#agent-skill, #ai-tool)
12 - npm agent-skills (npm search + install)
13 - Python SkillsMP (PyPI discovery)
14 - skills.sh Community
15 - Custom URL / Git repo
17Usage:
18 from agentos.marketplace.ecosystem_bridge import EcosystemBridge
20 bridge = EcosystemBridge(registry)
21 await bridge.sync_all() # Import from all enabled ecosystems
22 await bridge.search("pdf") # Cross-ecosystem search
23 count = await bridge.count() # Total available skills across all sources
24"""
26from __future__ import annotations
28import asyncio
29import os
30import re
31from dataclasses import dataclass, field
32from enum import StrEnum
33from pathlib import Path
34from typing import Any
36from agentos.marketplace.importer import (
37 OpenClawImporter,
38)
40# ── Ecosystem Registry ──────────────────────────────────────────────
43class EcosystemSource(StrEnum):
44 OPENCLAW = "openclaw"
45 HUGGINGFACE = "huggingface"
46 GITHUB_TOPICS = "github_topics"
47 NPM = "npm"
48 PYPI = "pypi"
49 SKILLS_SH = "skills_sh"
50 CUSTOM = "custom"
53@dataclass
54class EcosystemMeta:
55 """Metadata for a skill ecosystem source."""
57 source: EcosystemSource
58 name: str # Human-readable name
59 base_url: str # API / catalog URL
60 estimated_skills: int # Approximate count
61 category: str = "community" # community / official / experimental
62 enabled: bool = True
63 auth_required: bool = False
64 icon: str = "" # Icon URL or emoji
65 description: str = ""
66 api_docs: str = ""
69# Pre-registered ecosystems
70ECOSYSTEMS: dict[EcosystemSource, EcosystemMeta] = {
71 EcosystemSource.OPENCLAW: EcosystemMeta(
72 source=EcosystemSource.OPENCLAW,
73 name="OpenClaw Community",
74 base_url="https://github.com/nicepkg/openclaw-skill-store",
75 estimated_skills=60,
76 category="community",
77 icon="🔧",
78 description="The primary community skill store. Curated, reviewed, production-ready skills.",
79 api_docs="https://github.com/nicepkg/openclaw-skill-store",
80 ),
81 EcosystemSource.HUGGINGFACE: EcosystemMeta(
82 source=EcosystemSource.HUGGINGFACE,
83 name="HuggingFace Skills Hub",
84 base_url="https://huggingface.co/spaces",
85 estimated_skills=500,
86 category="community",
87 enabled=True,
88 auth_required=False,
89 icon="🤗",
90 description="AI/ML-focused skills: model inference, dataset processing, training pipelines.",
91 ),
92 EcosystemSource.GITHUB_TOPICS: EcosystemMeta(
93 source=EcosystemSource.GITHUB_TOPICS,
94 name="GitHub Topics Discovery",
95 base_url="https://api.github.com/search/repositories",
96 estimated_skills=2000,
97 category="community",
98 enabled=True,
99 icon="🐙",
100 description="Auto-discover skills via GitHub topics: #agent-skill, #ai-tool, #agent-framework.",
101 ),
102 EcosystemSource.NPM: EcosystemMeta(
103 source=EcosystemSource.NPM,
104 name="npm Agent Skills",
105 base_url="https://registry.npmjs.org",
106 estimated_skills=300,
107 category="community",
108 enabled=True,
109 icon="📦",
110 description="Node.js agent skills published as npm packages. Search: 'agent-skill'.",
111 ),
112 EcosystemSource.PYPI: EcosystemMeta(
113 source=EcosystemSource.PYPI,
114 name="PyPI Skills Marketplace",
115 base_url="https://pypi.org",
116 estimated_skills=200,
117 category="community",
118 enabled=True,
119 icon="🐍",
120 description="Python agent skills on PyPI. Search: 'agentos-skill-' prefix.",
121 ),
122 EcosystemSource.SKILLS_SH: EcosystemMeta(
123 source=EcosystemSource.SKILLS_SH,
124 name="skills.sh Community",
125 base_url="https://skills.sh",
126 estimated_skills=100,
127 category="community",
128 enabled=True,
129 icon="⚡",
130 description="Modern skill marketplace. GitHub-based, CLI-first.",
131 ),
132}
135# ── Ecosystem Bridge ─────────────────────────────────────────────────
138@dataclass
139class CrossEcosystemSkill:
140 """A skill discovered from any ecosystem, normalized to common schema."""
142 name: str
143 ecosystem: EcosystemSource
144 ecosystem_name: str
145 description: str = ""
146 author: str = ""
147 version: str = "0.1.0"
148 tags: list[str] = field(default_factory=list)
149 url: str = ""
150 download_url: str = ""
151 stars: int = 0
152 downloads: int = 0
153 license: str = "MIT"
154 language: str = "python" # python / node / shell / mixed
155 is_imported: bool = False # Already in local registry?
158class EcosystemBridge:
159 """Universal skill ecosystem bridge.
161 Single entry point to discover, search, and import skills
162 from all supported ecosystems.
164 Usage:
165 bridge = EcosystemBridge(registry)
166 await bridge.refresh_catalog() # Scan all ecosystems
167 results = await bridge.search("pdf edit")
168 skill = await bridge.import_skill("openclaw/pdf-tools")
169 stats = bridge.get_stats() # Cross-ecosystem stats
170 """
172 def __init__(self, registry, cache_dir: str = ""):
173 self._registry = registry
174 self._cache_dir = (
175 Path(cache_dir) if cache_dir else Path.home() / ".agentos" / "ecosystem_bridge"
176 )
177 self._cache_dir.mkdir(parents=True, exist_ok=True)
179 # Sub-importers (lazy init)
180 self._openclaw: OpenClawImporter | None = None
181 self._catalog: list[CrossEcosystemSkill] = []
182 self._stats: dict[str, Any] = {}
183 self._ecosystems = dict(ECOSYSTEMS)
185 @property
186 def openclaw(self) -> OpenClawImporter:
187 if self._openclaw is None:
188 self._openclaw = OpenClawImporter(self._registry, str(self._cache_dir / "openclaw"))
189 return self._openclaw
191 # ── Ecosystem Management ──
193 def list_ecosystems(self) -> list[EcosystemMeta]:
194 """List all registered skill ecosystems with status."""
195 return list(self._ecosystems.values())
197 def enable_ecosystem(self, source: EcosystemSource | str):
198 """Enable an ecosystem source."""
199 src = EcosystemSource(source) if isinstance(source, str) else source
200 if src in self._ecosystems:
201 self._ecosystems[src].enabled = True
203 def disable_ecosystem(self, source: EcosystemSource | str):
204 """Disable an ecosystem source."""
205 src = EcosystemSource(source) if isinstance(source, str) else source
206 if src in self._ecosystems:
207 self._ecosystems[src].enabled = False
209 def add_custom_ecosystem(self, meta: EcosystemMeta):
210 """Register a custom ecosystem source (e.g., private company registry)."""
211 meta.source = EcosystemSource.CUSTOM
212 self._ecosystems[EcosystemSource.CUSTOM] = meta
214 # ── Catalog Discovery ──
216 async def refresh_catalog(
217 self, ecosystems: list[str] | None = None
218 ) -> list[CrossEcosystemSkill]:
219 """Scan all (or specified) ecosystems and build a unified skill catalog.
221 Args:
222 ecosystems: Optional list of ecosystem names to scan. None = all enabled.
224 Returns:
225 Unified list of CrossEcosystemSkill across all sources.
226 """
227 tasks = []
228 enabled = [e for e in self._ecosystems.values() if e.enabled]
230 if ecosystems:
231 enabled = [e for e in enabled if e.source.value in ecosystems]
233 for eco in enabled:
234 tasks.append(self._scan_ecosystem(eco))
236 results = await asyncio.gather(*tasks, return_exceptions=True)
238 catalog: list[CrossEcosystemSkill] = []
239 for i, result in enumerate(results):
240 if isinstance(result, Exception):
241 print(f"[EcosystemBridge] Failed to scan {enabled[i].name}: {result}")
242 continue
243 catalog.extend(result)
245 self._catalog = catalog
246 self._compute_stats()
247 return catalog
249 async def _scan_ecosystem(self, eco: EcosystemMeta) -> list[CrossEcosystemSkill]:
250 """Scan a single ecosystem for skills."""
251 if eco.source == EcosystemSource.OPENCLAW:
252 return await self._scan_openclaw(eco)
253 elif eco.source == EcosystemSource.HUGGINGFACE:
254 return await self._scan_huggingface(eco)
255 elif eco.source == EcosystemSource.GITHUB_TOPICS:
256 return await self._scan_github_topics(eco)
257 elif eco.source == EcosystemSource.NPM:
258 return await self._scan_npm(eco)
259 elif eco.source == EcosystemSource.PYPI:
260 return await self._scan_pypi(eco)
261 elif eco.source == EcosystemSource.SKILLS_SH:
262 return await self._scan_skills_sh(eco)
263 else:
264 return []
266 async def _scan_openclaw(self, eco: EcosystemMeta) -> list[CrossEcosystemSkill]:
267 """Scan OpenClaw community (primary source)."""
268 remote_skills = await self.openclaw.list_available(refresh=True)
269 return [
270 CrossEcosystemSkill(
271 name=s.name,
272 ecosystem=EcosystemSource.OPENCLAW,
273 ecosystem_name=eco.name,
274 description=s.description,
275 author=s.author,
276 version=s.version,
277 tags=s.tags,
278 url=s.raw_url,
279 download_url=s.download_url,
280 language="python",
281 )
282 for s in remote_skills
283 ]
285 async def _scan_huggingface(self, eco: EcosystemMeta) -> list[CrossEcosystemSkill]:
286 """Scan HuggingFace for agent skills (spaces with 'agent-skill' tag)."""
287 skills: list[CrossEcosystemSkill] = []
288 try:
289 import aiohttp
291 async with aiohttp.ClientSession() as session:
292 url = "https://huggingface.co/api/spaces"
293 params = {"search": "agent-skill", "limit": 50, "full": "false"}
294 async with session.get(url, params=params, timeout=15) as resp:
295 if resp.status == 200:
296 data = await resp.json()
297 for item in data:
298 skills.append(
299 CrossEcosystemSkill(
300 name=f"hf/{item.get('id', 'unknown')}",
301 ecosystem=EcosystemSource.HUGGINGFACE,
302 ecosystem_name=eco.name,
303 description=item.get("sdk", ""),
304 author=item.get("author", ""),
305 tags=item.get("tags", []),
306 url=f"https://huggingface.co/spaces/{item.get('id', '')}",
307 stars=item.get("likes", 0),
308 language="python",
309 )
310 )
311 except Exception:
312 pass
313 return skills
315 async def _scan_github_topics(self, eco: EcosystemMeta) -> list[CrossEcosystemSkill]:
316 """Scan GitHub for repos tagged with agent-skill topics."""
317 skills: list[CrossEcosystemSkill] = []
318 topics = ["agent-skill", "ai-tool", "agent-framework", "skill-marketplace"]
319 try:
320 import aiohttp
322 async with aiohttp.ClientSession() as session:
323 for topic in topics[:2]: # Limit to avoid rate limits
324 url = "https://api.github.com/search/repositories"
325 params = {
326 "q": f"topic:{topic}",
327 "sort": "stars",
328 "per_page": 30,
329 }
330 headers = {"Accept": "application/vnd.github.v3+json"}
331 if os.environ.get("GITHUB_TOKEN"):
332 headers["Authorization"] = f"token {os.environ['GITHUB_TOKEN']}"
334 try:
335 async with session.get(
336 url, params=params, headers=headers, timeout=10
337 ) as resp:
338 if resp.status == 200:
339 data = await resp.json()
340 for item in data.get("items", [])[:15]:
341 skills.append(
342 CrossEcosystemSkill(
343 name=f"gh/{item['full_name']}",
344 ecosystem=EcosystemSource.GITHUB_TOPICS,
345 ecosystem_name=eco.name,
346 description=(item.get("description") or "")[:200],
347 author=item.get("owner", {}).get("login", ""),
348 tags=item.get("topics", []),
349 url=item.get("html_url", ""),
350 stars=item.get("stargazers_count", 0),
351 license=(
352 item.get("license", {}).get("spdx_id", "MIT")
353 if item.get("license")
354 else "MIT"
355 ),
356 language=item.get("language", "python").lower(),
357 )
358 )
359 except Exception:
360 continue
361 except ImportError:
362 pass
363 return skills
365 async def _scan_npm(self, eco: EcosystemMeta) -> list[CrossEcosystemSkill]:
366 """Scan npm registry for 'agent-skill' packages."""
367 skills: list[CrossEcosystemSkill] = []
368 try:
369 import aiohttp
371 async with aiohttp.ClientSession() as session:
372 url = "https://registry.npmjs.org/-/v1/search"
373 params = {"text": "agent-skill", "size": 50}
374 async with session.get(url, params=params, timeout=15) as resp:
375 if resp.status == 200:
376 data = await resp.json()
377 for obj in data.get("objects", [])[:20]:
378 pkg = obj.get("package", {})
379 skills.append(
380 CrossEcosystemSkill(
381 name=f"npm/{pkg.get('name', 'unknown')}",
382 ecosystem=EcosystemSource.NPM,
383 ecosystem_name=eco.name,
384 description=(pkg.get("description", ""))[:150],
385 author=pkg.get("publisher", {}).get("username", ""),
386 version=pkg.get("version", "0.1.0"),
387 tags=pkg.get("keywords", []),
388 url=pkg.get("links", {}).get("npm", ""),
389 language="node",
390 )
391 )
392 except ImportError:
393 pass
394 return skills
396 async def _scan_pypi(self, eco: EcosystemMeta) -> list[CrossEcosystemSkill]:
397 """Scan PyPI for 'agentos-skill-' prefixed packages."""
398 skills: list[CrossEcosystemSkill] = []
399 try:
400 import aiohttp
402 async with aiohttp.ClientSession() as session:
403 url = "https://pypi.org/simple/"
404 async with session.get(url, timeout=15) as resp:
405 if resp.status == 200:
406 text = await resp.text()
407 # Find agentos-skill-* packages
408 matches = re.findall(r"agentos-skill-[\w-]+", text)
409 for match in list(set(matches))[:20]:
410 skills.append(
411 CrossEcosystemSkill(
412 name=f"pypi/{match}",
413 ecosystem=EcosystemSource.PYPI,
414 ecosystem_name=eco.name,
415 description=f"PyPI agent skill: {match}",
416 tags=[match.replace("agentos-skill-", "")],
417 url=f"https://pypi.org/project/{match}/",
418 language="python",
419 )
420 )
421 except ImportError:
422 pass
423 return skills
425 async def _scan_skills_sh(self, eco: EcosystemMeta) -> list[CrossEcosystemSkill]:
426 """Scan skills.sh community."""
427 skills: list[CrossEcosystemSkill] = []
428 try:
429 import aiohttp
431 async with aiohttp.ClientSession() as session:
432 url = "https://skills.sh/api/skills"
433 try:
434 async with session.get(url, timeout=10) as resp:
435 if resp.status == 200:
436 data = await resp.json()
437 for item in data[:30]:
438 skills.append(
439 CrossEcosystemSkill(
440 name=f"skillssh/{item.get('slug', item.get('name', 'unknown'))}",
441 ecosystem=EcosystemSource.SKILLS_SH,
442 ecosystem_name=eco.name,
443 description=item.get("description", ""),
444 author=item.get("author", ""),
445 version=item.get("version", "0.1.0"),
446 tags=item.get("tags", []),
447 url=item.get("url", ""),
448 )
449 )
450 except Exception:
451 pass
452 except ImportError:
453 pass
454 return skills
456 # ── Search ──
458 async def search(
459 self,
460 query: str,
461 ecosystems: list[str] | None = None,
462 limit: int = 20,
463 refresh: bool = False,
464 ) -> list[CrossEcosystemSkill]:
465 """Cross-ecosystem skill search.
467 Args:
468 query: Search keywords (space-separated)
469 ecosystems: Limit to specific ecosystems
470 limit: Max results
471 refresh: Force catalog refresh before searching
473 Returns:
474 Ranked list of matching skills across all ecosystems.
475 """
476 if refresh or not self._catalog:
477 await self.refresh_catalog(ecosystems)
479 catalog = self._catalog
480 if ecosystems:
481 valid = set(ecosystems)
482 catalog = [s for s in catalog if s.ecosystem.value in valid]
484 keywords = query.lower().split()
485 scored: list[tuple[CrossEcosystemSkill, float]] = []
487 for skill in catalog:
488 score = 0.0
489 searchable = f"{skill.name} {skill.description} {' '.join(skill.tags)} {skill.ecosystem_name}".lower()
491 for kw in keywords:
492 if kw in skill.name.lower():
493 score += 10
494 elif kw in " ".join(skill.tags).lower():
495 score += 5
496 elif kw in skill.description.lower():
497 score += 2
498 elif kw in searchable:
499 score += 1
501 if score > 0:
502 # Bonus for popular skills
503 score += min(skill.stars / 1000, 5)
504 scored.append((skill, score))
506 scored.sort(key=lambda x: x[1], reverse=True)
507 return [s for s, _ in scored[:limit]]
509 # ── Import ──
511 async def import_skill(self, skill_ref: str) -> Any | None:
512 """Import a skill from any ecosystem.
514 Skill reference formats:
515 - "pdf-tools" → searches OpenClaw first, then all ecosystems
516 - "openclaw/pdf-tools" → specific ecosystem import
517 - "hf/user/repo" → HuggingFace
518 - "gh/user/repo" → GitHub
519 - "pypi/agentos-skill-foo" → PyPI
520 - "npm/agent-skill-bar" → npm
522 Returns:
523 SkillManifest if import succeeded, None otherwise.
524 """
525 # Parse ecosystem prefix
526 prefix_map = {
527 "openclaw": EcosystemSource.OPENCLAW,
528 "hf": EcosystemSource.HUGGINGFACE,
529 "gh": EcosystemSource.GITHUB_TOPICS,
530 "pypi": EcosystemSource.PYPI,
531 "npm": EcosystemSource.NPM,
532 "skillssh": EcosystemSource.SKILLS_SH,
533 }
535 ecosystem = None
536 name = skill_ref
537 for prefix, eco in prefix_map.items():
538 if skill_ref.startswith(f"{prefix}/"):
539 ecosystem = eco
540 name = skill_ref[len(prefix) + 1 :]
541 break
543 if ecosystem == EcosystemSource.OPENCLAW:
544 skill = await self.openclaw.import_skill(name)
545 if skill:
546 self._compute_stats()
547 return skill
548 elif ecosystem is not None:
549 # For non-OpenClaw sources, attempt to download and register
550 return await self._import_from_ecosystem(name, ecosystem)
551 else:
552 # No prefix: try OpenClaw first, then search all
553 try:
554 skill = await self.openclaw.import_skill(name)
555 if skill:
556 self._compute_stats()
557 return skill
558 except Exception:
559 pass
561 # Search across ecosystems and import first match
562 results = await self.search(name, limit=1)
563 if results:
564 return await self.import_skill(f"{results[0].ecosystem.value}/{results[0].name}")
565 return None
567 async def import_all(self, ecosystem: str | None = None) -> int:
568 """Bulk import all skills from enabled ecosystems.
570 Args:
571 ecosystem: Optional ecosystem name to limit import.
573 Returns:
574 Number of skills successfully imported.
575 """
576 await self.refresh_catalog()
577 imported = 0
579 catalog = self._catalog
580 if ecosystem:
581 catalog = [s for s in catalog if s.ecosystem.value == ecosystem]
583 for skill in catalog:
584 try:
585 result = await self.import_skill(f"{skill.ecosystem.value}/{skill.name}")
586 if result:
587 imported += 1
588 except Exception:
589 pass
591 self._compute_stats()
592 return imported
594 async def _import_from_ecosystem(self, name: str, ecosystem: EcosystemSource) -> Any | None:
595 """Import a skill from a non-OpenClaw ecosystem."""
596 # For now, register as an external reference
597 # Future: download skill package, convert manifest, register
598 for skill in self._catalog:
599 if skill.name == name and skill.ecosystem == ecosystem:
600 return skill
601 return None
603 # ── Stats & Reporting ──
605 def _compute_stats(self):
606 """Compute cross-ecosystem statistics."""
607 eco_counts: dict[str, int] = {}
608 for skill in self._catalog:
609 eco_counts[skill.ecosystem.value] = eco_counts.get(skill.ecosystem.value, 0) + 1
611 total = len(self._catalog)
612 imported = sum(1 for s in self._catalog if s.is_imported)
614 self._stats = {
615 "total_available": total,
616 "total_imported": imported,
617 "ecosystems_scanned": len(set(s.ecosystem.value for s in self._catalog)),
618 "by_ecosystem": eco_counts,
619 "by_language": self._count_by("language"),
620 "top_tags": sorted(
621 self._count_by_multi("tags").items(), key=lambda x: x[1], reverse=True
622 )[:10],
623 "most_popular": sorted(self._catalog, key=lambda s: s.stars, reverse=True)[:5],
624 }
626 def _count_by(self, attr: str) -> dict[str, int]:
627 counts: dict[str, int] = {}
628 for skill in self._catalog:
629 val = getattr(skill, attr, "unknown")
630 counts[val] = counts.get(val, 0) + 1
631 return counts
633 def _count_by_multi(self, attr: str) -> dict[str, int]:
634 counts: dict[str, int] = {}
635 for skill in self._catalog:
636 for val in getattr(skill, attr, []):
637 counts[val] = counts.get(val, 0) + 1
638 return counts
640 def get_stats(self) -> dict[str, Any]:
641 """Get cross-ecosystem statistics."""
642 if not self._stats:
643 self._compute_stats()
644 return self._stats
646 def get_catalog(self) -> list[CrossEcosystemSkill]:
647 """Get the current unified catalog."""
648 return self._catalog
650 async def sync_all(self) -> dict[str, Any]:
651 """Sync all ecosystems: refresh catalog + import all.
653 This is the one-liner for 'bring the world's skills into my agent'.
655 Returns:
656 Stats dict with import results.
657 """
658 await self.refresh_catalog()
659 imported = await self.import_all()
660 return {**self.get_stats(), "just_imported": imported}
663# ── Convenience Functions ──
666def discover_ecosystems() -> list[EcosystemMeta]:
667 """Quick list of all supported skill ecosystems."""
668 return list(ECOSYSTEMS.values())
671def count_worldwide_skills() -> int:
672 """Estimated total skills across all ecosystems."""
673 return sum(e.estimated_skills for e in ECOSYSTEMS.values())