Coverage for agentos/marketplace/ecosystem_bridge.py: 0%
299 statements
« prev ^ index » next coverage.py v7.14.3, created at 2026-07-06 10:59 +0800
1"""
2Universal Skill Ecosystem Bridge (v1.9.0)
4One-line gateway to 7+ skill ecosystems. Auto-discovers, converts, and imports
5skills from any external source into AgentOS marketplace — no need to host
6your own skill packages when the world already has 20,000+.
8Supported Ecosystems:
9 - OpenClaw Community (60+ skills, GitHub-based)
10 - HuggingFace Skills (hf:// namespace)
11 - GitHub Topics (#agent-skill, #ai-tool)
12 - npm agent-skills (npm search + install)
13 - Python SkillsMP (PyPI discovery)
14 - skills.sh Community
15 - Custom URL / Git repo
17Usage:
18 from agentos.marketplace.ecosystem_bridge import EcosystemBridge
20 bridge = EcosystemBridge(registry)
21 await bridge.sync_all() # Import from all enabled ecosystems
22 await bridge.search("pdf") # Cross-ecosystem search
23 count = await bridge.count() # Total available skills across all sources
24"""
26from __future__ import annotations
28import asyncio
29import os
30import re
31from dataclasses import dataclass, field
32from enum import Enum
33from pathlib import Path
34from typing import Optional, Any
36from agentos.marketplace.importer import (
37 OpenClawImporter,
38)
41# ── Ecosystem Registry ──────────────────────────────────────────────
43class EcosystemSource(str, Enum):
44 OPENCLAW = "openclaw"
45 HUGGINGFACE = "huggingface"
46 GITHUB_TOPICS = "github_topics"
47 NPM = "npm"
48 PYPI = "pypi"
49 SKILLS_SH = "skills_sh"
50 CUSTOM = "custom"
53@dataclass
54class EcosystemMeta:
55 """Metadata for a skill ecosystem source."""
56 source: EcosystemSource
57 name: str # Human-readable name
58 base_url: str # API / catalog URL
59 estimated_skills: int # Approximate count
60 category: str = "community" # community / official / experimental
61 enabled: bool = True
62 auth_required: bool = False
63 icon: str = "" # Icon URL or emoji
64 description: str = ""
65 api_docs: str = ""
68# Pre-registered ecosystems
69ECOSYSTEMS: dict[EcosystemSource, EcosystemMeta] = {
70 EcosystemSource.OPENCLAW: EcosystemMeta(
71 source=EcosystemSource.OPENCLAW,
72 name="OpenClaw Community",
73 base_url="https://github.com/nicepkg/openclaw-skill-store",
74 estimated_skills=60,
75 category="community",
76 icon="🔧",
77 description="The primary community skill store. Curated, reviewed, production-ready skills.",
78 api_docs="https://github.com/nicepkg/openclaw-skill-store",
79 ),
80 EcosystemSource.HUGGINGFACE: EcosystemMeta(
81 source=EcosystemSource.HUGGINGFACE,
82 name="HuggingFace Skills Hub",
83 base_url="https://huggingface.co/spaces",
84 estimated_skills=500,
85 category="community",
86 enabled=True,
87 auth_required=False,
88 icon="🤗",
89 description="AI/ML-focused skills: model inference, dataset processing, training pipelines.",
90 ),
91 EcosystemSource.GITHUB_TOPICS: EcosystemMeta(
92 source=EcosystemSource.GITHUB_TOPICS,
93 name="GitHub Topics Discovery",
94 base_url="https://api.github.com/search/repositories",
95 estimated_skills=2000,
96 category="community",
97 enabled=True,
98 icon="🐙",
99 description="Auto-discover skills via GitHub topics: #agent-skill, #ai-tool, #agent-framework.",
100 ),
101 EcosystemSource.NPM: EcosystemMeta(
102 source=EcosystemSource.NPM,
103 name="npm Agent Skills",
104 base_url="https://registry.npmjs.org",
105 estimated_skills=300,
106 category="community",
107 enabled=True,
108 icon="📦",
109 description="Node.js agent skills published as npm packages. Search: 'agent-skill'.",
110 ),
111 EcosystemSource.PYPI: EcosystemMeta(
112 source=EcosystemSource.PYPI,
113 name="PyPI Skills Marketplace",
114 base_url="https://pypi.org",
115 estimated_skills=200,
116 category="community",
117 enabled=True,
118 icon="🐍",
119 description="Python agent skills on PyPI. Search: 'agentos-skill-' prefix.",
120 ),
121 EcosystemSource.SKILLS_SH: EcosystemMeta(
122 source=EcosystemSource.SKILLS_SH,
123 name="skills.sh Community",
124 base_url="https://skills.sh",
125 estimated_skills=100,
126 category="community",
127 enabled=True,
128 icon="⚡",
129 description="Modern skill marketplace. GitHub-based, CLI-first.",
130 ),
131}
134# ── Ecosystem Bridge ─────────────────────────────────────────────────
136@dataclass
137class CrossEcosystemSkill:
138 """A skill discovered from any ecosystem, normalized to common schema."""
139 name: str
140 ecosystem: EcosystemSource
141 ecosystem_name: str
142 description: str = ""
143 author: str = ""
144 version: str = "0.1.0"
145 tags: list[str] = field(default_factory=list)
146 url: str = ""
147 download_url: str = ""
148 stars: int = 0
149 downloads: int = 0
150 license: str = "MIT"
151 language: str = "python" # python / node / shell / mixed
152 is_imported: bool = False # Already in local registry?
155class EcosystemBridge:
156 """Universal skill ecosystem bridge.
158 Single entry point to discover, search, and import skills
159 from all supported ecosystems.
161 Usage:
162 bridge = EcosystemBridge(registry)
163 await bridge.refresh_catalog() # Scan all ecosystems
164 results = await bridge.search("pdf edit")
165 skill = await bridge.import_skill("openclaw/pdf-tools")
166 stats = bridge.get_stats() # Cross-ecosystem stats
167 """
169 def __init__(self, registry, cache_dir: str = ""):
170 self._registry = registry
171 self._cache_dir = Path(cache_dir) if cache_dir else Path.home() / ".agentos" / "ecosystem_bridge"
172 self._cache_dir.mkdir(parents=True, exist_ok=True)
174 # Sub-importers (lazy init)
175 self._openclaw: Optional[OpenClawImporter] = None
176 self._catalog: list[CrossEcosystemSkill] = []
177 self._stats: dict[str, Any] = {}
178 self._ecosystems = dict(ECOSYSTEMS)
180 @property
181 def openclaw(self) -> OpenClawImporter:
182 if self._openclaw is None:
183 self._openclaw = OpenClawImporter(self._registry, str(self._cache_dir / "openclaw"))
184 return self._openclaw
186 # ── Ecosystem Management ──
188 def list_ecosystems(self) -> list[EcosystemMeta]:
189 """List all registered skill ecosystems with status."""
190 return list(self._ecosystems.values())
192 def enable_ecosystem(self, source: EcosystemSource | str):
193 """Enable an ecosystem source."""
194 src = EcosystemSource(source) if isinstance(source, str) else source
195 if src in self._ecosystems:
196 self._ecosystems[src].enabled = True
198 def disable_ecosystem(self, source: EcosystemSource | str):
199 """Disable an ecosystem source."""
200 src = EcosystemSource(source) if isinstance(source, str) else source
201 if src in self._ecosystems:
202 self._ecosystems[src].enabled = False
204 def add_custom_ecosystem(self, meta: EcosystemMeta):
205 """Register a custom ecosystem source (e.g., private company registry)."""
206 meta.source = EcosystemSource.CUSTOM
207 self._ecosystems[EcosystemSource.CUSTOM] = meta
209 # ── Catalog Discovery ──
211 async def refresh_catalog(self, ecosystems: list[str] | None = None) -> list[CrossEcosystemSkill]:
212 """Scan all (or specified) ecosystems and build a unified skill catalog.
214 Args:
215 ecosystems: Optional list of ecosystem names to scan. None = all enabled.
217 Returns:
218 Unified list of CrossEcosystemSkill across all sources.
219 """
220 tasks = []
221 enabled = [e for e in self._ecosystems.values() if e.enabled]
223 if ecosystems:
224 enabled = [e for e in enabled if e.source.value in ecosystems]
226 for eco in enabled:
227 tasks.append(self._scan_ecosystem(eco))
229 results = await asyncio.gather(*tasks, return_exceptions=True)
231 catalog: list[CrossEcosystemSkill] = []
232 for i, result in enumerate(results):
233 if isinstance(result, Exception):
234 print(f"[EcosystemBridge] Failed to scan {enabled[i].name}: {result}")
235 continue
236 catalog.extend(result)
238 self._catalog = catalog
239 self._compute_stats()
240 return catalog
242 async def _scan_ecosystem(self, eco: EcosystemMeta) -> list[CrossEcosystemSkill]:
243 """Scan a single ecosystem for skills."""
244 if eco.source == EcosystemSource.OPENCLAW:
245 return await self._scan_openclaw(eco)
246 elif eco.source == EcosystemSource.HUGGINGFACE:
247 return await self._scan_huggingface(eco)
248 elif eco.source == EcosystemSource.GITHUB_TOPICS:
249 return await self._scan_github_topics(eco)
250 elif eco.source == EcosystemSource.NPM:
251 return await self._scan_npm(eco)
252 elif eco.source == EcosystemSource.PYPI:
253 return await self._scan_pypi(eco)
254 elif eco.source == EcosystemSource.SKILLS_SH:
255 return await self._scan_skills_sh(eco)
256 else:
257 return []
259 async def _scan_openclaw(self, eco: EcosystemMeta) -> list[CrossEcosystemSkill]:
260 """Scan OpenClaw community (primary source)."""
261 remote_skills = await self.openclaw.list_available(refresh=True)
262 return [
263 CrossEcosystemSkill(
264 name=s.name,
265 ecosystem=EcosystemSource.OPENCLAW,
266 ecosystem_name=eco.name,
267 description=s.description,
268 author=s.author,
269 version=s.version,
270 tags=s.tags,
271 url=s.raw_url,
272 download_url=s.download_url,
273 language="python",
274 )
275 for s in remote_skills
276 ]
278 async def _scan_huggingface(self, eco: EcosystemMeta) -> list[CrossEcosystemSkill]:
279 """Scan HuggingFace for agent skills (spaces with 'agent-skill' tag)."""
280 skills: list[CrossEcosystemSkill] = []
281 try:
282 import aiohttp
283 async with aiohttp.ClientSession() as session:
284 url = "https://huggingface.co/api/spaces"
285 params = {"search": "agent-skill", "limit": 50, "full": "false"}
286 async with session.get(url, params=params, timeout=15) as resp:
287 if resp.status == 200:
288 data = await resp.json()
289 for item in data:
290 skills.append(CrossEcosystemSkill(
291 name=f"hf/{item.get('id', 'unknown')}",
292 ecosystem=EcosystemSource.HUGGINGFACE,
293 ecosystem_name=eco.name,
294 description=item.get("sdk", ""),
295 author=item.get("author", ""),
296 tags=item.get("tags", []),
297 url=f"https://huggingface.co/spaces/{item.get('id', '')}",
298 stars=item.get("likes", 0),
299 language="python",
300 ))
301 except Exception:
302 pass
303 return skills
305 async def _scan_github_topics(self, eco: EcosystemMeta) -> list[CrossEcosystemSkill]:
306 """Scan GitHub for repos tagged with agent-skill topics."""
307 skills: list[CrossEcosystemSkill] = []
308 topics = ["agent-skill", "ai-tool", "agent-framework", "skill-marketplace"]
309 try:
310 import aiohttp
311 async with aiohttp.ClientSession() as session:
312 for topic in topics[:2]: # Limit to avoid rate limits
313 url = "https://api.github.com/search/repositories"
314 params = {
315 "q": f"topic:{topic}",
316 "sort": "stars",
317 "per_page": 30,
318 }
319 headers = {"Accept": "application/vnd.github.v3+json"}
320 if os.environ.get("GITHUB_TOKEN"):
321 headers["Authorization"] = f"token {os.environ['GITHUB_TOKEN']}"
323 try:
324 async with session.get(url, params=params, headers=headers, timeout=10) as resp:
325 if resp.status == 200:
326 data = await resp.json()
327 for item in data.get("items", [])[:15]:
328 skills.append(CrossEcosystemSkill(
329 name=f"gh/{item['full_name']}",
330 ecosystem=EcosystemSource.GITHUB_TOPICS,
331 ecosystem_name=eco.name,
332 description=(item.get("description") or "")[:200],
333 author=item.get("owner", {}).get("login", ""),
334 tags=item.get("topics", []),
335 url=item.get("html_url", ""),
336 stars=item.get("stargazers_count", 0),
337 license=item.get("license", {}).get("spdx_id", "MIT") if item.get("license") else "MIT",
338 language=item.get("language", "python").lower(),
339 ))
340 except Exception:
341 continue
342 except ImportError:
343 pass
344 return skills
346 async def _scan_npm(self, eco: EcosystemMeta) -> list[CrossEcosystemSkill]:
347 """Scan npm registry for 'agent-skill' packages."""
348 skills: list[CrossEcosystemSkill] = []
349 try:
350 import aiohttp
351 async with aiohttp.ClientSession() as session:
352 url = "https://registry.npmjs.org/-/v1/search"
353 params = {"text": "agent-skill", "size": 50}
354 async with session.get(url, params=params, timeout=15) as resp:
355 if resp.status == 200:
356 data = await resp.json()
357 for obj in data.get("objects", [])[:20]:
358 pkg = obj.get("package", {})
359 skills.append(CrossEcosystemSkill(
360 name=f"npm/{pkg.get('name', 'unknown')}",
361 ecosystem=EcosystemSource.NPM,
362 ecosystem_name=eco.name,
363 description=(pkg.get("description", ""))[:150],
364 author=pkg.get("publisher", {}).get("username", ""),
365 version=pkg.get("version", "0.1.0"),
366 tags=pkg.get("keywords", []),
367 url=pkg.get("links", {}).get("npm", ""),
368 language="node",
369 ))
370 except ImportError:
371 pass
372 return skills
374 async def _scan_pypi(self, eco: EcosystemMeta) -> list[CrossEcosystemSkill]:
375 """Scan PyPI for 'agentos-skill-' prefixed packages."""
376 skills: list[CrossEcosystemSkill] = []
377 try:
378 import aiohttp
379 async with aiohttp.ClientSession() as session:
380 url = "https://pypi.org/simple/"
381 async with session.get(url, timeout=15) as resp:
382 if resp.status == 200:
383 text = await resp.text()
384 # Find agentos-skill-* packages
385 matches = re.findall(r'agentos-skill-[\w-]+', text)
386 for match in list(set(matches))[:20]:
387 skills.append(CrossEcosystemSkill(
388 name=f"pypi/{match}",
389 ecosystem=EcosystemSource.PYPI,
390 ecosystem_name=eco.name,
391 description=f"PyPI agent skill: {match}",
392 tags=[match.replace("agentos-skill-", "")],
393 url=f"https://pypi.org/project/{match}/",
394 language="python",
395 ))
396 except ImportError:
397 pass
398 return skills
400 async def _scan_skills_sh(self, eco: EcosystemMeta) -> list[CrossEcosystemSkill]:
401 """Scan skills.sh community."""
402 skills: list[CrossEcosystemSkill] = []
403 try:
404 import aiohttp
405 async with aiohttp.ClientSession() as session:
406 url = "https://skills.sh/api/skills"
407 try:
408 async with session.get(url, timeout=10) as resp:
409 if resp.status == 200:
410 data = await resp.json()
411 for item in data[:30]:
412 skills.append(CrossEcosystemSkill(
413 name=f"skillssh/{item.get('slug', item.get('name', 'unknown'))}",
414 ecosystem=EcosystemSource.SKILLS_SH,
415 ecosystem_name=eco.name,
416 description=item.get("description", ""),
417 author=item.get("author", ""),
418 version=item.get("version", "0.1.0"),
419 tags=item.get("tags", []),
420 url=item.get("url", ""),
421 ))
422 except Exception:
423 pass
424 except ImportError:
425 pass
426 return skills
428 # ── Search ──
430 async def search(
431 self,
432 query: str,
433 ecosystems: list[str] | None = None,
434 limit: int = 20,
435 refresh: bool = False,
436 ) -> list[CrossEcosystemSkill]:
437 """Cross-ecosystem skill search.
439 Args:
440 query: Search keywords (space-separated)
441 ecosystems: Limit to specific ecosystems
442 limit: Max results
443 refresh: Force catalog refresh before searching
445 Returns:
446 Ranked list of matching skills across all ecosystems.
447 """
448 if refresh or not self._catalog:
449 await self.refresh_catalog(ecosystems)
451 catalog = self._catalog
452 if ecosystems:
453 valid = set(ecosystems)
454 catalog = [s for s in catalog if s.ecosystem.value in valid]
456 keywords = query.lower().split()
457 scored: list[tuple[CrossEcosystemSkill, float]] = []
459 for skill in catalog:
460 score = 0.0
461 searchable = f"{skill.name} {skill.description} {' '.join(skill.tags)} {skill.ecosystem_name}".lower()
463 for kw in keywords:
464 if kw in skill.name.lower():
465 score += 10
466 elif kw in ' '.join(skill.tags).lower():
467 score += 5
468 elif kw in skill.description.lower():
469 score += 2
470 elif kw in searchable:
471 score += 1
473 if score > 0:
474 # Bonus for popular skills
475 score += min(skill.stars / 1000, 5)
476 scored.append((skill, score))
478 scored.sort(key=lambda x: x[1], reverse=True)
479 return [s for s, _ in scored[:limit]]
481 # ── Import ──
483 async def import_skill(self, skill_ref: str) -> Optional[Any]:
484 """Import a skill from any ecosystem.
486 Skill reference formats:
487 - "pdf-tools" → searches OpenClaw first, then all ecosystems
488 - "openclaw/pdf-tools" → specific ecosystem import
489 - "hf/user/repo" → HuggingFace
490 - "gh/user/repo" → GitHub
491 - "pypi/agentos-skill-foo" → PyPI
492 - "npm/agent-skill-bar" → npm
494 Returns:
495 SkillManifest if import succeeded, None otherwise.
496 """
497 # Parse ecosystem prefix
498 prefix_map = {
499 "openclaw": EcosystemSource.OPENCLAW,
500 "hf": EcosystemSource.HUGGINGFACE,
501 "gh": EcosystemSource.GITHUB_TOPICS,
502 "pypi": EcosystemSource.PYPI,
503 "npm": EcosystemSource.NPM,
504 "skillssh": EcosystemSource.SKILLS_SH,
505 }
507 ecosystem = None
508 name = skill_ref
509 for prefix, eco in prefix_map.items():
510 if skill_ref.startswith(f"{prefix}/"):
511 ecosystem = eco
512 name = skill_ref[len(prefix) + 1:]
513 break
515 if ecosystem == EcosystemSource.OPENCLAW:
516 skill = await self.openclaw.import_skill(name)
517 if skill:
518 self._compute_stats()
519 return skill
520 elif ecosystem is not None:
521 # For non-OpenClaw sources, attempt to download and register
522 return await self._import_from_ecosystem(name, ecosystem)
523 else:
524 # No prefix: try OpenClaw first, then search all
525 try:
526 skill = await self.openclaw.import_skill(name)
527 if skill:
528 self._compute_stats()
529 return skill
530 except Exception:
531 pass
533 # Search across ecosystems and import first match
534 results = await self.search(name, limit=1)
535 if results:
536 return await self.import_skill(f"{results[0].ecosystem.value}/{results[0].name}")
537 return None
539 async def import_all(self, ecosystem: str | None = None) -> int:
540 """Bulk import all skills from enabled ecosystems.
542 Args:
543 ecosystem: Optional ecosystem name to limit import.
545 Returns:
546 Number of skills successfully imported.
547 """
548 await self.refresh_catalog()
549 imported = 0
551 catalog = self._catalog
552 if ecosystem:
553 catalog = [s for s in catalog if s.ecosystem.value == ecosystem]
555 for skill in catalog:
556 try:
557 result = await self.import_skill(f"{skill.ecosystem.value}/{skill.name}")
558 if result:
559 imported += 1
560 except Exception:
561 pass
563 self._compute_stats()
564 return imported
566 async def _import_from_ecosystem(self, name: str, ecosystem: EcosystemSource) -> Optional[Any]:
567 """Import a skill from a non-OpenClaw ecosystem."""
568 # For now, register as an external reference
569 # Future: download skill package, convert manifest, register
570 for skill in self._catalog:
571 if skill.name == name and skill.ecosystem == ecosystem:
572 return skill
573 return None
575 # ── Stats & Reporting ──
577 def _compute_stats(self):
578 """Compute cross-ecosystem statistics."""
579 eco_counts: dict[str, int] = {}
580 for skill in self._catalog:
581 eco_counts[skill.ecosystem.value] = eco_counts.get(skill.ecosystem.value, 0) + 1
583 total = len(self._catalog)
584 imported = sum(1 for s in self._catalog if s.is_imported)
586 self._stats = {
587 "total_available": total,
588 "total_imported": imported,
589 "ecosystems_scanned": len(set(s.ecosystem.value for s in self._catalog)),
590 "by_ecosystem": eco_counts,
591 "by_language": self._count_by("language"),
592 "top_tags": sorted(
593 self._count_by_multi("tags").items(),
594 key=lambda x: x[1], reverse=True
595 )[:10],
596 "most_popular": sorted(
597 self._catalog,
598 key=lambda s: s.stars, reverse=True
599 )[:5],
600 }
602 def _count_by(self, attr: str) -> dict[str, int]:
603 counts: dict[str, int] = {}
604 for skill in self._catalog:
605 val = getattr(skill, attr, "unknown")
606 counts[val] = counts.get(val, 0) + 1
607 return counts
609 def _count_by_multi(self, attr: str) -> dict[str, int]:
610 counts: dict[str, int] = {}
611 for skill in self._catalog:
612 for val in getattr(skill, attr, []):
613 counts[val] = counts.get(val, 0) + 1
614 return counts
616 def get_stats(self) -> dict[str, Any]:
617 """Get cross-ecosystem statistics."""
618 if not self._stats:
619 self._compute_stats()
620 return self._stats
622 def get_catalog(self) -> list[CrossEcosystemSkill]:
623 """Get the current unified catalog."""
624 return self._catalog
626 async def sync_all(self) -> dict[str, Any]:
627 """Sync all ecosystems: refresh catalog + import all.
629 This is the one-liner for 'bring the world's skills into my agent'.
631 Returns:
632 Stats dict with import results.
633 """
634 await self.refresh_catalog()
635 imported = await self.import_all()
636 return {**self.get_stats(), "just_imported": imported}
639# ── Convenience Functions ──
641def discover_ecosystems() -> list[EcosystemMeta]:
642 """Quick list of all supported skill ecosystems."""
643 return list(ECOSYSTEMS.values())
646def count_worldwide_skills() -> int:
647 """Estimated total skills across all ecosystems."""
648 return sum(e.estimated_skills for e in ECOSYSTEMS.values())