ctfy.sdk.admin_resources.eval

  1"""``client.admin.llm_models`` + ``client.admin.eval_runs`` — model evaluation (admin)."""
  2
  3from __future__ import annotations
  4
  5import builtins
  6from typing import Any
  7
  8from ctfy.core.enums import RegistryReclaimPolicy
  9from ctfy.core.state.models import (
 10    EvalCampaignState,
 11    EvalRunState,
 12    HarnessState,
 13    LlmModelState,
 14    VendorState,
 15    VendorTokenState,
 16)
 17from ctfy.sdk._helpers import PagedList, _extract_items, _raise_for_status
 18from ctfy.sdk.base import BaseHttpClient
 19from ctfy.server.models import (
 20    AdminHarnessRow,
 21    AdminTaskInfo,
 22    EvalAllowanceRow,
 23    EvalArtifactReport,
 24    EvalRangeFailureReport,
 25    EvalRunCoverage,
 26    EvalRunRangeFailure,
 27    RegistryGcStatus,
 28    RegistryReclaimPlan,
 29    RegistryRepoList,
 30    RegistryTagList,
 31    RegistryUsageReport,
 32)
 33
 34
 35class AdminLlmModelsResource:
 36    """The registry of models under evaluation: register / list / delete.
 37
 38    No API key is ever stored — a registered model is addressing only
 39    (flavor + base_url + provider model id); credentials are supplied to the
 40    run at dispatch time via env.
 41    """
 42
 43    def __init__(self, http: BaseHttpClient) -> None:
 44        self._http = http
 45
 46    def list(
 47        self, *, active_only: bool = False, offset: int = 0, limit: int = 100
 48    ) -> PagedList[LlmModelState]:
 49        resp = self._http.request(
 50            "GET",
 51            "/admin/llm-models",
 52            params={"active_only": active_only, "offset": offset, "limit": limit},
 53        )
 54        _raise_for_status(resp)
 55        return _extract_items(resp.json(), LlmModelState)
 56
 57    def register(
 58        self,
 59        *,
 60        id: str,
 61        flavor: str,
 62        display_name: str = "",
 63        provider_model: str = "",
 64        base_url: str = "",
 65        vendor: str = "",
 66        notes: str = "",
 67        input_price_per_mtok: float = 0.0,
 68        output_price_per_mtok: float = 0.0,
 69    ) -> LlmModelState:
 70        """Register (or overwrite) a model. ``flavor`` is ``anthropic`` / ``openai``.
 71
 72        ``*_price_per_mtok`` is USD per 1M tokens (0 = unpriced) — drives the
 73        leaderboard's cost columns.
 74        """
 75        resp = self._http.request(
 76            "POST",
 77            "/admin/llm-models",
 78            json={
 79                "id": id,
 80                "flavor": flavor,
 81                "display_name": display_name,
 82                "provider_model": provider_model,
 83                "base_url": base_url,
 84                "vendor": vendor,
 85                "notes": notes,
 86                "input_price_per_mtok": input_price_per_mtok,
 87                "output_price_per_mtok": output_price_per_mtok,
 88            },
 89        )
 90        _raise_for_status(resp)
 91        return LlmModelState.model_validate(resp.json())
 92
 93    def delete(self, model_id: str) -> bool:
 94        resp = self._http.request("DELETE", f"/admin/llm-models/{model_id}")
 95        _raise_for_status(resp)
 96        return bool(resp.json().get("deleted"))
 97
 98    def mint_token(
 99        self,
100        model_id: str,
101        *,
102        competition_id: str,
103        label: str = "",
104        expires_in_days: int | None = None,
105    ) -> dict[str, Any]:
106        """Mint a play credential bound to this model's synthetic team.
107
108        Returns ``{token, model_id, competition_id, team_id,
109        user_id}``; the plaintext ``pf_`` bearer is shown **once** —
110        store it now, only its hash is persisted. Scoped to
111        ``competition_id`` and carrying only what a run needs
112        (``challenges: read`` + ``instances: write`` + ``submissions:
113        write``).
114
115        This is what an *external* harness — one the platform does not
116        dispatch itself — uses to keep each model on its own team.
117        Sharing a team makes ``challenge_fully_solved`` inherit between
118        models and makes them share one wrong-answer budget, silently.
119        """
120        resp = self._http.request(
121            "POST",
122            f"/admin/llm-models/{model_id}/tokens",
123            json={
124                "competition_id": competition_id,
125                "label": label,
126                "expires_in_days": expires_in_days,
127            },
128        )
129        _raise_for_status(resp)
130        data: dict[str, Any] = resp.json()
131        return data
132
133    def review(self, model_id: str, *, status: str, note: str = "") -> LlmModelState:
134        """Approve / reject a model. ``status`` is ``approved`` / ``rejected`` /
135        ``pending``; only ``approved`` models may back an official campaign."""
136        resp = self._http.request(
137            "POST",
138            f"/admin/llm-models/{model_id}/review",
139            json={"status": status, "note": note},
140        )
141        _raise_for_status(resp)
142        return LlmModelState.model_validate(resp.json())
143
144
145class AdminEvalVendorsResource:
146    """The registry of model vendors / orgs: register / list / delete.
147
148    A vendor's ``id`` is a slug matching the ``vendor`` label on each of its
149    registered models, so the leaderboard's vendor scorecard resolves display
150    metadata from one curated row.
151    """
152
153    def __init__(self, http: BaseHttpClient) -> None:
154        self._http = http
155
156    def list(
157        self, *, active_only: bool = False, offset: int = 0, limit: int = 100
158    ) -> PagedList[VendorState]:
159        resp = self._http.request(
160            "GET",
161            "/admin/eval-vendors",
162            params={"active_only": active_only, "offset": offset, "limit": limit},
163        )
164        _raise_for_status(resp)
165        return _extract_items(resp.json(), VendorState)
166
167    def register(
168        self,
169        *,
170        id: str,
171        display_name: str = "",
172        homepage: str = "",
173        contact_email: str = "",
174        active: bool = True,
175        notes: str = "",
176        max_concurrent_campaigns: int = 1,
177        daily_run_quota: int = 200,
178        monthly_cost_budget_usd: float = 0.0,
179    ) -> VendorState:
180        """Register (or overwrite) a vendor by slug ``id``.
181
182        The quota trio caps the vendor's self-service campaign usage
183        (docs/model-eval-vendor-tenancy.md §6.2); ``0`` on any axis means "no
184        limit" there. A re-register overwrites the whole row, so pass the
185        current quota values to keep them."""
186        resp = self._http.request(
187            "POST",
188            "/admin/eval-vendors",
189            json={
190                "id": id,
191                "display_name": display_name,
192                "homepage": homepage,
193                "contact_email": contact_email,
194                "active": active,
195                "notes": notes,
196                "max_concurrent_campaigns": max_concurrent_campaigns,
197                "daily_run_quota": daily_run_quota,
198                "monthly_cost_budget_usd": monthly_cost_budget_usd,
199            },
200        )
201        _raise_for_status(resp)
202        return VendorState.model_validate(resp.json())
203
204    def delete(self, vendor_id: str) -> bool:
205        resp = self._http.request("DELETE", f"/admin/eval-vendors/{vendor_id}")
206        _raise_for_status(resp)
207        return bool(resp.json().get("deleted"))
208
209    # -- vendor access tokens (tenancy) -------------------------------------
210
211    def mint_token(self, vendor_id: str, *, label: str = "", scope: str = "read") -> dict[str, Any]:
212        """Mint a ``pv_`` vendor bearer. The plaintext ``token`` in the reply
213        is shown **once** — store it now; only its hash is persisted. ``scope``
214        is ``read`` (default) or ``submit`` (self-service writes)."""
215        resp = self._http.request(
216            "POST",
217            f"/admin/eval-vendors/{vendor_id}/tokens",
218            json={"label": label, "scope": scope},
219        )
220        _raise_for_status(resp)
221        data: dict[str, Any] = resp.json()
222        return data
223
224    def list_tokens(self, vendor_id: str) -> builtins.list[VendorTokenState]:
225        """List a vendor's tokens (hashes blanked server-side)."""
226        resp = self._http.request("GET", f"/admin/eval-vendors/{vendor_id}/tokens")
227        _raise_for_status(resp)
228        return [VendorTokenState.model_validate(row) for row in resp.json()]
229
230    def revoke_token(self, vendor_id: str, token_id: str) -> bool:
231        resp = self._http.request("DELETE", f"/admin/eval-vendors/{vendor_id}/tokens/{token_id}")
232        _raise_for_status(resp)
233        return bool(resp.json().get("deleted"))
234
235
236class AdminEvalHarnessesResource:
237    """The registry of eval harnesses (agentic scaffolds): register / list / delete.
238
239    A harness's ``id`` is a slug (``ctfy`` / ``claude-code`` / …); ``image`` is
240    the container a node runs for it. Lets harness be a comparison variable
241    alongside the model.
242    """
243
244    def __init__(self, http: BaseHttpClient) -> None:
245        self._http = http
246
247    def list(
248        self, *, active_only: bool = False, offset: int = 0, limit: int = 100
249    ) -> PagedList[AdminHarnessRow]:
250        resp = self._http.request(
251            "GET",
252            "/admin/eval-harnesses",
253            params={"active_only": active_only, "offset": offset, "limit": limit},
254        )
255        _raise_for_status(resp)
256        return _extract_items(resp.json(), AdminHarnessRow)
257
258    def register(
259        self,
260        *,
261        id: str,
262        image: str = "",
263        display_name: str = "",
264        version: str = "",
265        notes: str = "",
266        active: bool = True,
267    ) -> HarnessState:
268        """Register (or overwrite) one of the *platform's* harnesses.
269
270        ⚠️ Refused with 409 on a row a user owns. The stamping here says
271        "the platform published this" — ``owner_user_id: ""`` is exactly
272        what the dispatch reads as our own image, run as it stands — so
273        aiming it at somebody's registration would strip the owner and
274        exempt the image from its admission and its isolation.
275        """
276        resp = self._http.request(
277            "POST",
278            "/admin/eval-harnesses",
279            json={
280                "id": id,
281                "image": image,
282                "display_name": display_name,
283                "version": version,
284                "notes": notes,
285                "active": active,
286            },
287        )
288        _raise_for_status(resp)
289        return HarnessState.model_validate(resp.json())
290
291    def check(self, harness_id: str) -> AdminHarnessRow:
292        """Look at the image now and record what was found.
293
294        It records evidence the automatic admission reads (#2614): a user's
295        image runs once this check and the dry run pass on one digest.
296
297        ⚠️ **A registry that would not answer comes back 200 carrying
298        ``conformance_status == "error"``**, which is a recorded answer
299        about the registry rather than a claim about the image. Only a
300        row naming no image refuses (400 ``harness_has_no_image``):
301        "there is nothing to check" and "we could not look" send the
302        reader to different places.
303        """
304        resp = self._http.request("POST", f"/admin/eval-harnesses/{harness_id}/conformance")
305        _raise_for_status(resp)
306        return AdminHarnessRow.model_validate(resp.json())
307
308    def probe(self, harness_id: str) -> AdminHarnessRow:
309        """Put the image in the dry-run line (#2608); the row as it stands.
310
311        ⚠️ **Answers the row in line, never a verdict**: ``probe_status`` is
312        ``queued`` (with ``probe_reason`` and ``probe_position``) or
313        ``running``; the verdict lands on it later — poll :meth:`list`.
314        """
315        resp = self._http.request("POST", f"/admin/eval-harnesses/{harness_id}/probe")
316        _raise_for_status(resp)
317        return AdminHarnessRow.model_validate(resp.json())
318
319    def delete(self, harness_id: str) -> bool:
320        resp = self._http.request("DELETE", f"/admin/eval-harnesses/{harness_id}")
321        _raise_for_status(resp)
322        return bool(resp.json().get("deleted"))
323
324
325class AdminEvalAllowancesResource:
326    """Grants of the platform's own model key: list / grant / revoke.
327
328    ⚠️ This is the one custody mode that spends the deployment's money on
329    somebody else's behalf, so it is an explicit grant per
330    ``(account, competition)`` rather than a fall-through from "the user
331    has no key". Re-granting keeps what was already spent.
332    """
333
334    def __init__(self, http: BaseHttpClient) -> None:
335        self._http = http
336
337    def list(self, *, competition_id: str = "") -> builtins.list[EvalAllowanceRow]:
338        """Every grant, or one competition's. ``""`` means every one."""
339        resp = self._http.request(
340            "GET", "/admin/eval-allowances", params={"competition_id": competition_id}
341        )
342        _raise_for_status(resp)
343        return [EvalAllowanceRow.model_validate(row) for row in resp.json()]
344
345    def grant(
346        self, user_id: str, competition_id: str, cap_usd: float, *, note: str = ""
347    ) -> EvalAllowanceRow:
348        """Let this account spend the platform's key, up to ``cap_usd``.
349
350        A repeat ``(account, competition)`` re-caps and **keeps the
351        spend** — raising a cap does not refund what went before.
352        """
353        resp = self._http.request(
354            "POST",
355            "/admin/eval-allowances",
356            json={
357                "user_id": user_id,
358                "competition_id": competition_id,
359                "cap_usd": cap_usd,
360                "note": note,
361            },
362        )
363        _raise_for_status(resp)
364        return EvalAllowanceRow.model_validate(resp.json())
365
366    def revoke(self, user_id: str, competition_id: str) -> bool:
367        """Take it back. Runs already paid for stay paid for."""
368        resp = self._http.request("DELETE", f"/admin/eval-allowances/{user_id}/{competition_id}")
369        _raise_for_status(resp)
370        return bool(resp.json().get("deleted", False))
371
372
373class AdminEvalRunsResource:
374    """Submit + read model × challenge evaluation runs."""
375
376    def __init__(self, http: BaseHttpClient) -> None:
377        self._http = http
378
379    def artifacts(self) -> EvalArtifactReport:
380        """How much disk the collected agent archives are using.
381
382        ⚠️ **Read ``by_status`` beside the headline.** Zero stored bytes
383        is four different facts — nobody has run an agent, this
384        deployment has no bucket, collection is failing, or a retention
385        window reclaimed them — and they are identical as a single
386        number.
387        """
388        resp = self._http.request("GET", "/admin/eval-artifacts")
389        _raise_for_status(resp)
390        return EvalArtifactReport.model_validate(resp.json())
391
392    def list(
393        self,
394        *,
395        model_id: str = "",
396        competition_id: str = "",
397        status: str = "",
398        campaign_id: str = "",
399        offset: int = 0,
400        limit: int = 50,
401    ) -> PagedList[EvalRunState]:
402        resp = self._http.request(
403            "GET",
404            "/admin/eval-runs",
405            params={
406                "model_id": model_id,
407                "competition_id": competition_id,
408                "status": status,
409                "campaign_id": campaign_id,
410                "offset": offset,
411                "limit": limit,
412            },
413        )
414        _raise_for_status(resp)
415        return _extract_items(resp.json(), EvalRunState)
416
417    def get(self, eval_run_id: str) -> EvalRunState:
418        resp = self._http.request("GET", f"/admin/eval-runs/{eval_run_id}")
419        _raise_for_status(resp)
420        return EvalRunState.model_validate(resp.json())
421
422    def coverage(self, eval_run_id: str) -> EvalRunCoverage:
423        """How much of the range this run reached, question by question.
424
425        The other number beside the Reporting grade: that one says
426        whether what was written is true, this one says how much there
427        was to write about. Composed from the challenge's own spec, so it
428        moves when the range does.
429        """
430        resp = self._http.request("GET", f"/admin/eval-runs/{eval_run_id}/coverage")
431        _raise_for_status(resp)
432        return EvalRunCoverage.model_validate(resp.json())
433
434    def range_failure(self, eval_run_id: str) -> EvalRunRangeFailure:
435        """The ranges this run's attempts started, each joined to its
436        instance record: node, stop reason and the node's own error (#3233)."""
437        resp = self._http.request("GET", f"/admin/eval-runs/{eval_run_id}/range-failure")
438        _raise_for_status(resp)
439        return EvalRunRangeFailure.model_validate(resp.json())
440
441    def range_failures(
442        self, *, campaign_id: str = "", competition_id: str = "", since_ts: float = 0.0
443    ) -> EvalRangeFailureReport:
444        """``range_failed`` runs' attempts grouped by challenge × node (#3233)."""
445        resp = self._http.request(
446            "GET",
447            "/admin/eval-range-failures",
448            params={
449                "campaign_id": campaign_id,
450                "competition_id": competition_id,
451                "since_ts": since_ts,
452            },
453        )
454        _raise_for_status(resp)
455        return EvalRangeFailureReport.model_validate(resp.json())
456
457    def submit(
458        self,
459        *,
460        model_id: str,
461        challenge_id: str,
462        competition_id: str = "",
463        harness_id: str = "",
464        max_turns: int | None = None,
465        samples: int = 1,
466    ) -> AdminTaskInfo:
467        """Queue a ``model_eval_run`` task for one model × challenge. ``harness_id``
468        selects the agentic scaffold (``""`` = the built-in ctfy harness);
469        ``samples`` > 1 queues k runs (pass@k). Returns the first pending task;
470        poll :meth:`list` for the resulting run records."""
471        body: dict[str, Any] = {
472            "model_id": model_id,
473            "challenge_id": challenge_id,
474            "competition_id": competition_id,
475            "harness_id": harness_id,
476            "samples": samples,
477        }
478        if max_turns is not None:
479            body["max_turns"] = max_turns
480        resp = self._http.request("POST", "/admin/eval-runs", json=body)
481        _raise_for_status(resp)
482        return AdminTaskInfo.model_validate(resp.json())
483
484
485class AdminEvalCampaignsResource:
486    """Submit + read operator batch evaluations (a model over a standard set)."""
487
488    def __init__(self, http: BaseHttpClient) -> None:
489        self._http = http
490
491    def list(
492        self,
493        *,
494        model_id: str = "",
495        vendor_id: str = "",
496        status: str = "",
497        tier: str = "",
498        offset: int = 0,
499        limit: int = 50,
500    ) -> PagedList[EvalCampaignState]:
501        resp = self._http.request(
502            "GET",
503            "/admin/eval-campaigns",
504            params={
505                "model_id": model_id,
506                "vendor_id": vendor_id,
507                "status": status,
508                "tier": tier,
509                "offset": offset,
510                "limit": limit,
511            },
512        )
513        _raise_for_status(resp)
514        return _extract_items(resp.json(), EvalCampaignState)
515
516    def get(self, eval_campaign_id: str) -> EvalCampaignState:
517        resp = self._http.request("GET", f"/admin/eval-campaigns/{eval_campaign_id}")
518        _raise_for_status(resp)
519        return EvalCampaignState.model_validate(resp.json())
520
521    def submit(
522        self,
523        *,
524        model_id: str,
525        standard_set_id: str,
526        name: str = "",
527        harness_id: str = "",
528        num_samples: int = 1,
529        max_cost_usd: float = 0.0,
530        timeout_s: int = 0,
531        max_turns: int = 0,
532    ) -> AdminTaskInfo:
533        """Queue an ``eval_campaign_run`` fan-out over a standard set. The model
534        must be ``approved``; the standard set (a competition id) must carry
535        challenges. Returns the driving task; poll :meth:`get` for the rollup."""
536        resp = self._http.request(
537            "POST",
538            "/admin/eval-campaigns",
539            json={
540                "model_id": model_id,
541                "standard_set_id": standard_set_id,
542                "name": name,
543                "harness_id": harness_id,
544                "num_samples": num_samples,
545                "max_cost_usd": max_cost_usd,
546                "timeout_s": timeout_s,
547                "max_turns": max_turns,
548            },
549        )
550        _raise_for_status(resp)
551        return AdminTaskInfo.model_validate(resp.json())
552
553    def cancel(self, eval_campaign_id: str) -> EvalCampaignState:
554        """Cooperatively cancel a campaign via its driving task."""
555        resp = self._http.request("POST", f"/admin/eval-campaigns/{eval_campaign_id}/cancel")
556        _raise_for_status(resp)
557        return EvalCampaignState.model_validate(resp.json())
558
559
560class AdminRegistryResource:
561    """The registry this deployment runs: who holds what, and reclaiming it.
562
563    ⚠️ **There is deliberately no automatic GC**, so this is the whole
564    reclaim surface. ``approved_digest`` is what a dispatch *pulls*, so a
565    rule like "keep the newest K tags" would delete the image a campaign
566    that has not started yet will ask for — and the failure lands hours
567    later on somebody else's evaluation as an unexplained pull error.
568    """
569
570    def __init__(self, http: BaseHttpClient) -> None:
571        self._http = http
572
573    def usage(self) -> RegistryUsageReport:
574        """What each account holds, against the ceiling it is judged by.
575
576        ⚠️ ``over_quota`` on each row is the **server's** verdict, from
577        the same predicate that refuses a push. Do not recompute it from
578        ``bytes``/``tags``: a second opinion is the one that eventually
579        disagrees with the one that decides.
580        """
581        resp = self._http.request("GET", "/admin/registry/usage")
582        _raise_for_status(resp)
583        return RegistryUsageReport.model_validate(resp.json())
584
585    def delete_manifest(self, repository: str, digest: str) -> str:
586        """Remove one manifest, and every tag pointing at it.
587
588        ⚠️ **Refused (409 ``registry_digest_approved``) when a harness is
589        approved at that digest**, naming the harnesses — a dispatch
590        pulls by digest, so removing it breaks runs that have not
591        started yet. Un-approve or re-approve those agents first.
592        """
593        resp = self._http.request(
594            "DELETE",
595            "/admin/registry/manifests",
596            params={"repository": repository, "digest": digest},
597        )
598        _raise_for_status(resp)
599        return str(resp.json().get("digest", ""))
600
601    def repos(self) -> RegistryRepoList:
602        """Every repository, with its tag count and kind — the browse list."""
603        resp = self._http.request("GET", "/admin/registry/repos")
604        _raise_for_status(resp)
605        return RegistryRepoList.model_validate(resp.json())
606
607    def tags(self, repository: str, *, limit: int = 500) -> RegistryTagList:
608        """One repository's tags — size, push time, digest, references."""
609        resp = self._http.request(
610            "GET",
611            "/admin/registry/tags",
612            params={"repository": repository, "limit": limit},
613        )
614        _raise_for_status(resp)
615        return RegistryTagList.model_validate(resp.json())
616
617    def reclaim(
618        self, policy: RegistryReclaimPolicy | str, *, dry_run: bool = True
619    ) -> RegistryReclaimPlan:
620        """Preview (``dry_run=True``) or run a reference-checked reclaim.
621
622        ⚠️ Dry run is the default; only ``dry_run=False`` deletes, and it
623        never removes a manifest a live instance, published competition or
624        harness pin references.
625        """
626        resp = self._http.request(
627            "POST",
628            "/admin/registry/reclaim",
629            json={"policy": str(policy), "dry_run": dry_run},
630        )
631        _raise_for_status(resp)
632        return RegistryReclaimPlan.model_validate(resp.json())
633
634    def gc(self) -> RegistryGcStatus:
635        """Probe the registry and preview what garbage collection would free."""
636        resp = self._http.request("GET", "/admin/registry/gc")
637        _raise_for_status(resp)
638        return RegistryGcStatus.model_validate(resp.json())