ctfy.sdk.admin_resources.eval

  1"""``client.admin.llm_models`` + ``client.admin.eval_runs`` — model evaluation (admin)."""
  2
  3from __future__ import annotations
  4
  5import builtins
  6from typing import Any
  7
  8from ctfy.core.enums import RegistryReclaimPolicy
  9from ctfy.core.state.models import (
 10    EvalCampaignState,
 11    EvalRunState,
 12    HarnessState,
 13    LlmModelState,
 14    VendorState,
 15    VendorTokenState,
 16)
 17from ctfy.sdk._helpers import PagedList, _extract_items, _raise_for_status
 18from ctfy.sdk.base import BaseHttpClient
 19from ctfy.server.models import (
 20    AdminHarnessRow,
 21    AdminTaskInfo,
 22    EvalAllowanceRow,
 23    EvalArtifactReport,
 24    EvalRunCoverage,
 25    RegistryGcStatus,
 26    RegistryReclaimPlan,
 27    RegistryRepoList,
 28    RegistryTagList,
 29    RegistryUsageReport,
 30)
 31
 32
 33class AdminLlmModelsResource:
 34    """The registry of models under evaluation: register / list / delete.
 35
 36    No API key is ever stored — a registered model is addressing only
 37    (flavor + base_url + provider model id); credentials are supplied to the
 38    run at dispatch time via env.
 39    """
 40
 41    def __init__(self, http: BaseHttpClient) -> None:
 42        self._http = http
 43
 44    def list(
 45        self, *, active_only: bool = False, offset: int = 0, limit: int = 100
 46    ) -> PagedList[LlmModelState]:
 47        resp = self._http.request(
 48            "GET",
 49            "/admin/llm-models",
 50            params={"active_only": active_only, "offset": offset, "limit": limit},
 51        )
 52        _raise_for_status(resp)
 53        return _extract_items(resp.json(), LlmModelState)
 54
 55    def register(
 56        self,
 57        *,
 58        id: str,
 59        flavor: str,
 60        display_name: str = "",
 61        provider_model: str = "",
 62        base_url: str = "",
 63        vendor: str = "",
 64        notes: str = "",
 65        input_price_per_mtok: float = 0.0,
 66        output_price_per_mtok: float = 0.0,
 67    ) -> LlmModelState:
 68        """Register (or overwrite) a model. ``flavor`` is ``anthropic`` / ``openai``.
 69
 70        ``*_price_per_mtok`` is USD per 1M tokens (0 = unpriced) — drives the
 71        leaderboard's cost columns.
 72        """
 73        resp = self._http.request(
 74            "POST",
 75            "/admin/llm-models",
 76            json={
 77                "id": id,
 78                "flavor": flavor,
 79                "display_name": display_name,
 80                "provider_model": provider_model,
 81                "base_url": base_url,
 82                "vendor": vendor,
 83                "notes": notes,
 84                "input_price_per_mtok": input_price_per_mtok,
 85                "output_price_per_mtok": output_price_per_mtok,
 86            },
 87        )
 88        _raise_for_status(resp)
 89        return LlmModelState.model_validate(resp.json())
 90
 91    def delete(self, model_id: str) -> bool:
 92        resp = self._http.request("DELETE", f"/admin/llm-models/{model_id}")
 93        _raise_for_status(resp)
 94        return bool(resp.json().get("deleted"))
 95
 96    def mint_token(
 97        self,
 98        model_id: str,
 99        *,
100        competition_id: str,
101        label: str = "",
102        expires_in_days: int | None = None,
103    ) -> dict[str, Any]:
104        """Mint a play credential bound to this model's synthetic team.
105
106        Returns ``{token, model_id, competition_id, team_id,
107        user_id}``; the plaintext ``pf_`` bearer is shown **once** —
108        store it now, only its hash is persisted. Scoped to
109        ``competition_id`` and carrying only what a run needs
110        (``challenges: read`` + ``instances: write`` + ``submissions:
111        write``).
112
113        This is what an *external* harness — one the platform does not
114        dispatch itself — uses to keep each model on its own team.
115        Sharing a team makes ``challenge_fully_solved`` inherit between
116        models and makes them share one wrong-answer budget, silently.
117        """
118        resp = self._http.request(
119            "POST",
120            f"/admin/llm-models/{model_id}/tokens",
121            json={
122                "competition_id": competition_id,
123                "label": label,
124                "expires_in_days": expires_in_days,
125            },
126        )
127        _raise_for_status(resp)
128        data: dict[str, Any] = resp.json()
129        return data
130
131    def review(self, model_id: str, *, status: str, note: str = "") -> LlmModelState:
132        """Approve / reject a model. ``status`` is ``approved`` / ``rejected`` /
133        ``pending``; only ``approved`` models may back an official campaign."""
134        resp = self._http.request(
135            "POST",
136            f"/admin/llm-models/{model_id}/review",
137            json={"status": status, "note": note},
138        )
139        _raise_for_status(resp)
140        return LlmModelState.model_validate(resp.json())
141
142
143class AdminEvalVendorsResource:
144    """The registry of model vendors / orgs: register / list / delete.
145
146    A vendor's ``id`` is a slug matching the ``vendor`` label on each of its
147    registered models, so the leaderboard's vendor scorecard resolves display
148    metadata from one curated row.
149    """
150
151    def __init__(self, http: BaseHttpClient) -> None:
152        self._http = http
153
154    def list(
155        self, *, active_only: bool = False, offset: int = 0, limit: int = 100
156    ) -> PagedList[VendorState]:
157        resp = self._http.request(
158            "GET",
159            "/admin/eval-vendors",
160            params={"active_only": active_only, "offset": offset, "limit": limit},
161        )
162        _raise_for_status(resp)
163        return _extract_items(resp.json(), VendorState)
164
165    def register(
166        self,
167        *,
168        id: str,
169        display_name: str = "",
170        homepage: str = "",
171        contact_email: str = "",
172        active: bool = True,
173        notes: str = "",
174        max_concurrent_campaigns: int = 1,
175        daily_run_quota: int = 200,
176        monthly_cost_budget_usd: float = 0.0,
177    ) -> VendorState:
178        """Register (or overwrite) a vendor by slug ``id``.
179
180        The quota trio caps the vendor's self-service campaign usage
181        (docs/model-eval-vendor-tenancy.md §6.2); ``0`` on any axis means "no
182        limit" there. A re-register overwrites the whole row, so pass the
183        current quota values to keep them."""
184        resp = self._http.request(
185            "POST",
186            "/admin/eval-vendors",
187            json={
188                "id": id,
189                "display_name": display_name,
190                "homepage": homepage,
191                "contact_email": contact_email,
192                "active": active,
193                "notes": notes,
194                "max_concurrent_campaigns": max_concurrent_campaigns,
195                "daily_run_quota": daily_run_quota,
196                "monthly_cost_budget_usd": monthly_cost_budget_usd,
197            },
198        )
199        _raise_for_status(resp)
200        return VendorState.model_validate(resp.json())
201
202    def delete(self, vendor_id: str) -> bool:
203        resp = self._http.request("DELETE", f"/admin/eval-vendors/{vendor_id}")
204        _raise_for_status(resp)
205        return bool(resp.json().get("deleted"))
206
207    # -- vendor access tokens (tenancy) -------------------------------------
208
209    def mint_token(self, vendor_id: str, *, label: str = "", scope: str = "read") -> dict[str, Any]:
210        """Mint a ``pv_`` vendor bearer. The plaintext ``token`` in the reply
211        is shown **once** — store it now; only its hash is persisted. ``scope``
212        is ``read`` (default) or ``submit`` (self-service writes)."""
213        resp = self._http.request(
214            "POST",
215            f"/admin/eval-vendors/{vendor_id}/tokens",
216            json={"label": label, "scope": scope},
217        )
218        _raise_for_status(resp)
219        data: dict[str, Any] = resp.json()
220        return data
221
222    def list_tokens(self, vendor_id: str) -> builtins.list[VendorTokenState]:
223        """List a vendor's tokens (hashes blanked server-side)."""
224        resp = self._http.request("GET", f"/admin/eval-vendors/{vendor_id}/tokens")
225        _raise_for_status(resp)
226        return [VendorTokenState.model_validate(row) for row in resp.json()]
227
228    def revoke_token(self, vendor_id: str, token_id: str) -> bool:
229        resp = self._http.request("DELETE", f"/admin/eval-vendors/{vendor_id}/tokens/{token_id}")
230        _raise_for_status(resp)
231        return bool(resp.json().get("deleted"))
232
233
234class AdminEvalHarnessesResource:
235    """The registry of eval harnesses (agentic scaffolds): register / list / delete.
236
237    A harness's ``id`` is a slug (``ctfy`` / ``claude-code`` / …); ``image`` is
238    the container a node runs for it. Lets harness be a comparison variable
239    alongside the model.
240    """
241
242    def __init__(self, http: BaseHttpClient) -> None:
243        self._http = http
244
245    def list(
246        self, *, active_only: bool = False, offset: int = 0, limit: int = 100
247    ) -> PagedList[AdminHarnessRow]:
248        resp = self._http.request(
249            "GET",
250            "/admin/eval-harnesses",
251            params={"active_only": active_only, "offset": offset, "limit": limit},
252        )
253        _raise_for_status(resp)
254        return _extract_items(resp.json(), AdminHarnessRow)
255
256    def register(
257        self,
258        *,
259        id: str,
260        image: str = "",
261        display_name: str = "",
262        version: str = "",
263        notes: str = "",
264        active: bool = True,
265    ) -> HarnessState:
266        """Register (or overwrite) one of the *platform's* harnesses.
267
268        ⚠️ Refused with 409 on a row a user owns. The stamping here says
269        "the platform published this" — ``owner_user_id: ""`` is exactly
270        what the dispatch gate reads as needing no review — so aiming it
271        at somebody's registration would strip the owner and grant a
272        permanent exemption while reading like an approval. Use
273        :meth:`review` for a verdict.
274        """
275        resp = self._http.request(
276            "POST",
277            "/admin/eval-harnesses",
278            json={
279                "id": id,
280                "image": image,
281                "display_name": display_name,
282                "version": version,
283                "notes": notes,
284                "active": active,
285            },
286        )
287        _raise_for_status(resp)
288        return HarnessState.model_validate(resp.json())
289
290    def review(self, harness_id: str, status: str, *, note: str = "") -> HarnessState:
291        """Decide whether a user's agent image may run.
292
293        ``status`` is ``approved`` / ``rejected`` / ``pending``; a
294        verdict can be taken back. Never touches ``owner_user_id`` — the
295        field the gate turns on — so the decision stays about this
296        image rather than becoming a permanent exemption.
297        """
298        resp = self._http.request(
299            "POST",
300            f"/admin/eval-harnesses/{harness_id}/review",
301            json={"status": status, "note": note},
302        )
303        _raise_for_status(resp)
304        return HarnessState.model_validate(resp.json())
305
306    def check(self, harness_id: str) -> AdminHarnessRow:
307        """Look at the image now and record what was found.
308
309        ⚠️ **Evidence, never a gate.** It does not touch
310        ``review_status``, and a ``failed`` verdict does not stop a later
311        :meth:`review` approving the row — an operator may know something
312        the check does not.
313
314        ⚠️ **A registry that would not answer comes back 200 carrying
315        ``conformance_status == "error"``**, which is a recorded answer
316        about the registry rather than a claim about the image. Only a
317        row naming no image refuses (400 ``harness_has_no_image``):
318        "there is nothing to check" and "we could not look" send the
319        reader to different places.
320        """
321        resp = self._http.request("POST", f"/admin/eval-harnesses/{harness_id}/conformance")
322        _raise_for_status(resp)
323        return AdminHarnessRow.model_validate(resp.json())
324
325    def probe(self, harness_id: str) -> AdminTaskInfo:
326        """Run the image once, under the profile a dispatch uses.
327
328        ⚠️ **Answers a queued task, never a verdict.** The image is
329        pulled before the container starts, so the answer lands on the
330        harness row minutes later — poll :meth:`list`, not this return
331        value.
332        """
333        resp = self._http.request("POST", f"/admin/eval-harnesses/{harness_id}/probe")
334        _raise_for_status(resp)
335        return AdminTaskInfo.model_validate(resp.json())
336
337    def delete(self, harness_id: str) -> bool:
338        resp = self._http.request("DELETE", f"/admin/eval-harnesses/{harness_id}")
339        _raise_for_status(resp)
340        return bool(resp.json().get("deleted"))
341
342
343class AdminEvalAllowancesResource:
344    """Grants of the platform's own model key: list / grant / revoke.
345
346    ⚠️ This is the one custody mode that spends the deployment's money on
347    somebody else's behalf, so it is an explicit grant per
348    ``(account, competition)`` rather than a fall-through from "the user
349    has no key". Re-granting keeps what was already spent.
350    """
351
352    def __init__(self, http: BaseHttpClient) -> None:
353        self._http = http
354
355    def list(self, *, competition_id: str = "") -> builtins.list[EvalAllowanceRow]:
356        """Every grant, or one competition's. ``""`` means every one."""
357        resp = self._http.request(
358            "GET", "/admin/eval-allowances", params={"competition_id": competition_id}
359        )
360        _raise_for_status(resp)
361        return [EvalAllowanceRow.model_validate(row) for row in resp.json()]
362
363    def grant(
364        self, user_id: str, competition_id: str, cap_usd: float, *, note: str = ""
365    ) -> EvalAllowanceRow:
366        """Let this account spend the platform's key, up to ``cap_usd``.
367
368        A repeat ``(account, competition)`` re-caps and **keeps the
369        spend** — raising a cap does not refund what went before.
370        """
371        resp = self._http.request(
372            "POST",
373            "/admin/eval-allowances",
374            json={
375                "user_id": user_id,
376                "competition_id": competition_id,
377                "cap_usd": cap_usd,
378                "note": note,
379            },
380        )
381        _raise_for_status(resp)
382        return EvalAllowanceRow.model_validate(resp.json())
383
384    def revoke(self, user_id: str, competition_id: str) -> bool:
385        """Take it back. Runs already paid for stay paid for."""
386        resp = self._http.request("DELETE", f"/admin/eval-allowances/{user_id}/{competition_id}")
387        _raise_for_status(resp)
388        return bool(resp.json().get("deleted", False))
389
390
391class AdminEvalRunsResource:
392    """Submit + read model × challenge evaluation runs."""
393
394    def __init__(self, http: BaseHttpClient) -> None:
395        self._http = http
396
397    def artifacts(self) -> EvalArtifactReport:
398        """How much disk the collected agent archives are using.
399
400        ⚠️ **Read ``by_status`` beside the headline.** Zero stored bytes
401        is four different facts — nobody has run an agent, this
402        deployment has no bucket, collection is failing, or a retention
403        window reclaimed them — and they are identical as a single
404        number.
405        """
406        resp = self._http.request("GET", "/admin/eval-artifacts")
407        _raise_for_status(resp)
408        return EvalArtifactReport.model_validate(resp.json())
409
410    def list(
411        self,
412        *,
413        model_id: str = "",
414        competition_id: str = "",
415        status: str = "",
416        campaign_id: str = "",
417        offset: int = 0,
418        limit: int = 50,
419    ) -> PagedList[EvalRunState]:
420        resp = self._http.request(
421            "GET",
422            "/admin/eval-runs",
423            params={
424                "model_id": model_id,
425                "competition_id": competition_id,
426                "status": status,
427                "campaign_id": campaign_id,
428                "offset": offset,
429                "limit": limit,
430            },
431        )
432        _raise_for_status(resp)
433        return _extract_items(resp.json(), EvalRunState)
434
435    def get(self, eval_run_id: str) -> EvalRunState:
436        resp = self._http.request("GET", f"/admin/eval-runs/{eval_run_id}")
437        _raise_for_status(resp)
438        return EvalRunState.model_validate(resp.json())
439
440    def coverage(self, eval_run_id: str) -> EvalRunCoverage:
441        """How much of the range this run reached, question by question.
442
443        The other number beside the Reporting grade: that one says
444        whether what was written is true, this one says how much there
445        was to write about. Composed from the challenge's own spec, so it
446        moves when the range does.
447        """
448        resp = self._http.request("GET", f"/admin/eval-runs/{eval_run_id}/coverage")
449        _raise_for_status(resp)
450        return EvalRunCoverage.model_validate(resp.json())
451
452    def submit(
453        self,
454        *,
455        model_id: str,
456        challenge_id: str,
457        competition_id: str = "",
458        harness_id: str = "",
459        max_turns: int | None = None,
460        samples: int = 1,
461    ) -> AdminTaskInfo:
462        """Queue a ``model_eval_run`` task for one model × challenge. ``harness_id``
463        selects the agentic scaffold (``""`` = the built-in ctfy harness);
464        ``samples`` > 1 queues k runs (pass@k). Returns the first pending task;
465        poll :meth:`list` for the resulting run records."""
466        body: dict[str, Any] = {
467            "model_id": model_id,
468            "challenge_id": challenge_id,
469            "competition_id": competition_id,
470            "harness_id": harness_id,
471            "samples": samples,
472        }
473        if max_turns is not None:
474            body["max_turns"] = max_turns
475        resp = self._http.request("POST", "/admin/eval-runs", json=body)
476        _raise_for_status(resp)
477        return AdminTaskInfo.model_validate(resp.json())
478
479
480class AdminEvalCampaignsResource:
481    """Submit + read operator batch evaluations (a model over a standard set)."""
482
483    def __init__(self, http: BaseHttpClient) -> None:
484        self._http = http
485
486    def list(
487        self,
488        *,
489        model_id: str = "",
490        vendor_id: str = "",
491        status: str = "",
492        tier: str = "",
493        offset: int = 0,
494        limit: int = 50,
495    ) -> PagedList[EvalCampaignState]:
496        resp = self._http.request(
497            "GET",
498            "/admin/eval-campaigns",
499            params={
500                "model_id": model_id,
501                "vendor_id": vendor_id,
502                "status": status,
503                "tier": tier,
504                "offset": offset,
505                "limit": limit,
506            },
507        )
508        _raise_for_status(resp)
509        return _extract_items(resp.json(), EvalCampaignState)
510
511    def get(self, eval_campaign_id: str) -> EvalCampaignState:
512        resp = self._http.request("GET", f"/admin/eval-campaigns/{eval_campaign_id}")
513        _raise_for_status(resp)
514        return EvalCampaignState.model_validate(resp.json())
515
516    def submit(
517        self,
518        *,
519        model_id: str,
520        standard_set_id: str,
521        name: str = "",
522        harness_id: str = "",
523        num_samples: int = 1,
524        max_cost_usd: float = 0.0,
525        timeout_s: int = 0,
526        max_turns: int = 0,
527    ) -> AdminTaskInfo:
528        """Queue an ``eval_campaign_run`` fan-out over a standard set. The model
529        must be ``approved``; the standard set (a competition id) must carry
530        challenges. Returns the driving task; poll :meth:`get` for the rollup."""
531        resp = self._http.request(
532            "POST",
533            "/admin/eval-campaigns",
534            json={
535                "model_id": model_id,
536                "standard_set_id": standard_set_id,
537                "name": name,
538                "harness_id": harness_id,
539                "num_samples": num_samples,
540                "max_cost_usd": max_cost_usd,
541                "timeout_s": timeout_s,
542                "max_turns": max_turns,
543            },
544        )
545        _raise_for_status(resp)
546        return AdminTaskInfo.model_validate(resp.json())
547
548    def cancel(self, eval_campaign_id: str) -> EvalCampaignState:
549        """Cooperatively cancel a campaign via its driving task."""
550        resp = self._http.request("POST", f"/admin/eval-campaigns/{eval_campaign_id}/cancel")
551        _raise_for_status(resp)
552        return EvalCampaignState.model_validate(resp.json())
553
554
555class AdminRegistryResource:
556    """The registry this deployment runs: who holds what, and reclaiming it.
557
558    ⚠️ **There is deliberately no automatic GC**, so this is the whole
559    reclaim surface. ``approved_digest`` is what a dispatch *pulls*, so a
560    rule like "keep the newest K tags" would delete the image a campaign
561    that has not started yet will ask for — and the failure lands hours
562    later on somebody else's evaluation as an unexplained pull error.
563    """
564
565    def __init__(self, http: BaseHttpClient) -> None:
566        self._http = http
567
568    def usage(self) -> RegistryUsageReport:
569        """What each account holds, against the ceiling it is judged by.
570
571        ⚠️ ``over_quota`` on each row is the **server's** verdict, from
572        the same predicate that refuses a push. Do not recompute it from
573        ``bytes``/``tags``: a second opinion is the one that eventually
574        disagrees with the one that decides.
575        """
576        resp = self._http.request("GET", "/admin/registry/usage")
577        _raise_for_status(resp)
578        return RegistryUsageReport.model_validate(resp.json())
579
580    def delete_manifest(self, repository: str, digest: str) -> str:
581        """Remove one manifest, and every tag pointing at it.
582
583        ⚠️ **Refused (409 ``registry_digest_approved``) when a harness is
584        approved at that digest**, naming the harnesses — a dispatch
585        pulls by digest, so removing it breaks runs that have not
586        started yet. Un-approve or re-approve those agents first.
587        """
588        resp = self._http.request(
589            "DELETE",
590            "/admin/registry/manifests",
591            params={"repository": repository, "digest": digest},
592        )
593        _raise_for_status(resp)
594        return str(resp.json().get("digest", ""))
595
596    def repos(self) -> RegistryRepoList:
597        """Every repository, with its tag count and kind — the browse list."""
598        resp = self._http.request("GET", "/admin/registry/repos")
599        _raise_for_status(resp)
600        return RegistryRepoList.model_validate(resp.json())
601
602    def tags(self, repository: str, *, limit: int = 500) -> RegistryTagList:
603        """One repository's tags — size, push time, digest, references."""
604        resp = self._http.request(
605            "GET",
606            "/admin/registry/tags",
607            params={"repository": repository, "limit": limit},
608        )
609        _raise_for_status(resp)
610        return RegistryTagList.model_validate(resp.json())
611
612    def reclaim(
613        self, policy: RegistryReclaimPolicy | str, *, dry_run: bool = True
614    ) -> RegistryReclaimPlan:
615        """Preview (``dry_run=True``) or run a reference-checked reclaim.
616
617        ⚠️ Dry run is the default; only ``dry_run=False`` deletes, and it
618        never removes a manifest a live instance, published competition or
619        harness pin references.
620        """
621        resp = self._http.request(
622            "POST",
623            "/admin/registry/reclaim",
624            json={"policy": str(policy), "dry_run": dry_run},
625        )
626        _raise_for_status(resp)
627        return RegistryReclaimPlan.model_validate(resp.json())
628
629    def gc(self) -> RegistryGcStatus:
630        """Probe the registry and preview what garbage collection would free."""
631        resp = self._http.request("GET", "/admin/registry/gc")
632        _raise_for_status(resp)
633        return RegistryGcStatus.model_validate(resp.json())