1"""``client.eval`` — the public leaderboard, plus your own agent shelf."""
  2
  3from __future__ import annotations
  4
  5from collections.abc import Sequence
  6from typing import Any
  7
  8from ctfy.core.eval_atomic import CancelMode
  9from ctfy.core.state.models import (
 10    EvalCampaignState,
 11    EvalFundingSource,
 12    HarnessState,
 13    HarnessVersionState,
 14)
 15from ctfy.sdk._helpers import PagedList, _extract_items, _raise_for_status
 16from ctfy.sdk.base import BaseHttpClient
 17from ctfy.server.models import (
 18    AgentLogPage,
 19    EvalAllowanceRow,
 20    EvalBatchCancelResult,
 21    EvalBatchLimits,
 22    EvalBatchModel,
 23    EvalBatchRequest,
 24    EvalBatchState,
 25    EvalCampaignAtomicSummary,
 26    EvalCancelRequest,
 27    EvalLeaderboard,
 28    EvalQueueChange,
 29    EvalRunEventList,
 30    HarnessTags,
 31    InstanceTrafficPage,
 32    LlmAccountChoice,
 33    LlmBalanceRefresh,
 34    LlmCallBody,
 35    LlmCallPage,
 36    LlmConnectionRef,
 37    LlmLivePage,
 38    LlmModelOption,
 39    LlmProviderChoice,
 40    LlmSpendCapUpdate,
 41    LlmUsagePage,
 42    MyEvalRunPage,
 43    MyEvalRunRow,
 44    MyHarness,
 45    MyLlmConnection,
 46    MyLlmConnectionCreate,
 47    MyLlmKey,
 48    MyLlmKeyProbe,
 49    RegistryPushTarget,
 50    RunWriteup,
 51)
 52
 53
 54class EvalResource:
 55    """The public model leaderboard (no auth), and the signed-in caller's
 56    own harness registry (``/eval/harnesses``)."""
 57
 58    def __init__(self, http: BaseHttpClient) -> None:
 59        self._http = http
 60
 61    def leaderboard(
 62        self,
 63        competition_id: str,
 64        *,
 65        tier: str = "official",
 66        corpus_sha: str = "",
 67        q: str = "",
 68        vendor: str = "",
 69        harness_id: str | None = None,
 70        offset: int = 0,
 71        limit: int | None = None,
 72    ) -> EvalLeaderboard:
 73        """Ranked model standings + per-dimension (difficulty / bucket / tag)
 74        breakdowns for **one** competition or suite. There is no platform-wide
 75        board (#2341): runs graded under different sets do not rank together.
 76
 77        ``tier`` selects the credibility band: ``official`` (default,
 78        platform-run), ``self_reported`` (vendor-run, unverified), or ``all``
 79        for both. Sandbox runs are never exposed here. ``corpus_sha`` counts
 80        only runs graded against that version (a suite's pin); empty is every
 81        version.
 82
 83        #2733: ``harness_id`` scopes the aggregate to one harness (``""`` =
 84        the built-in one; ``None`` = every harness) and ranks inside it.
 85        ``q`` (model search), ``vendor`` (slug), ``offset`` and ``limit``
 86        pick rows out of the scope without re-ranking; ``result.scope``
 87        reports the revision, manifest, freshness, filters and totals."""
 88        if not competition_id:
 89            raise ValueError("competition_id is required: there is no platform-wide board")
 90        params: dict[str, str | int] = {
 91            "competition_id": competition_id,
 92            "tier": tier,
 93            "corpus_sha": corpus_sha,
 94            "q": q,
 95            "vendor": vendor,
 96            "offset": offset,
 97        }
 98        # ⚠️ ``""`` is a real harness (the built-in one), so only ``None``
 99        # leaves the parameter out.
100        if harness_id is not None:
101            params["harness_id"] = harness_id
102        if limit is not None:
103            params["limit"] = limit
104        resp = self._http.request("GET", "/eval/leaderboard", params=params)
105        _raise_for_status(resp)
106        return EvalLeaderboard.model_validate(resp.json())
107
108    # -- your own agent images -------------------------------------------
109
110    def harnesses(self) -> list[HarnessState]:
111        """The harnesses *you* registered, newest first.
112
113        Scoped to the caller — the platform's own images and other
114        users' are not listed here (``/admin/eval-harnesses`` is the
115        operator's whole-registry view).
116        """
117        resp = self._http.request("GET", "/eval/harnesses")
118        _raise_for_status(resp)
119        return [HarnessState.model_validate(row) for row in resp.json()]
120
121    def harness_versions(self, harness_id: str) -> list[HarnessVersionState]:
122        """Your agent's saved immutable releases, newest first."""
123        resp = self._http.request("GET", f"/eval/harnesses/{harness_id}/versions")
124        _raise_for_status(resp)
125        return [HarnessVersionState.model_validate(row) for row in resp.json()]
126
127    def create_harness_version(self, harness_id: str) -> HarnessVersionState:
128        """Save the currently reviewed image; the first release becomes default."""
129        resp = self._http.request("POST", f"/eval/harnesses/{harness_id}/versions")
130        _raise_for_status(resp)
131        return HarnessVersionState.model_validate(resp.json())
132
133    def register_harness(
134        self,
135        harness_id: str,
136        image: str,
137        *,
138        display_name: str = "",
139        version: str = "",
140        notes: str = "",
141        dialect: str = "",
142    ) -> HarnessState:
143        """Register one of your own agent images.
144
145        ⚠️ The returned row is ``review_status="pending"`` and is **not
146        dispatchable** until an operator approves it — registering an
147        image and being trusted to run one are deliberately two things.
148        ``owner_user_id`` and ``review_status`` are stamped by the
149        server; anything you pass for them is ignored.
150        """
151        resp = self._http.request(
152            "POST",
153            "/eval/harnesses",
154            json={
155                "id": harness_id,
156                "image": image,
157                "display_name": display_name,
158                "version": version,
159                "notes": notes,
160                # #3473: the wire surface the agent calls; "" is not declared.
161                "dialect": dialect,
162            },
163        )
164        _raise_for_status(resp)
165        return HarnessState.model_validate(resp.json())
166
167    def update_harness(
168        self,
169        harness_id: str,
170        *,
171        image: str | None = None,
172        display_name: str | None = None,
173        version: str | None = None,
174        notes: str | None = None,
175        source_url: str | None = None,
176        active: bool | None = None,
177        default_version_id: str | None = None,
178        dialect: str | None = None,
179    ) -> HarnessState:
180        """Edit one of your own. Anything left ``None`` is untouched.
181
182        ⚠️ **Changing ``image`` puts the row back to
183        ``review_status="pending"``** and clears the approved digest —
184        the verdict was about specific content, and a dispatch pulls the
185        digest rather than the tag. Carrying the approval across would
186        leave the row naming a new reference while the node kept running
187        the old bytes. Re-sending the *same* image is a no-op and costs
188        nothing.
189
190        Anything that is not yours answers 404 rather than 403, like
191        :meth:`delete_harness`.
192        """
193        # ⚠️ Absence is what "leave it alone" means on the wire, so an
194        # unset argument must not appear in the body at all — the server
195        # reads ``model_fields_set``, and a key sent as ``null`` is a key
196        # that was sent.
197        body: dict[str, object] = {
198            name: value
199            for name, value in (
200                ("image", image),
201                ("display_name", display_name),
202                ("version", version),
203                ("notes", notes),
204                ("source_url", source_url),
205                ("active", active),
206                ("default_version_id", default_version_id),
207                ("dialect", dialect),
208            )
209            if value is not None
210        }
211        resp = self._http.request("PATCH", f"/eval/harnesses/{harness_id}", json=body)
212        _raise_for_status(resp)
213        return HarnessState.model_validate(resp.json())
214
215    def delete_harness(self, harness_id: str) -> bool:
216        """Remove one of your own. Anything that is not yours — another
217        user's, or a platform image — answers 404 rather than 403, so
218        this cannot be used to probe the registry."""
219        resp = self._http.request("DELETE", f"/eval/harnesses/{harness_id}")
220        _raise_for_status(resp)
221        return bool(resp.json().get("deleted", False))
222
223    def llm_providers(self) -> list[LlmProviderChoice]:
224        """The upstreams this deployment will relay to.
225
226        ⚠️ **Ask rather than assume.** The gateway makes the request to
227        whatever address a key names — from inside its own network, and
228        it returns the answer — so the set is closed and an operator may
229        replace it. A caller carrying its own list would offer addresses
230        this deployment refuses.
231
232        Each entry carries its dialect: a provider *is* a dialect plus an
233        address, and DeepSeek, Moonshot and OpenAI all speak ``openai``.
234        """
235        resp = self._http.request("GET", "/eval/llm-providers")
236        _raise_for_status(resp)
237        return [LlmProviderChoice.model_validate(r) for r in resp.json()]
238
239    def my_llm_keys(self) -> list[MyLlmKey]:
240        """Your own upstream credentials — what they are, never what they say.
241
242        ⚠️ There is no ``api_key`` on the way back, by design. A key is
243        pasted once; every read after that lets you tell two apart, not
244        copy one.
245        """
246        resp = self._http.request("GET", "/eval/llm-keys")
247        _raise_for_status(resp)
248        return [MyLlmKey.model_validate(r) for r in resp.json()]
249
250    def put_my_llm_key(
251        self, flavor: str, api_key: str, *, label: str = "", base_url: str = ""
252    ) -> MyLlmKey:
253        """Store or rotate the credential this account spends for ``flavor``.
254
255        A repeat dialect replaces: rotation is a write, because
256        delete-then-add leaves a window in which a run has no key.
257        Answers 503 on a deployment that cannot seal secrets at rest —
258        the key is never stored in the clear.
259        """
260        resp = self._http.request(
261            "PUT",
262            f"/eval/llm-keys/{flavor}",
263            json={"api_key": api_key, "label": label, "base_url": base_url},
264        )
265        _raise_for_status(resp)
266        return MyLlmKey.model_validate(resp.json())
267
268    def delete_my_llm_key(self, flavor: str, *, base_url: str = "") -> bool:
269        """Remove it. 404 for a provider you hold no key for — the same
270        answer somebody else's key gets, so this cannot report who holds
271        what.
272
273        ⚠️ ``base_url`` is part of the address, not a filter. Naming only
274        the dialect is ambiguous the moment you hold two ``openai``
275        providers, and resolving that by picking one is how somebody
276        loses the key they meant to keep. ``""`` names that dialect's
277        official endpoint, which is a value rather than a wildcard.
278        """
279        resp = self._http.request(
280            "DELETE", f"/eval/llm-keys/{flavor}", params={"base_url": base_url}
281        )
282        _raise_for_status(resp)
283        return bool(resp.json().get("deleted", False))
284
285    def probe_my_llm_key(self, flavor: str, *, base_url: str = "") -> MyLlmKeyProbe:
286        """Spend one token on your own credential and report what happened.
287
288        Storing a key says the write landed; it says nothing about
289        whether the credential **works**. The only other way to find
290        that out is to enter an agent, let a run be dispatched, and read
291        the failure out of the artifact tarball — two hours, on a full
292        range, to learn that a key is invalid.
293
294        ⚠️ **A code, never a boolean.** A rejected key, an address
295        nothing answers at, an account out of credit and a deployment
296        with no model registered at this endpoint are four different
297        fields to go and fix.
298
299        ⚠️ **200 with the outcome in the body** for every one of them —
300        so this does not raise on a failed credential. A 404 (and only a
301        404) means you hold no key for that provider.
302
303        ⚠️ ``base_url`` is part of the address, exactly as it is for the
304        removal beside it: naming the dialect alone picks whichever row
305        matched first the moment you hold two ``openai`` providers.
306        """
307        resp = self._http.request(
308            "POST", f"/eval/llm-keys/{flavor}/probe", params={"base_url": base_url}
309        )
310        _raise_for_status(resp)
311        return MyLlmKeyProbe.model_validate(resp.json())
312
313    # -- #3487: model connections — one vendor account, one key ------------
314
315    def llm_accounts(self) -> list[LlmAccountChoice]:
316        """The vendor accounts this deployment relays to, one row per account.
317
318        Never per dialect: the gateway routes each request to the account's
319        endpoint in the dialect the agent sent, so there is nothing to pick.
320        """
321        resp = self._http.request("GET", "/eval/llm-accounts")
322        _raise_for_status(resp)
323        return [LlmAccountChoice.model_validate(r) for r in resp.json()]
324
325    def my_llm_connections(self) -> list[MyLlmConnection]:
326        """Your connections, with each surface's last check and the model list."""
327        resp = self._http.request("GET", "/eval/llm-connections")
328        _raise_for_status(resp)
329        return [MyLlmConnection.model_validate(r) for r in resp.json()]
330
331    def connect_llm(self, account: str, api_key: str, *, label: str = "") -> MyLlmConnection:
332        """Save (or rotate) the key for one vendor account, then check it.
333
334        The check probes every surface of the account once with the key — a
335        tiny call each, on your own bill — and reads the vendor's model list.
336        A key that fails is still saved; the answer says which surface refused.
337        """
338        body = MyLlmConnectionCreate(account=account, api_key=api_key, label=label)
339        resp = self._http.request("POST", "/eval/llm-connections", json=body.model_dump())
340        _raise_for_status(resp)
341        return MyLlmConnection.model_validate(resp.json())
342
343    def recheck_llm_connection(self, connection_id: str) -> MyLlmConnection:
344        """Probe every surface again and re-read the model list (200 either way)."""
345        body = LlmConnectionRef(id=connection_id).model_dump()
346        resp = self._http.request("POST", "/eval/llm-connections/check", json=body)
347        _raise_for_status(resp)
348        return MyLlmConnection.model_validate(resp.json())
349
350    def refresh_llm_connection_models(self, connection_id: str) -> MyLlmConnection:
351        """Re-read only the vendor's model list with this key (free, no probe)."""
352        body = LlmConnectionRef(id=connection_id).model_dump()
353        resp = self._http.request("POST", "/eval/llm-connections/models", json=body)
354        _raise_for_status(resp)
355        return MyLlmConnection.model_validate(resp.json())
356
357    def llm_connections_usage(self) -> LlmUsagePage:
358        """Your connections' last-30-day tokens, runs and estimated cost."""
359        resp = self._http.request("GET", "/eval/llm-connections/usage")
360        _raise_for_status(resp)
361        return LlmUsagePage.model_validate(resp.json())
362
363    def read_llm_connection_balance(
364        self, connection_id: str, *, force: bool = False
365    ) -> MyLlmConnection:
366        """Read the account balance with this key (``force`` skips the 10-min reuse)."""
367        body = LlmBalanceRefresh(id=connection_id, force=force).model_dump()
368        resp = self._http.request("POST", "/eval/llm-connections/balance", json=body)
369        _raise_for_status(resp)
370        return MyLlmConnection.model_validate(resp.json())
371
372    def set_llm_connection_cap(
373        self, connection_id: str, *, usd: float = 0.0, tokens: int = 0
374    ) -> MyLlmConnection:
375        """Set one connection's monthly cap; zeros clear it."""
376        body = LlmSpendCapUpdate(id=connection_id, usd=usd, tokens=tokens).model_dump()
377        resp = self._http.request("PUT", "/eval/llm-connections/cap", json=body)
378        _raise_for_status(resp)
379        return MyLlmConnection.model_validate(resp.json())
380
381    def delete_llm_connection(self, connection_id: str) -> bool:
382        """Remove one connection. 404 for one that is not yours."""
383        resp = self._http.request(
384            "DELETE", "/eval/llm-connections", params={"connection_id": connection_id}
385        )
386        _raise_for_status(resp)
387        return bool(resp.json().get("deleted", False))
388
389    def my_allowances(self) -> list[EvalAllowanceRow]:
390        """What an operator has granted this account, and how much is left.
391
392        ⚠️ Read ``remaining_usd`` rather than subtracting: the server
393        clamps an overshoot at zero, and a client doing its own
394        arithmetic would eventually disagree with the refusal that
395        actually decides.
396        """
397        resp = self._http.request("GET", "/eval/allowances")
398        _raise_for_status(resp)
399        return [EvalAllowanceRow.model_validate(r) for r in resp.json()]
400
401    def models(self) -> list[LlmModelOption]:
402        """The models an entry may name.
403
404        ⚠️ Approved and active only, decided by the server. The registry
405        holds more than this — the operator's own notes and the
406        addressing that says where their credential is spent — and none
407        of it travels.
408        """
409        resp = self._http.request("GET", "/eval/models")
410        _raise_for_status(resp)
411        return [LlmModelOption.model_validate(r) for r in resp.json()]
412
413    def enter(
414        self,
415        *,
416        harness_id: str,
417        model_id: str = "",
418        competition_id: str,
419        num_samples: int = 1,
420        harness_version_id: str = "",
421        funding_source: EvalFundingSource = "auto",
422        llm_connection_id: str = "",
423        scheduling_cost_limit_usd: float = 0.0,
424        timeout_s: int = 0,
425        max_turns: int = 0,
426        challenge_ids: Sequence[str] = (),
427        provider_model: str = "",
428    ) -> EvalCampaignState:
429        """Put one of your own agents into one competition.
430
431        Name a registered ``model_id`` (``models()``), **or** — #3471 — type
432        the provider's own model id as ``provider_model`` with
433        ``funding_source="byok"`` and the ``llm_connection_id`` of the saved
434        connection whose provider serves it. No operator row is needed; the
435        platform spends one small call on your key to check the pair first
436        and refuses with a ``byok_model_check_*`` code when it fails.
437
438        ⚠️ **The result is `self_reported`, and the server decides that.**
439        The default public leaderboard shows operator-run results; this
440        is your own claim about your own agent, so read it back with
441        ``tier="self_reported"`` or ``"all"``. There is no parameter for
442        it here because there is none on the wire.
443
444        ⚠️ **409 carries the reason as its ``code``** — whether to wait
445        for review, add a key, or pick another event. A caller that
446        reports only "entry refused" throws away the only actionable
447        part of the answer.
448
449        ⚠️ **``funding_source`` defaults to ``"auto"`` here only for the
450        callers that predate it.** Pass ``"platform"`` or ``"byok"`` (with
451        ``llm_connection_id``) to say who pays: an explicit choice never
452        falls back to the other, which is what ``ctfy eval run`` does.
453        ``scheduling_cost_limit_usd`` stops *new* dispatches once recorded
454        spend reaches it; it is a scheduling threshold, not a hard cap.
455        ``0`` for ``timeout_s`` / ``max_turns`` inherits the platform's.
456
457        ``challenge_ids`` makes it a **trial run** (#2551): only those
458        challenges of the target, ranked on no board. Empty is the whole set.
459        """
460        body: dict[str, Any] = {
461            "harness_id": harness_id,
462            "harness_version_id": harness_version_id,
463            "model_id": model_id,
464            "competition_id": competition_id,
465            "num_samples": num_samples,
466        }
467        # ⚠️ Only what was asked for: the bodies older callers send stay
468        # byte-for-byte what they were, so `auto` is never spelled out.
469        if funding_source != "auto":
470            body["funding_source"] = funding_source
471        if llm_connection_id:
472            body["llm_connection_id"] = llm_connection_id
473        if scheduling_cost_limit_usd:
474            body["scheduling_cost_limit_usd"] = scheduling_cost_limit_usd
475        if timeout_s:
476            body["timeout_s"] = timeout_s
477        if max_turns:
478            body["max_turns"] = max_turns
479        if challenge_ids:
480            body["challenge_ids"] = list(challenge_ids)
481        if provider_model:
482            body["provider_model"] = provider_model
483        resp = self._http.request("POST", "/eval/entries", json=body)
484        _raise_for_status(resp)
485        return EvalCampaignState.model_validate(resp.json())
486
487    def my_entries(
488        self, *, competition_id: str = "", offset: int = 0, limit: int = 50
489    ) -> PagedList[EvalCampaignState]:
490        """The campaigns this account entered, newest first — one page,
491        with the total (``limit`` at most 200).
492
493        Only entries: an operator campaign the same account queued is
494        not one of them, because the two are different claims about the
495        same board. ``competition_id`` narrows to entries into that one
496        competition, on the server.
497
498        ⚠️ This returned a bare list, the server's newest 200, with
499        nothing to say the list was cut; ``total`` says how many exist.
500        """
501        params: dict[str, Any] = {"offset": offset, "limit": limit}
502        if competition_id:
503            params["competition_id"] = competition_id
504        resp = self._http.request("GET", "/eval/entries", params=params)
505        _raise_for_status(resp)
506        return _extract_items(resp.json(), EvalCampaignState)
507
508    def runs(self, *, competition_id: str = "", offset: int = 0, limit: int = 50) -> MyEvalRunPage:
509        """Evaluation runs you may read — your own plus your team-mates'.
510
511        ⚠️ The author set is the same membership join the per-run routes
512        enforce, so a run this lists can always be opened and a run it
513        omits always 404s. Two readings of "the same team" would drift
514        in the open direction.
515
516        ⚠️ ``mine`` on each row says whether *you* may open the
517        conversations. It is composed server-side because it gates a
518        control, and two clients deriving it would disagree.
519        Wraps ``GET /eval/runs``."""
520        resp = self._http.request(
521            "GET",
522            "/eval/runs",
523            params={"competition_id": competition_id, "offset": offset, "limit": limit},
524        )
525        _raise_for_status(resp)
526        return MyEvalRunPage.model_validate(resp.json())
527
528    def run(self, run_id: str) -> MyEvalRunRow:
529        """One of your runs, projected as the shelf projects it.
530
531        ⚠️ Addressable because :meth:`runs` is paginated: one campaign
532        over a large bucket is hundreds of runs, so finding a row by
533        scanning the first page is a bug with a threshold.
534
535        Metadata scope — a team-mate reads the row, and its ``traffic``
536        field says ``not_yours`` rather than the route refusing.
537        Wraps ``GET /eval/runs/{run_id}``."""
538        resp = self._http.request("GET", f"/eval/runs/{run_id}")
539        _raise_for_status(resp)
540        return MyEvalRunRow.model_validate(resp.json())
541
542    def campaign_runs(self, campaign_id: str) -> MyEvalRunPage:
543        """Every atomic task of one campaign, in dispatch order (#2585).
544
545        ⚠️ Whole, never a page: the server answers a ``campaign_id`` listing
546        in full, so counting stages or numbering the queue from it is sound.
547        Wraps ``GET /eval/runs?campaign_id=``."""
548        resp = self._http.request("GET", "/eval/runs", params={"campaign_id": campaign_id})
549        _raise_for_status(resp)
550        return MyEvalRunPage.model_validate(resp.json())
551
552    def run_events(self, run_id: str) -> EvalRunEventList:
553        """One atomic task's event log, in time order (#2585).
554        Wraps ``GET /eval/runs/{run_id}/events``."""
555        resp = self._http.request("GET", f"/eval/runs/{run_id}/events")
556        _raise_for_status(resp)
557        return EvalRunEventList.model_validate(resp.json())
558
559    def cancel_run(
560        self, run_id: str, *, mode: CancelMode = "drain", requeue: bool = False
561    ) -> EvalQueueChange:
562        """Cancel one atomic task (#2585): queued → cancelled now; started →
563        stopped (``mode="now"``) or left to finish (``"drain"``). A task
564        being graded answers ``409 eval_run_grading``.
565        Wraps ``POST /eval/runs/{run_id}/cancel``."""
566        resp = self._http.request(
567            "POST",
568            f"/eval/runs/{run_id}/cancel",
569            json=EvalCancelRequest(mode=mode, requeue=requeue).model_dump(),
570        )
571        _raise_for_status(resp)
572        return EvalQueueChange.model_validate(resp.json())
573
574    def campaign_summary(self, campaign_id: str) -> EvalCampaignAtomicSummary:
575        """A campaign's tasks counted per stage, and whether it is
576        ``incomplete`` (off the official board) (#2585).
577        Wraps ``GET /eval/campaigns/{campaign_id}/summary``."""
578        resp = self._http.request("GET", f"/eval/campaigns/{campaign_id}/summary")
579        _raise_for_status(resp)
580        return EvalCampaignAtomicSummary.model_validate(resp.json())
581
582    def cancel_campaign(self, campaign_id: str, *, mode: CancelMode = "drain") -> EvalQueueChange:
583        """Cancel a campaign's queued tasks now, its started ones per
584        ``mode`` (#2585). Wraps ``POST /eval/campaigns/{campaign_id}/cancel``."""
585        resp = self._http.request(
586            "POST",
587            f"/eval/campaigns/{campaign_id}/cancel",
588            json=EvalCancelRequest(mode=mode).model_dump(),
589        )
590        _raise_for_status(resp)
591        return EvalQueueChange.model_validate(resp.json())
592
593    def batch_limits(self) -> EvalBatchLimits:
594        """The most cells one batch may hold, and the ranges your runs may
595        hold at once across every campaign of yours (#2677).
596        Wraps ``GET /eval/batches/limits``."""
597        resp = self._http.request("GET", "/eval/batches/limits")
598        _raise_for_status(resp)
599        return EvalBatchLimits.model_validate(resp.json())
600
601    def enter_batch(
602        self,
603        *,
604        competition_id: str,
605        harness_ids: Sequence[str],
606        models: Sequence[EvalBatchModel],
607        num_samples: int = 1,
608        scheduling_cost_limit_usd: float = 0.0,
609        timeout_s: int = 0,
610        max_turns: int = 0,
611        challenge_ids: Sequence[str] = (),
612    ) -> EvalBatchState:
613        """Enter each of your agents with each model on one target: one
614        campaign per cell, grouped under a batch (#2677).
615
616        Every cell is checked as :meth:`enter` checks one entry before any is
617        created, so a refusal carries that cell's own ``code`` and nothing
618        is created. More cells than the deployment allows is ``409
619        eval_batch_too_large``. ``scheduling_cost_limit_usd`` is per cell.
620        Wraps ``POST /eval/batches``."""
621        body = EvalBatchRequest(
622            competition_id=competition_id,
623            harness_ids=list(harness_ids),
624            models=list(models),
625            num_samples=num_samples,
626            scheduling_cost_limit_usd=scheduling_cost_limit_usd,
627            timeout_s=timeout_s,
628            max_turns=max_turns,
629            challenge_ids=list(challenge_ids),
630        )
631        resp = self._http.request("POST", "/eval/batches", json=body.model_dump())
632        _raise_for_status(resp)
633        return EvalBatchState.model_validate(resp.json())
634
635    def batch(self, batch_id: str) -> EvalBatchState:
636        """One of your batches: its cells' campaigns (#2677).
637        Wraps ``GET /eval/batches/{batch_id}``."""
638        resp = self._http.request("GET", f"/eval/batches/{batch_id}")
639        _raise_for_status(resp)
640        return EvalBatchState.model_validate(resp.json())
641
642    def cancel_batch(self, batch_id: str, *, mode: CancelMode = "drain") -> EvalBatchCancelResult:
643        """Cancel every cell of a batch still running, each as
644        :meth:`cancel_campaign` would (#2677).
645        Wraps ``POST /eval/batches/{batch_id}/cancel``."""
646        resp = self._http.request(
647            "POST",
648            f"/eval/batches/{batch_id}/cancel",
649            json=EvalCancelRequest(mode=mode).model_dump(),
650        )
651        _raise_for_status(resp)
652        return EvalBatchCancelResult.model_validate(resp.json())
653
654    def rerun_unfinished(self, campaign_id: str) -> EvalQueueChange:
655        """Queue one fresh task per cancelled task nothing has filled (#2585).
656        Wraps ``POST /eval/campaigns/{campaign_id}/rerun-unfinished``."""
657        resp = self._http.request("POST", f"/eval/campaigns/{campaign_id}/rerun-unfinished")
658        _raise_for_status(resp)
659        return EvalQueueChange.model_validate(resp.json())
660
661    def run_llm_calls(self, run_id: str, *, offset: int = 0, limit: int = 50) -> LlmCallPage:
662        """Every model request one of your runs made, newest first.
663
664        Metadata only — token split, latency, queue wait, the estimated
665        cost. **Scoped to the run's team**, so a captain can see who is
666        burning the allowance without reading anybody's prompts, and to
667        ``principal == "agent"``, so a challenge container's own
668        conversation is never on this route.
669        Wraps ``GET /eval/runs/{run_id}/llm-calls``."""
670        resp = self._http.request(
671            "GET",
672            f"/eval/runs/{run_id}/llm-calls",
673            params={"offset": offset, "limit": limit},
674        )
675        _raise_for_status(resp)
676        return LlmCallPage.model_validate(resp.json())
677
678    def run_agent_log(self, run_id: str, *, after: int = 0) -> AgentLogPage:
679        """One of your runs' own stdout and stderr, live, after a cursor (#2401).
680
681        **The run author's only** (``403 agent_log_owner_only`` for a
682        team-mate). Pass the last ``seq`` you hold as *after*; a jump in
683        ``seq`` is lines that are no longer held. Scrubbed on the platform
684        before it was held.
685        Wraps ``GET /eval/runs/{run_id}/agent-log``."""
686        resp = self._http.request("GET", f"/eval/runs/{run_id}/agent-log", params={"after": after})
687        _raise_for_status(resp)
688        return AgentLogPage.model_validate(resp.json())
689
690    def run_llm_live(self, run_id: str, *, after: int = 0) -> LlmLivePage:
691        """What one of your runs' model is sending back, live, after a cursor
692        (#2401 slice 2).
693
694        The body's readers only — the author, or a platform admin
695        (``403 llm_live_owner_only`` for a team-mate). Pass the last ``seq``
696        you hold as *after*; a jump in ``seq`` is deltas no longer held.
697        Wraps ``GET /eval/runs/{run_id}/llm-live``."""
698        resp = self._http.request("GET", f"/eval/runs/{run_id}/llm-live", params={"after": after})
699        _raise_for_status(resp)
700        return LlmLivePage.model_validate(resp.json())
701
702    def run_llm_call(self, run_id: str, call_id: str) -> LlmCallBody:
703        """The conversation itself — **the run author's only**.
704
705        ⚠️ Narrower than :meth:`run_llm_calls` on purpose: a team-mate
706        who can see the row is answered ``403
707        transcript_body_owner_only`` here, since an agent's prompt is its
708        author's work and ``HarnessStanding`` ranks the two of them
709        against each other.
710
711        ⚠️ ``available=False`` is an ordinary answer, not an error —
712        retention reclaims bodies and leaves the row behind.
713        Wraps ``GET /eval/runs/{run_id}/llm-calls/{call_id}``."""
714        resp = self._http.request("GET", f"/eval/runs/{run_id}/llm-calls/{call_id}")
715        _raise_for_status(resp)
716        return LlmCallBody.model_validate(resp.json())
717
718    def run_artifact(self, run_id: str) -> bytes:
719        """The files one of your runs left behind, as ``.tar.gz`` bytes.
720
721        Its working directory and, when the harness is Claude Code, its
722        session — collected from the container before it was removed.
723        The writing half, where :meth:`run_llm_call` is the deciding one
724        and :meth:`run_traffic` the doing one.
725
726        ⚠️ **The run author's only**, the narrowest of the three: a
727        team-mate who can see the row is answered ``403
728        run_artifact_owner_only``. An archive carries whatever the agent
729        wrote down, which for a solved challenge includes the flag.
730
731        ⚠️ **Bytes, not a model, and no URL.** The read goes through the
732        platform rather than handing out a bucket address, so there is
733        nothing to return but the archive itself. ⚠️ A run that kept no
734        files answers ``404 run_artifact_not_found`` rather than empty
735        bytes — which of the several reasons it is (the agent wrote
736        nothing, collection failed, retention took them) is on the run
737        row's ``artifact`` field, so the caller never has to guess from
738        an absence.
739
740        Wraps ``GET /eval/runs/{run_id}/artifact``."""
741        resp = self._http.request("GET", f"/eval/runs/{run_id}/artifact")
742        _raise_for_status(resp)
743        return bytes(resp.content)
744
745    def run_traffic(self, run_id: str, *, cursor: str = "", limit: int = 0) -> InstanceTrafficPage:
746        """Every request one of your runs sent at the challenge.
747
748        The doing half, where :meth:`run_llm_call` is the deciding half.
749        **The run author's only** — a team-mate who can see the row is
750        answered ``403 run_traffic_owner_only``, the same split the
751        conversation body makes, because the payloads an agent wrote are
752        its author's work.
753
754        ⚠️ Carry ``cursor`` forward from the previous page and you get
755        only what arrived since; pass an empty one and you get the whole
756        capture again. Every failure — an unreachable node, a run whose
757        estate the reaper has taken — answers 200 with your own cursor
758        and no flows, so a blip never re-delivers the capture. Which of
759        those it was, ``MyEvalRunRow.traffic`` says.
760
761        ⚠️ Redacted: this instance's minted answers are scrubbed out.
762        Wraps ``GET /eval/runs/{run_id}/traffic``."""
763        resp = self._http.request(
764            "GET",
765            f"/eval/runs/{run_id}/traffic",
766            params={"cursor": cursor, "limit": limit},
767        )
768        _raise_for_status(resp)
769        return InstanceTrafficPage.model_validate(resp.json())
770
771    def run_writeup(self, run_id: str) -> RunWriteup:
772        """What one of your runs said it did, in its own words.
773
774        ``CTFY-0001`` §3 invites an optional ``writeup`` in the result
775        file. The saying half, where :meth:`run_llm_call` is the
776        deciding one, :meth:`run_traffic` the doing one and
777        :meth:`run_artifact` the writing one.
778
779        ⚠️ **Telemetry, never a grading path.** Nothing in the result
780        file is scored — the run's own ``POST /submissions`` is — so a
781        write-up cannot earn a point and is not read as an answer.
782
783        ⚠️ **The run author's only**: a team-mate who can see the row is
784        answered ``403 run_writeup_owner_only``, the same split the
785        other three bodies make. ⚠️ A run whose image wrote none answers
786        ``404 run_writeup_not_found`` rather than empty prose, and the
787        run row's ``writeup`` field already said which — so the caller
788        never has to reach here to find out.
789
790        Wraps ``GET /eval/runs/{run_id}/writeup``."""
791        resp = self._http.request("GET", f"/eval/runs/{run_id}/writeup")
792        _raise_for_status(resp)
793        return RunWriteup.model_validate(resp.json())
794
795    def set_run_wall(self, run_id: str, *, published: bool) -> MyEvalRunRow:
796        """Put this run's agent on the event's projector wall, or take it off.
797
798        ``GET /competitions/{id}/agent-actions`` draws one scrubbed line
799        per agent saying what it is doing. The scrub answers what the
800        bytes make unsafe; this answers the part it cannot — the
801        conversation belongs to whoever ran it, so the wall shows an
802        entrant only once their author has said yes.
803
804        ⚠️ **The switch is on the *entrant*, not on this run.** The
805        transcript index groups by team and an agent's team is one per
806        ``(owner, harness, model, competition)``, so every run of the
807        same agent at the same event shares one answer — which is what
808        "this agent, at this event" means. The row that comes back is
809        the run's, with ``wall_published`` re-read from the entrant.
810
811        ⚠️ **The author's only**: a team-mate who can see the row is
812        answered ``403 run_wall_not_yours``, the same split
813        :meth:`run_writeup` makes, and for the same reason — the line
814        is read out of a conversation they already meet a 403 on.
815
816        ⚠️ A run with no agent entrant answers ``409`` naming which
817        kind of nothing it is; ``MyEvalRunRow.wall`` carries the same
818        code, so a caller never has to press to find out.
819
820        Wraps ``POST /eval/runs/{run_id}/wall``."""
821        resp = self._http.request(
822            "POST",
823            f"/eval/runs/{run_id}/wall",
824            json={"published": published},
825        )
826        _raise_for_status(resp)
827        return MyEvalRunRow.model_validate(resp.json())
828
829    def set_run_transcript(self, run_id: str, *, published: bool) -> MyEvalRunRow:
830        """Publish this run's agent's **full conversation**, or take it back.
831
832        A **second, stronger** consent than :meth:`set_run_wall`. The wall
833        shows one scrubbed line of what the agent is doing; this opens the
834        whole conversation — every turn the agent wrote — to spectators at
835        ``GET /competitions/{id}/awd/transcript/{call_id}``. The projection
836        still emits only the agent's own scrubbed turns (never the system
837        prompt, never a tool result), but the decision to publish a
838        conversation at all is the author's alone and separate from the
839        wall's.
840
841        ⚠️ **The switch is on the *entrant***, exactly as the wall's —
842        one answer per ``(owner, harness, model, competition)``.
843
844        ⚠️ **Refused for an anonymous entrant** (``403
845        run_transcript_anonymous``): a conversation de-anonymises whoever
846        wrote it, so the withheld-identity promise wins over the consent.
847        The author's-only and no-agent refusals mirror :meth:`set_run_wall`.
848
849        Wraps ``POST /eval/runs/{run_id}/transcript``."""
850        resp = self._http.request(
851            "POST",
852            f"/eval/runs/{run_id}/transcript",
853            json={"published": published},
854        )
855        _raise_for_status(resp)
856        return MyEvalRunRow.model_validate(resp.json())
857
858    def registry_push_target(self) -> RegistryPushTarget:
859        """Where this account pushes an agent image, if anywhere.
860
861        ⚠️ Read ``enabled``, not a non-empty ``repository_prefix``. The
862        server composes the verdict from four facts — a host, a token
863        service, and both halves of the signing pair — and a deployment
864        with a hostname but no signing pair refuses every login.
865        """
866        resp = self._http.request("GET", "/eval/registry")
867        _raise_for_status(resp)
868        return RegistryPushTarget.model_validate(resp.json())
869
870    def harness_tags(self, harness_id: str) -> HarnessTags:
871        """Which versions the registry publishes for one of your agents.
872
873        ⚠️ Addressed by **harness id**, not by image: the platform looks
874        the reference up in your own registry row, which is what keeps
875        this from being an outbound proxy. Answers 404 for anything that
876        is not yours, and 502 when the registry itself would not answer
877        — never an empty list, which would read as "no versions".
878        """
879        resp = self._http.request("GET", f"/eval/harnesses/{harness_id}/tags")
880        _raise_for_status(resp)
881        return HarnessTags.model_validate(resp.json())
882
883    def check_harness(self, harness_id: str) -> HarnessState:
884        """Ask the platform to look at one of your agent images.
885
886        Returns the row with the verdict on it: ``conformance_status``
887        (``""`` never checked / ``passed`` / ``failed`` / ``error``),
888        the digest the verdict describes, and a finding per thing worth
889        saying.
890
891        ⚠️ **Evidence, never a gate.** Nothing here changes
892        ``review_status``, and a ``failed`` verdict does not stop an
893        operator approving the image — the check exists so somebody can
894        see *why* a dispatch would go wrong, before it does.
895
896        ⚠️ **``error`` is not ``failed``, and it arrives as a 200.** A
897        registry that would not answer is not a fact about the image;
898        the message is on ``conformance_error``.
899
900        ⚠️ Addressed by harness id, like :meth:`harness_tags` and for
901        the same reason: the image is read off your own registry row, so
902        this cannot be pointed at an arbitrary host.
903        """
904        resp = self._http.request("POST", f"/eval/harnesses/{harness_id}/conformance")
905        _raise_for_status(resp)
906        return HarnessState.model_validate(resp.json())
907
908    def probe_harness(self, harness_id: str) -> MyHarness:
909        """Put one of your agent images in line to be run once, to check it.
910
911        Where :meth:`check_harness` reads a manifest, this runs the
912        container on a worker that may hold a user's code, under the
913        isolation a real dispatch uses, with a briefing marked ``dry_run``,
914        and asks the one question a manifest cannot answer: **did it write
915        a result file at the address it was given.**
916
917        ⚠️ **Returns the row in line, not a verdict** (#2608):
918        ``probe_status`` is ``queued`` (with ``probe_reason`` and
919        ``probe_position``) or ``running``; poll :meth:`harnesses` for
920        ``passed`` / ``failed`` (with a reason) / ``error``.
921
922        ⚠️ **No credential travels with the probe** but the row's own
923        scoped pull ticket, or the owner's stored registry login for the
924        image's own host (#3475), and no platform URL is named. See §3.1 of
925        ``docs/standards/CTFY-0001.md``.
926        """
927        resp = self._http.request("POST", f"/eval/harnesses/{harness_id}/probe")
928        _raise_for_status(resp)
929        return MyHarness.model_validate(resp.json())
930
931    def cancel_probe(self, harness_id: str) -> MyHarness:
932        """Take a queued dry run out of line (409 once it has started)."""
933        resp = self._http.request("DELETE", f"/eval/harnesses/{harness_id}/probe")
934        _raise_for_status(resp)
935        return MyHarness.model_validate(resp.json())
936
937    def set_registry_credential(self, harness_id: str, *, username: str, secret: str) -> MyHarness:
938        """Store the login your agent's image is pulled with from a private registry (#3475).
939
940        The registry is the one the agent's image names; a host the platform
941        does not pull from is refused (400 ``harness_registry_not_allowed``).
942        The answer shows ``registry_credential`` (who and when), never the
943        token. Run the check and dry run again afterwards.
944        """
945        resp = self._http.request(
946            "PUT",
947            f"/eval/harnesses/{harness_id}/registry-credential",
948            json={"username": username, "secret": secret},
949        )
950        _raise_for_status(resp)
951        return MyHarness.model_validate(resp.json())
952
953    def clear_registry_credential(self, harness_id: str) -> MyHarness:
954        """Remove your agent's registry login; its image is then pulled anonymously."""
955        resp = self._http.request("DELETE", f"/eval/harnesses/{harness_id}/registry-credential")
956        _raise_for_status(resp)
957        return MyHarness.model_validate(resp.json())
958
959
960class RegistryResource:
961    """The container registry's token endpoint.
962
963    ⚠️ **Not a method anybody calls from Python.** ``docker`` follows
964    the registry's ``WWW-Authenticate`` realm here on its own, with HTTP
965    Basic; this exists so the SDK covers the surface and so a test can
966    drive the exchange the way docker does. The credential goes in the
967    *password* half — that is where ``docker login -p`` puts it.
968    """
969
970    def __init__(self, http: BaseHttpClient) -> None:
971        self._http = http
972
973    def token(self, *, service: str, scope: str | Sequence[str] = "") -> dict[str, Any]:
974        # ⚠️ A sequence is sent as a repeated ``scope`` parameter, because
975        # that is what docker does for a cross-repository blob mount —
976        # ``push`` on the destination and ``pull`` on the source, in one
977        # exchange. This method's whole stated purpose is that a test can
978        # drive the exchange the way docker does, and it could not.
979        params: list[tuple[str, str]] = [("service", service)]
980        if isinstance(scope, str):
981            if scope:
982                params.append(("scope", scope))
983        else:
984            params.extend(("scope", s) for s in scope if s)
985        resp = self._http.request("GET", "/registry/token", params=params)
986        _raise_for_status(resp)
987        body: dict[str, Any] = resp.json()
988        return body