# Bless recipes (e2e replay fixtures) + the one-command notebook stack.
#
# The local dev stack is `screamingface up` (OME-1001): the `notebooks` recipe SHELLS
# that CLI rather than assembling a stack of its own, so devs keep running the exact
# code path users run. It adds one thing the CLI cannot: a second Engine carrying the
# `inspect` extra, because `inspect-ai` and the runtime extra's `litellm` pin are
# declared uncoinstallable — the imported boards can only be served from their own env.

set default-list := true

# One blesser, four flag shapes — the path moves in a single place.
blesser := "uv run python tests/e2e/fixtures/slice_snapshot.py"
# The GSM8K vote lane (OME-1289 E4a, EV-12) has its own two-step script: record (paid), bless (free).
reducer_lane := "uv run python tests/e2e/fixtures/reducer_lane.py"
# The GSM8K Fusion regression lane (OME-1289 E4a, FR#13): the same two steps, record (paid), bless (free).
fusion_lane := "uv run python tests/e2e/fixtures/fusion_lane.py"

# Where each benchmark's downloaded dataset is written — resolved the SAME way the blesser
# resolves it (`tests/e2e/fixtures/slice_snapshot.py`: SCREAMINGFACE_E2E_ASSETS, else
# `default_data_dir()/benchmark-assets`, which SCREAMINGFACE_DATA_DIR moves).
#
# NOT URL4_BENCHMARK_ASSETS: that is what the Engine PROCESS is handed so it can find the
# assets, an output of this file rather than an input. Keying the precondition on it was
# wrong in both directions — a dev who had moved SCREAMINGFACE_DATA_DIR and run
# `screamingface prepare --all` got told to run the command they had just run, and a stale
# value in the shell let the precondition pass while the blesser died deep inside, which is
# the half-written-fixtures failure the precondition exists to prevent.
data_dir := env("SCREAMINGFACE_DATA_DIR", home_directory() / ".screamingface")
assets := env("SCREAMINGFACE_E2E_ASSETS", data_dir / "benchmark-assets")

# source_directory(), NOT justfile_directory(): reached through the root justfile's `mod`,
# the latter resolves to the REPO ROOT and every relative path below lands two levels too high.
engine_dir := source_directory() / ".." / ".." / "apps" / "screamingface-engine"
# Not 9108: `screamingface up` owns that one, and this Engine runs beside it. Overridable
# because two worktrees running this recipe at once would otherwise contend for one port —
# which this repo expects, since sessions run concurrently against one clone.
engine_port := env("SCREAMINGFACE_NOTEBOOK_ENGINE_PORT", "9111")

# Every bless recipe's preconditions, checked BEFORE its body — a run that discovers a
# missing docker or unprepared assets deep inside Python may already have written partial
# fixtures. A dependency rather than a top-level variable because `require()` and `error()`
# both evaluate eagerly there, which would abort `local-stack-notebooks` too (it needs
# neither, and prepares its own datasets).
[private]
_bless-preconditions:
    @echo "docker: {{ require('docker') }}"
    @echo "assets: {{ if path_exists(assets) == 'true' { assets } else { error('no benchmark assets at ' + assets + ' — run `uv run screamingface prepare --all`') } }}"

# Open EVERY example notebook against a stack serving both board families: ours
# (7 boards, origin=screamingface) and imported (10 inspect_evals boards). Downloads each
# benchmark's dataset on first run — network, minutes — then reuses it forever.
#
# Layout: `screamingface up` brings the Gateway (:9105) and Scoreboard (:9106); the
# Engine started here serves ALL 17 boards on :9111 and reaches that same Gateway, so
# the stack's own Engine simply goes unused.
#
# Shutting down: quit JupyterLab (Ctrl-C in this terminal, or File > Shut Down) and the
# recipe tears down what it started. The Engine it launched always goes; the stack goes
# only if this recipe was what started it — a stack you already had running is left alone,
# because tearing down someone else's is not this recipe's business. Stop that one yourself
# with `screamingface down` when you are finished with it.
[doc("Open every example notebook against a stack serving ours AND imported boards")]
local-stack-notebooks:
    #!/usr/bin/env bash
    set -euo pipefail
    UV="{{ require('uv') }}"
    # require() so a missing binary aborts loudly. Bare `lsof` was worse than useless:
    # its "not found" went to the same /dev/null as a clean result, so on a box without
    # it the port guard below silently never fired.
    LSOF="{{ require('lsof') }}"
    CURL="{{ require('curl') }}"

    # Each benchmark's dataset is downloaded once and written to disk as fixed files (the
    # repo calls this baking); a run afterwards only reads them. That is slow and it is
    # exactly what a half-finished attempt must not repeat, so this resumes one benchmark
    # at a time. Every preparer writes `cases.json` LAST, so a directory containing it is
    # finished; a directory with anything else is wreckage from a run that died mid-write,
    # which the preparer would refuse to write over. Ask the Engine which exist, clear
    # only the unfinished directories, and redo exactly those. Nothing complete is fetched
    # twice, and nothing usable is deleted (without cases.json a benchmark cannot be served).
    #
    # `benchmarks` is the Engine's preparer extra — `inspect` (the imported boards' own
    # scorers, plus the pinned `datasets`) plus the pdfplumber/python-docx readers that
    # flatten GDPval's reference PDFs into text. It is the SAME extra Dockerfile.benchmark
    # syncs, so a notebook reads data built exactly the way production builds it.
    ( cd "{{ engine_dir }}" && "$UV" sync --extra benchmarks )

    # A bundle counts as finished only when its cases.json PARSES. Existence alone is not
    # enough: the preparers write that file non-atomically, so a run killed mid-write (or a
    # full disk) leaves a truncated, unparseable file that `-f` would happily call complete
    # — and the bundle would then be skipped forever while the Engine could never serve it.
    # Reading it is cheap next to re-downloading a dataset by mistake.
    missing=()
    while read -r bundle; do
        if "$UV" run python -c "import json,sys;json.load(open(sys.argv[1]))" \
            "{{ assets }}/$bundle/cases.json" >/dev/null 2>&1; then
            continue
        fi
        # Unfinished or corrupt: clear the debris so the preparer sees a clean directory.
        # Nothing usable is lost — a bundle whose cases.json will not parse cannot be served.
        rm -rf "{{ assets }}/${bundle:?}"
        missing+=("--bundle" "$bundle")
    done < <( cd "{{ engine_dir }}" && "$UV" run python -m screamingface_engine.benchmarks.prepare --list-bundles )

    # Gated datasets (xstest_safe, OME-1269) need a Hugging Face token: HF_TOKEN, or a
    # `huggingface-cli login`, from an account that accepted the dataset's terms. With
    # neither, the preparer skips those boards with a warning (they stay unfinished and
    # are retried next time) instead of stopping every bundle after them. Set
    # SCREAMINGFACE_SKIP_BENCHMARKS_NEEDING_HF_TOKEN=0 to make a missing token fail instead.
    if [ ${#missing[@]} -gt 0 ]; then
        echo "==> downloading $((${#missing[@]} / 2)) benchmark dataset(s) into {{ assets }} — slow, once"
        ( cd "{{ engine_dir }}" \
            && SCREAMINGFACE_SKIP_BENCHMARKS_NEEDING_HF_TOKEN="${SCREAMINGFACE_SKIP_BENCHMARKS_NEEDING_HF_TOKEN:-1}" \
                "$UV" run python -m screamingface_engine.benchmarks.prepare \
                --root "{{ assets }}" "${missing[@]}" )
    fi

    # Refuse the port BEFORE starting anything. `screamingface up` guards its own three and
    # says so plainly; this one had nothing, and the two ways it goes wrong are both silent.
    # A listener on the same address makes uvicorn exit with EADDRINUSE, which downstream
    # looks like a missing dependency. A listener on the OTHER address family (IPv6 `*` vs
    # our IPv4 127.0.0.1) does not collide at all: both bind, and which one answers is luck.
    if "$LSOF" -nP -iTCP:{{ engine_port }} -sTCP:LISTEN >/dev/null 2>&1; then
        echo "error: port {{ engine_port }} is already in use, so the Engine cannot start there." >&2
        echo "  Another worktree running this recipe is the usual cause. Either stop it, or" >&2
        echo "  pick a free port: SCREAMINGFACE_NOTEBOOK_ENGINE_PORT=9121 just …" >&2
        exit 1
    fi

    # Note whether a stack was ALREADY up, so the trap tears down only what this recipe
    # started — quitting a notebook must not kill a gateway someone started for other work.
    stack_was_running=false
    if "$UV" run --extra runtime screamingface status --json 2>/dev/null \
        | grep -q '"status":"running"'; then
        stack_was_running=true
    fi

    # `up` runs EITHER WAY, including over a stack that is already running. Skipping it then
    # looked harmless — it prints "already running" and returns — but `up` is where
    # `_ensure_adoptable` lives, the check that refuses to adopt a stack belonging to a
    # DIFFERENT checkout (OME-1001). `status` reports ownership and health; it does not
    # enforce them, so the shortcut silently ran notebooks against another branch's services.
    # --extra runtime: the stack's servers live in that optional extra, and the default
    # project env (notebook + dev only) cannot boot them.
    echo "==> checking the Gateway and Scoreboard (screamingface up)"
    "$UV" run --extra runtime screamingface up

    echo "==> starting an inspect-capable Engine on :{{ engine_port }}"
    # WHY full activity: the notebook's live activity panel shows only the Engine's activity
    # records, and a local Engine emits none unless URL4_CLOUD_ACTIVITY_LEVEL=full (default
    # "off", which exists for private deployments). Off here left the panel stuck on "No
    # structured activity received yet" for the whole run. Export it as "off" to opt out.
    ( cd "{{ engine_dir }}" \
        && URL4_BENCHMARK_ASSETS="{{ assets }}" \
            URL4_CLOUD_ACTIVITY_LEVEL="${URL4_CLOUD_ACTIVITY_LEVEL:-full}" \
            exec "$UV" run uvicorn \
            --factory screamingface_engine.local:create_local_app \
            --host 127.0.0.1 --port {{ engine_port }} ) &
    engine_pid=$!
    # Runs on any exit, including the Ctrl-C that ends Jupyter. Killing the `uv run` wrapper
    # takes uvicorn with it (verified), so the port is released.
    trap 'kill "$engine_pid" 2>/dev/null || true; \
          [ "$stack_was_running" = true ] || "$UV" run --extra runtime screamingface down || true' EXIT

    ready=false
    for _ in $(seq 60); do
        if "$CURL" -fsS --max-time 2 "http://127.0.0.1:{{ engine_port }}/v1/benchmarks" \
            >/dev/null 2>&1; then
            ready=true
            break
        fi
        if ! kill -0 "$engine_pid" 2>/dev/null; then
            echo "error: the Engine exited before it served anything." >&2
            echo "  Run the uvicorn command above by hand to see why." >&2
            exit 1
        fi
        sleep 1
    done
    # WHY an explicit failure: without it the loop simply ran out and fell through, opening
    # Jupyter against an Engine that never answered.
    if [ "$ready" != true ]; then
        echo "error: the Engine is still not answering on :{{ engine_port }} after 60s." >&2
        exit 1
    fi

    # Explicit URL beats local-stack adoption (OME-998), so the kernel reaches THIS Engine.
    #
    # The Scoreboard has to be named too, and for a reason easy to miss: the SDK adopts the
    # local stack ONLY when no engine URL is set, so naming the Engine turns that adoption
    # off for BOTH services and the Scoreboard silently falls back to the hosted leaderboard.
    # `sf.leaderboards.submit(...)` from a notebook would then publish to the deployment
    # instead of the Scoreboard this recipe just started — the "local engine + hosted
    # leaderboard" hybrid `_default_client` warns about in its own comment. Read from
    # `status` rather than assuming 9106, so a port override still lands.
    export SCREAMINGFACE_ENGINE_URL="http://127.0.0.1:{{ engine_port }}"
    export SCREAMINGFACE_SCOREBOARD_URL="$(
        "$UV" run --extra runtime screamingface status --json \
            | "$UV" run python -c "import json,sys;print(json.load(sys.stdin)['services']['scoreboard']['url'])"
    )"
    echo "==> Engine ready at $SCREAMINGFACE_ENGINE_URL"
    echo "==> Scoreboard at $SCREAMINGFACE_SCOREBOARD_URL — opening the examples"
    # --extra inspect: `report.export(format="inspect")` writes a .eval log through inspect-ai,
    # so the kernel needs it importable. This is the SDK's exporter extra, NOT the Engine's
    # `benchmarks` extra (a different project) — the kernel writes a log, it never bakes assets.
    # Safe here precisely because this line does NOT ask for `runtime`: the stack's servers are
    # what conflict with inspect-ai, and they were started above in their own uv invocation.
    "$UV" run --extra inspect jupyter lab examples/

# Bless one board's e2e replay fixtures from the owner-held recordings (OME-964):
# capture → re-key → verified replay → slice → snapshot + manifest + golden under
# tests/e2e/fixtures/. Needs docker and prepared benchmark assets
# (`uv run screamingface prepare --all`); extra flags pass through (--limit,
# --expect-score, --expect-coverage, --max-snapshot-mb, and the judge re-key
# phases: --dump-judge-bodies / --judge-bodies / --judge-param).
# Example:
#   just e2e-bless draco-3pass openrouter/google/gemini-3-flash-preview \
#       <cache-dump.sql.gz> <eval_results.eval.jsonl>
[doc("Bless one board's e2e fixtures from an owner-held recording (dump + answers)")]
e2e-bless board model dump answers *flags: _bless-preconditions
    {{ blesser }} --board {{board}} --model {{model}} \
        --dump {{dump}} --answers {{answers}} {{flags}}

# Bless a FUSION board's e2e replay fixtures from the saved SDK report alone
# (OME-978) — no production dump, no answers file: the report carries every
# response text, and the capture→splice loop synthesizes the cache tape from it.
# The replay must reproduce the report's score/coverage/statuses exactly.
# Example:
#   just e2e-bless-report healthbench-worst30 <report.json>
[doc("Bless a FUSION board's e2e fixtures from its saved SDK report alone")]
e2e-bless-report board report *flags: _bless-preconditions
    {{ blesser }} --board {{board}} \
        --report {{report}} {{flags}}

# Bless from a FRESH cache dump (OME-1098) — one recorded through THIS checkout's
# gateway, so the keys are already correct: no re-key, no tape synthesis, any
# candidate shape (built for the ifeval CorrectiveLoop golden). Takes the pg_dump,
# the run's saved SDK report (the expected outcome) and the candidate spec JSON;
# the replay must reproduce the report's score/coverage/statuses AND rendered
# expression exactly. Example:
#   just e2e-bless-fresh ifeval <cache-dump.sql.gz> <report.json> <candidate.json> --limit 50
[doc("Bless from a FRESH cache dump recorded through this checkout's gateway")]
e2e-bless-fresh board dump report candidate *flags: _bless-preconditions
    {{ blesser }} --board {{board}} --dump-fresh \
        --dump {{dump}} --report {{report}} --candidate {{candidate}} {{flags}}

# Re-author one board's golden from its COMMITTED snapshot (OME-1094) — no
# recording needed: replays the board keylessly and writes the per-case failure
# codes into the golden. Refuses if the replayed expression / statuses / counters /
# score differ from the committed file (a refresh may only ADD failure codes).
# Needs docker and prepared benchmark assets. Example:
#   just e2e-refresh-golden healthbench-worst30
#
# Confirmed because it rewrites a COMMITTED golden in place, and a golden is a
# byte-exact promise: a mistyped board name would otherwise silently rewrite one.
# Without a TTY it refuses rather than hanging, so CI fails closed.
[confirm("Rewrite the committed golden for this board in place? [y/N]")]
[doc("Re-author one board's golden from its COMMITTED snapshot (rewrites it in place)")]
e2e-refresh-golden board *flags: _bless-preconditions
    {{ blesser }} --board {{board}} \
        --refresh-golden {{flags}}

# Record the GSM8K vote replay lane (OME-1289 E4a, EV-12): ONE LIVE RUN that SPENDS REAL MONEY
# through OPENROUTER_API_KEY (default 20 cases x 5 samples = 100 short calls, well under $0.10,
# 5-15 minutes) and writes a recording directory. Also needs SCREAMINGFACE_RECORD_PAID=1, docker and
# the prepared `inspect-gsm8k` bundle (`uv sync --extra benchmarks` in the engine, then its
# `benchmarks.prepare --bundle inspect-gsm8k`). It refuses before any spend unless the model takes a
# `seed`. `--out <dir>` is required; other flags: --samples, --temperature, --limit, --extract,
# --normalize. Example:
#   just e2e-record-reducer-lane openrouter/<seed-capable-model> --out ~/sf-recordings/gsm8k-vote
[confirm("This spends real OpenRouter money. Run the live recording? [y/N]")]
[doc("Record the GSM8K vote replay lane with ONE live run (spends money)")]
e2e-record-reducer-lane model *flags: _bless-preconditions
    {{ reducer_lane }} record --model {{model}} {{flags}}

# Bless the GSM8K vote replay lane from a recording (OME-1289 E4a, EV-12): FREE and keyless. Replays
# the recording, refuses unless it reproduces the saved report, then writes the three fixture files
# (snapshot, manifest, reducer golden) under tests/e2e/fixtures/. Example:
#   just e2e-bless-reducer-lane ~/sf-recordings/gsm8k-vote
[doc("Bless the GSM8K vote lane's e2e fixtures from a recording (free, keyless)")]
e2e-bless-reducer-lane recording *flags: _bless-preconditions
    {{ reducer_lane }} bless --recording {{recording}} {{flags}}

# Record the GSM8K Fusion regression lane (OME-1289 E4a, FR#13): LIVE RUNS THAT SPEND REAL MONEY
# through OPENROUTER_API_KEY (default 5 cases x 2 Fusion shapes, at most 35 short calls, well under
# $0.10). Also needs SCREAMINGFACE_RECORD_PAID=1, docker and the prepared `inspect-gsm8k` bundle. It
# refuses before any spend unless the model takes a `seed`. `--out <dir>` is required; other flags:
# --samples, --temperature, --limit. Example:
#   just e2e-record-fusion-lane openrouter/<seed-capable-model> --out ~/sf-recordings/gsm8k-fusion
[confirm("This spends real OpenRouter money. Run the live recording? [y/N]")]
[doc("Record the GSM8K Fusion regression lane with live runs (spends money)")]
e2e-record-fusion-lane model *flags: _bless-preconditions
    {{ fusion_lane }} record --model {{model}} {{flags}}

# Bless the GSM8K Fusion regression lane from a recording (OME-1289 E4a, FR#13): FREE and keyless.
# Replays the recording through a capture proxy, refuses unless it reproduces the saved reports
# (url4, outputs, request body digests), then writes the three fixture files (snapshot, manifest, fusion golden) under
# tests/e2e/fixtures/. Example:
#   just e2e-bless-fusion-lane ~/sf-recordings/gsm8k-fusion
[doc("Bless the GSM8K Fusion lane's e2e fixtures from a recording (free, keyless)")]
e2e-bless-fusion-lane recording *flags: _bless-preconditions
    {{ fusion_lane }} bless --recording {{recording}} {{flags}}

# Paid smoke of every Benchmark (OME-1275): boots the real stack (Postgres + gateway +
# engine) and spends real money through OPENROUTER_API_KEY: every Benchmark the engine
# lists × 2 Cases × a 3-model Fusion of flash-tier seeds, plus the frontier-model Judge
# calls of every LLM-judged Benchmark (frontierscience and HealthBench judge with
# gpt-5.4, DRACO and GDPval with gemini-3.1-pro, once per rubric criterion). `scope`
# narrows the shelf: all (default) · imported (the inspect evals) · hand-built (the
# Benchmarks written in the engine). `benchmarks` (comma-separated ids, e.g.
# `just screamingface test-paid-benchmarks all musique`) replaces the scope with exactly
# those; a name the engine does not list fails before any paid call. The test fails only on infrastructure failure
# codes, never on scores. Never wired into merge gates; the on-demand CI twin is
# .github/workflows/screamingface-paid-benchmark-smoke.yml.
[doc("Run the paid OpenRouter smoke over every Benchmark (spends money); scope: all | imported | hand-built; benchmarks: comma-separated ids, overrides scope")]
test-paid-benchmarks scope="all" benchmarks="":
    #!/usr/bin/env bash
    set -euo pipefail
    UV="{{ require('uv') }}"
    DOCKER="{{ require('docker') }}"
    : "${OPENROUTER_API_KEY:?OPENROUTER_API_KEY must be set — the paid smoke needs a real key}"
    # Fail here, not minutes later inside a booted stack, if the daemon is down.
    "$DOCKER" info >/dev/null 2>&1 || { echo "error: docker daemon unreachable (Postgres testcontainer needs it)" >&2; exit 1; }

    # The Imported Benchmarks' extra is uncoinstallable with the SDK's runtime extra, so
    # the engine always runs from its own venv, synced here.
    ( cd "{{ engine_dir }}" && "$UV" sync --extra benchmarks )

    # Prepare any missing bundle, whatever is picked: one assets root serves every
    # press, and a prepared bundle costs nothing next time. Same resumable rule as
    # local-stack-notebooks: a bundle counts as finished only when its cases.json
    # PARSES; anything else is debris from a dead run and is cleared before re-preparing.
    # AIDEV-NOTE: near-copy of that recipe's loop — fold both (and the workflow's third
    # copy) into one place when any of them changes again; candidate home: a
    # --only-missing flag on the engine preparer.
    #
    # WHY a temp file, not `done < <(...)` (PR #1035 review): a failed --list-bundles
    # inside process substitution does NOT trip `set -e` — the loop would see an empty
    # list, prepare nothing, and the run would end as a green no-op.
    bundles_file="$(mktemp)"
    trap 'rm -f "$bundles_file"' EXIT
    ( cd "{{ engine_dir }}" \
        && "$UV" run python -m screamingface_engine.benchmarks.prepare --list-bundles ) > "$bundles_file"
    missing=()
    while read -r bundle; do
        # A bundle id is a flat directory name; anything path-like would make the
        # rm -rf below reach outside the assets root. Refuse, never sanitize.
        case "$bundle" in */*|*..*) echo "error: suspicious bundle id '$bundle' from --list-bundles" >&2; exit 1 ;; esac
        if "$UV" run python -c "import json,sys;json.load(open(sys.argv[1]))" \
            "{{ assets }}/$bundle/cases.json" >/dev/null 2>&1; then
            continue
        fi
        rm -rf "{{ assets }}/${bundle:?}"
        missing+=("--bundle" "$bundle")
    done < "$bundles_file"
    if [ ${#missing[@]} -gt 0 ]; then
        echo "==> downloading $((${#missing[@]} / 2)) benchmark dataset(s) into {{ assets }} — slow, once"
        ( cd "{{ engine_dir }}" \
            && "$UV" run python -m screamingface_engine.benchmarks.prepare \
                --root "{{ assets }}" "${missing[@]}" )
    fi

    # One timestamped directory per press holds everything needed to debug it: the
    # gateway/engine logs, one full Report per Benchmark under reports/, and the junit
    # verdicts. Pinned here (not pytest's tmp dir, which keeps only the last three
    # sessions under an unprinted path) and printed below on pass AND fail.
    log_dir="{{ data_dir }}/paid-smoke-logs/$(date -u +%Y%m%dT%H%M%SZ)"
    mkdir -p "$log_dir"

    # REQUIRED: this recipe IS the paid button — an unavailable stack must FAIL the
    # run (a pytest skip exits 0 and would report a green no-op). -rs still prints
    # any skip reason from non-button contexts. No `-m paid` filter: the lane's free
    # gate tests run nowhere else (merge CI never collects tests/paid), so every press
    # runs them too — in well under a second.
    status=0
    SCREAMINGFACE_TEST_PAID=1 SCREAMINGFACE_PAID_REQUIRED=1 SCREAMINGFACE_E2E_ASSETS="{{ assets }}" \
        SCREAMINGFACE_PAID_LOG_DIR="$log_dir" SCREAMINGFACE_PAID_SCOPE="{{ scope }}" \
        SCREAMINGFACE_PAID_BENCHMARKS="{{ benchmarks }}" \
        "$UV" run pytest tests/paid -rs -v --junitxml "$log_dir/junit.xml" || status=$?
    echo "==> logs, per-Benchmark reports and verdicts: $log_dir"
    exit "$status"
