{% extends "base.html" %}
{% block title %}New run · kneebench{% endblock %}
{% block main %}
<div class="eyebrow">New benchmark</div>
<h1>Benchmark a server</h1>
<p class="muted">kneebench only sends requests to your existing server. It never starts, stops or
  changes it.</p>

<form id="run-form" method="post" action="/runs" hx-post="/runs" hx-target="#start-result"
      hx-indicator="#start-spin">
  <section class="card">
    <h3><span class="step-num">01</span>Connect</h3>
    <p class="muted small" style="margin-top:0">Fields marked <span class="req" aria-hidden="true">*</span> are required; the rest start at sensible defaults.</p>
    <div class="row"><label for="url">Endpoint URL<span class="req" aria-hidden="true">*</span><span class="tip" tabindex="0" role="note" data-tip="The root address of your vLLM server; kneebench adds /v1/chat/completions and /metrics itself.">?</span></label>
      <div><input id="url" name="url" required placeholder="http://my-vllm-host:8000" value="{{ defaults.url or '' }}"
                  oninput="lockSteps('The URL changed. Detect the server again to continue.')"></div></div>
    <div class="row"><label for="api_key">API key<span class="tip" tabindex="0" role="note" data-tip="Only if your server needs one. Used for this run and never saved, logged or written into reports.">?</span></label>
      <div><input id="api_key" name="api_key" type="password" autocomplete="off" placeholder="optional"></div></div>
    <div class="row"><span></span><div>
      <button type="button" class="secondary" hx-post="/detect" hx-include="#url,#api_key"
              hx-target="#detected" hx-indicator="#detect-spin">Detect server</button>
      <span id="detect-spin" class="htmx-indicator muted small"><span class="spinner"></span> Detecting…</span></div></div>
  </section>

  <div id="detected">
    <section class="card"><h3><span class="step-num">02</span>What kneebench found</h3>
      <p class="muted small" style="margin-bottom:0">Press <b>Detect server</b> to read the vLLM version, model,
        context length and KV cache from the endpoint. The rest of the form unlocks once the server answers.</p></section>
  </div>

  <fieldset id="after-detect" class="locked" disabled>

  <section class="card">
    <h3><span class="step-num">03</span>Hardware and flags</h3>
    <div class="row"><label for="gpu">GPU<span class="req" aria-hidden="true">*</span><span class="tip" tabindex="0" role="note" data-tip="The GPU the server runs on. Used for the physics limits; you can correct it after the run without rerunning.">?</span></label>
      <div><select id="gpu" name="gpu" required>{% for g in gpus %}<option value="{{ g.id }}">{{ g.name }} · {{ "%.0f"|format(g.memory_gib) }} GiB · {{ g.bandwidth_tbps }} TB/s</option>{% endfor %}</select></div></div>
    <div class="row"><label for="gpu_count">GPUs per replica<span class="req" aria-hidden="true">*</span><span class="tip" tabindex="0" role="note" data-tip="How many GPUs one copy of the model is split across: your --tensor-parallel-size.">?</span></label>
      <div><input id="gpu_count" name="gpu_count" type="number" min="1" value="1" required></div></div>
    <div class="row"><label for="max_num_seqs">--max-num-seqs<span class="tip" tabindex="0" role="note" data-tip="The most requests your server runs at once. Leave empty if you never set it; kneebench uses the default or infers it.">?</span></label>
      <div><input id="max_num_seqs" name="max_num_seqs" type="number" min="1" placeholder="empty: the default for your vLLM and GPU"></div></div>
    <div class="row"><label>--enforce-eager<span class="tip" tabindex="0" role="note" data-tip="Whether the server was started with --enforce-eager, which turns off CUDA graphs and slows each token.">?</span></label>
      <div class="choices">
        <label><input type="radio" name="enforce_eager" value="" checked> Not sure</label>
        <label><input type="radio" name="enforce_eager" value="false"> Not set (CUDA graphs on)</label>
        <label><input type="radio" name="enforce_eager" value="true"> Set</label></div></div>
  </section>

  <section class="card">
    <h3><span class="step-num">04</span>Your traffic<span class="req" aria-hidden="true">*</span><span class="tip" tabindex="0" role="note" data-tip="Pick the closest match. It sets the latency targets below and which findings come first.">?</span></h3>
    <p class="muted small">This decides which shape is swept and which findings come first in the report.</p>
    {% for key, p in profiles.items() %}
    <label class="choice-card"><input type="radio" name="product" value="{{ key }}" {% if key == "chat" %}checked{% endif %}
      onchange="applyPreset(this.value)"> <b>{{ p.title }}</b>
      <span class="muted">{{ presets[key].explain }}</span></label>
    {% endfor %}
    <div class="row" style="margin-top:14px"><label>Typical request<span class="tip" tabindex="0" role="note" data-tip="Optional. Median prompt and answer length from your logs; it becomes the main size tested.">?</span></label>
      <div><div style="display:flex;gap:8px;align-items:center">
        <input name="shape_isl" type="number" min="1" placeholder="prompt tokens" style="max-width:160px">
        <span class="muted">in,</span>
        <input name="shape_osl" type="number" min="1" placeholder="output tokens" style="max-width:160px">
        <span class="muted">out</span></div></div></div>
  </section>

  <section class="card">
    <h3><span class="step-num">05</span>Latency targets</h3>
    <p class="muted small">Filled in from your traffic type; change them to your own, or clear one for no target.</p>
    <div class="row"><label for="slo_ttft_ms">Time to first token<span class="tip" tabindex="0" role="note" data-tip="How long a user waits before the answer starts, in milliseconds, with the network removed.">?</span></label>
      <div><input id="slo_ttft_ms" name="slo_ttft_ms" type="number" min="1" placeholder="ms, empty for no target"></div></div>
    <div class="row"><label for="slo_tpot_ms">Time per token<span class="tip" tabindex="0" role="note" data-tip="The gap between words once the answer streams, in milliseconds.">?</span></label>
      <div><input id="slo_tpot_ms" name="slo_tpot_ms" type="number" min="1" step="0.1" placeholder="ms, empty for no target">
        <div class="help" id="tpot-help">The gap between words once the answer streams.</div></div></div>
    <div class="row"><label for="slo_percentile">Met by<span class="tip" tabindex="0" role="note" data-tip="How many requests must meet the targets. P99 means the slowest 1 in 100 counts.">?</span></label>
      <div><select id="slo_percentile" name="slo_percentile"><option value="99">99 out of 100 requests (P99)</option><option value="90">90 out of 100 requests (P90)</option></select></div></div>
  </section>

  <section class="card">
    <h3><span class="step-num">06</span>What to run</h3>
    <div class="row"><label>Phases<span class="req" aria-hidden="true">*</span><span class="tip" tabindex="0" role="note" data-tip="What to measure. At least one; the default runs everything in about an hour.">?</span></label>
      <div class="choices">
        <label><input type="checkbox" name="phase_baseline" value="1" checked> Single-request baseline <span class="muted small">how close one request gets to the hardware limit</span></label>
        <label><input type="checkbox" name="phase_concurrency" value="1" checked> Concurrency sweep <span class="muted small">capacity, the three walls, and what to change</span></label>
        <label><input type="checkbox" name="phase_rate" value="1" checked> Request-rate sweep <span class="muted small">random and bursty arrivals; adds about 15 minutes</span></label>
        <label><input type="checkbox" name="phase_interference" value="1" checked> Long-prompt interference <span class="muted small">do long prompts freeze other users' answers? adds about 3 minutes</span></label>
        <label><input type="checkbox" name="phase_reuse" value="1" checked> Prompt reuse <span class="muted small">what shared system prompts, multi-turn chats and retrieved passages get from prefix caching; adds about 4 minutes</span></label>
        <label><input type="checkbox" name="phase_cancel" value="1" checked> Abandoned requests <span class="muted small">whether the server stops generating when a user leaves mid-answer; adds about 1 minute</span></label>
        <label><input type="checkbox" name="phase_shape" value="1" checked> Request-size sweep <span class="muted small">prompt and output length, and every shape at the same load; adds about 30 minutes</span></label>
      </div></div>
    <div class="row"><label for="max_concurrency">Highest concurrency<span class="req" aria-hidden="true">*</span><span class="tip" tabindex="0" role="note" data-tip="The most users in flight the sweep tries. It doubles from 1 and stops early once it finds the limits.">?</span></label>
      <div><input id="max_concurrency" name="max_concurrency" type="number" min="1" value="256" required></div></div>
    <div class="row"><label for="budget_min">Time budget<span class="tip" tabindex="0" role="note" data-tip="Optional limit in minutes. If the plan is longer, the highest loads are dropped.">?</span></label>
      <div><input id="budget_min" name="budget_min" type="number" min="1" placeholder="minutes, empty for no limit"></div></div>
    <div class="row"><label for="label">Label<span class="tip" tabindex="0" role="note" data-tip="Optional note to tell runs apart, for example before and after a change.">?</span></label>
      <div><input id="label" name="label" placeholder="e.g. before lowering max-num-seqs"></div></div>
    <div class="row"><span></span><div>
      <button type="button" class="secondary" hx-post="/plan" hx-include="#run-form" hx-target="#plan"
              hx-indicator="#plan-spin">Preview plan</button>
      <span id="plan-spin" class="htmx-indicator muted small"><span class="spinner"></span> Planning…</span></div></div>
    <div id="plan"></div>
  </section>

  <div style="display:flex;align-items:center;gap:12px;margin-top:20px">
    <button type="submit">Start benchmark</button>
    <span id="start-spin" class="htmx-indicator muted small"><span class="spinner"></span> Detecting the server and planning…</span>
  </div>
  <div id="start-result"></div>
  </fieldset>
</form>
{% endblock %}
{% block scripts %}
<script>
  function unlockSteps() {
    const f = document.getElementById("after-detect");
    f.disabled = false;
    f.classList.remove("locked");
  }
  function lockSteps(message) {
    const f = document.getElementById("after-detect");
    if (f.disabled) return;
    f.disabled = true;
    f.classList.add("locked");
    document.getElementById("detected").innerHTML =
      `<section class="card"><h3><span class="step-num">02</span>What kneebench found</h3>` +
      `<p class="muted small" style="margin-bottom:0">${message}</p></section>`;
  }
  const PRESETS = {{ presets_json | safe }};
  function applyPreset(product) {
    const p = PRESETS[product];
    document.getElementById("slo_ttft_ms").value = p.ttft_ms ?? "";
    document.getElementById("slo_tpot_ms").value = p.tpot_ms ?? "";
    document.getElementById("slo_percentile").value = String(p.percentile);
    const tps = p.tpot_ms ? Math.round(1000 / p.tpot_ms) : null;
    document.getElementById("tpot-help").textContent = tps
      ? `The gap between words once the answer streams. ${p.tpot_ms} ms is ${tps} tokens per second per user.`
      : "The gap between words once the answer streams.";
  }
  applyPreset("chat");
</script>
{% endblock %}