{% extends "base.html" %}
{% block title %}asiai — Benchmark{% endblock %}

{% block head %}
<link rel="stylesheet" href="/static/bench.css?v=1.0.0">
{% endblock %}

{% block content %}
<div class="page-header">
    <h2>Benchmark</h2>
    <p>Run cross-engine benchmarks with live progress</p>
</div>

<div class="bn-layout fade-in" id="bn-root" data-bench-running="{{ '1' if bench_running else '0' }}">
<div class="bn-main">

    <!-- Quick Bench hero -->
    <div class="bn-hero">
        <button type="button" id="quick-bench-btn" class="bn-run-btn bn-hero-btn"
                {% if bench_running %}disabled{% endif %}>
            <svg width="11" height="12" viewBox="0 0 11 12" fill="none" aria-hidden="true"><path d="M1 1.3v9.4c0 .8.9 1.3 1.6.9l7.4-4.7c.6-.4.6-1.4 0-1.8L2.6.4C1.9 0 1 .5 1 1.3Z" fill="currentColor"/></svg>
            <span>Quick Bench</span>
        </button>
        <div class="bn-hero-copy">
            <div class="bn-hero-title">One-click inference benchmark</div>
            <div class="bn-hero-meta">1 prompt &middot; 1 run &middot; ~15 s &middot; shareable card</div>
        </div>
        <div class="bn-hero-side">
            {% set reachable_engines = engines | selectattr('reachable') | list %}
            {% if reachable_engines | length == 1 %}
            <div>uses <span class="bn-mono-accent">{{ reachable_engines[0].name }}</span> &middot; first loaded model</div>
            {% elif reachable_engines %}
            <div>auto-detected engines &middot; first common loaded model</div>
            {% else %}
            <div>no engine reachable</div>
            {% endif %}
            <div class="bn-hero-sub">zero configuration &middot; <a href="https://asiai.dev/commands/bench/" target="_blank" rel="noopener">documentation</a></div>
        </div>
    </div>

    <!-- Divider -->
    <div class="bn-divider"><span class="bn-divider-label">or configure a full run</span></div>

    <!-- Bench type selector: 7 types as primary navigation -->
    <div class="bn-types" role="tablist" aria-label="Bench type">
        <div class="bn-type-group">
            <span class="bn-group-label">Performance</span>
            <div class="bn-chip-row">
                <button type="button" class="bn-chip active" data-btype="">throughput</button>
                <button type="button" class="bn-chip" data-btype="agentic">agentic</button>
                <button type="button" class="bn-chip" data-btype="burst">burst</button>
            </div>
        </div>
        <span class="bn-type-sep" aria-hidden="true"></span>
        <div class="bn-type-group">
            <span class="bn-group-label">Quality</span>
            <div class="bn-chip-row">
                <button type="button" class="bn-chip" data-btype="code">code</button>
                <button type="button" class="bn-chip" data-btype="language">language</button>
                <button type="button" class="bn-chip" data-btype="instruct">instruct</button>
                <button type="button" class="bn-chip" data-btype="thinking-ablation">thinking</button>
            </div>
        </div>
    </div>
    <div class="bn-type-desc"><span class="bn-caret">&rsaquo;</span> <span id="bn-type-desc"></span></div>

    <!-- Collapsed run summary (shown while running / after a run) -->
    <div class="bn-summary" id="bn-run-summary" hidden>
        <span class="bn-summary-pill" id="bn-summary-type">throughput</span>
        <span class="bn-summary-text" id="bn-summary-text"></span>
        <button type="button" class="bn-ghost-btn" id="bn-edit-rerun" hidden>Edit &amp; re-run &blacktriangleright;</button>
    </div>

    <!-- ══ Standard (throughput) form ══ -->
    <div class="bn-form-card" id="standard-form-card">
        <form id="bench-form">
            <!-- 01 TARGET -->
            <section class="bn-section">
                <div class="bn-section-head">
                    <span class="bn-step">01</span>
                    <span class="bn-section-title">Target</span>
                    <span class="bn-section-hint">engines &middot; model &middot; runs</span>
                </div>

                <div class="bn-field" id="engines-group">
                    <span class="bn-label">Engines</span>
                    <div class="bn-engine-grid">
                        {% for engine in engines %}
                        <label class="bn-engine-card">
                            <input type="checkbox" name="engines" value="{{ engine.name }}"
                                   {% if engine.reachable %}checked{% else %}disabled{% endif %}>
                            <span class="bn-engine-face">
                                <span class="bn-engine-row">
                                    <span class="fl-dot sm {{ 'st-running' if engine.reachable else 'st-stopped' }}"></span>
                                    <span class="bn-engine-name">{{ engine.name }}</span>
                                </span>
                                <span class="bn-engine-meta">{% if engine.reachable %}serving &middot; {{ engine.models | length }} model{{ 's' if engine.models | length != 1 }}{% else %}offline{% endif %}</span>
                            </span>
                        </label>
                        {% endfor %}
                    </div>
                    {% if not engines %}
                    <p class="bn-empty">No engines detected</p>
                    {% endif %}
                </div>

                <div class="bn-target-grid">
                    <div class="bn-field" id="model-group">
                        <span class="bn-label">Model</span>
                        <select name="model" class="bn-select" id="model-select">
                            <option value="">(auto-detect)</option>
                            {% for engine in engines %}
                            {% if engine.reachable %}
                            {% for model in engine.models %}
                            <option value="{{ model }}">{{ model }} ({{ engine.name }})</option>
                            {% endfor %}
                            {% endif %}
                            {% endfor %}
                        </select>
                        <input type="text" name="model_custom" class="bn-input" id="model-custom"
                               placeholder="or type a model name" style="margin-top: 0.5rem; display: none;">
                        <a href="#" id="model-custom-toggle" class="bn-mini-link">Type custom model name</a>
                        <div class="bn-note">blank = first loaded model common to the selected engines</div>
                    </div>
                    <div class="bn-field">
                        <span class="bn-label">Runs</span>
                        <div class="bn-seg" id="runs-seg">
                            <button type="button" data-val="1">1</button>
                            <button type="button" data-val="3" class="active">3</button>
                            <button type="button" data-val="5">5</button>
                            <button type="button" data-val="10">10</button>
                        </div>
                        {# Kept for the form contract (name="runs"); driven by the segments. #}
                        <input type="range" name="runs" class="bn-visually-hidden" min="1" max="10" value="3"
                               id="runs-range" tabindex="-1" aria-hidden="true">
                        <div class="bn-note bn-mono" id="runs-ci-note">CI95 on every number &middot; n=3</div>
                    </div>
                </div>
            </section>

            <!-- 02 OPTIONS -->
            <section class="bn-section">
                <div class="bn-section-head">
                    <span class="bn-step">02</span>
                    <span class="bn-section-title">Options</span>
                    <span class="bn-section-type">throughput</span>
                </div>

                <div class="bn-field">
                    <span class="bn-label">Prompts</span>
                    <div class="bn-suites">
                        {% for prompt in prompts %}
                        <label class="bn-suite">
                            <input type="checkbox" name="prompts" value="{{ prompt.name }}" checked>
                            <span class="bn-suite-box"></span>
                            <span class="bn-suite-label">{{ prompt.label }}</span>
                            <span class="bn-suite-meta">{{ prompt.max_tokens }} tok</span>
                        </label>
                        {% endfor %}
                    </div>
                </div>

                <div class="bn-opt-row">
                    <div class="bn-field bn-w-190">
                        <span class="bn-label">Context fill</span>
                        <select name="context_size" class="bn-select">
                            <option value="">None (default prompts)</option>
                            <option value="4k">4K tokens</option>
                            <option value="16k">16K tokens</option>
                            <option value="32k">32K tokens</option>
                            <option value="64k">64K tokens</option>
                        </select>
                    </div>
                </div>
            </section>

            <!-- 03 ADVANCED -->
            <details class="bn-adv" id="advanced-options">
                <summary>
                    <span class="bn-step bn-step-muted">03</span>
                    <span class="bn-section-title">Advanced</span>
                    <span class="bn-adv-note" id="std-adv-note">compare: off &middot; card + share on</span>
                    <span class="bn-adv-chevron">&blacktriangleright;</span>
                </summary>
                <div class="bn-adv-body">
                    <div class="bn-adv-col">
                        <span class="bn-label">Compare mode</span>
                        <label class="bn-switch">
                            <input type="checkbox" id="compare-toggle" name="compare_mode">
                            <span class="bn-switch-track"></span>
                            <span class="bn-switch-label">Compare multiple models</span>
                        </label>
                        <div class="bn-field" id="compare-group" style="display: none;">
                            <span class="bn-label">Compare models (select 2&ndash;8)</span>
                            <select name="compare_models" class="bn-select" id="compare-select" multiple
                                    size="4" style="min-height: 100px;">
                                {% for engine in engines %}
                                {% if engine.reachable %}
                                {% for model in engine.models %}
                                <option value="{{ model }}@{{ engine.name }}">{{ model }} @ {{ engine.name }}</option>
                                {% endfor %}
                                {% endif %}
                                {% endfor %}
                            </select>
                            <div class="bn-note">Hold Cmd/Ctrl to select multiple. Format: model@engine.</div>
                        </div>
                    </div>
                    <div class="bn-adv-col">
                        <span class="bn-label">Power &middot; card &middot; share</span>
                        {% if ioreport_power %}
                        <div class="bn-note" style="margin-top: 0;">Power monitoring: always-on via IOReport (no sudo needed)</div>
                        {% if power_available %}
                        <label class="bn-switch">
                            <input type="checkbox" name="power">
                            <span class="bn-switch-track"></span>
                            <span class="bn-switch-label">Cross-validate with powermetrics (sudo)</span>
                        </label>
                        {% endif %}
                        {% else %}
                        <label class="bn-switch">
                            <input type="checkbox" name="power" {% if not power_available %}disabled{% endif %}>
                            <span class="bn-switch-track"></span>
                            <span class="bn-switch-label">Measure power (tok/s per watt){% if not power_available %} &mdash; unavailable{% endif %}</span>
                        </label>
                        {% if not power_available %}
                        <div class="bn-note">Enable with: <code>echo "$USER ALL=(ALL) NOPASSWD: /usr/bin/powermetrics" | sudo tee /etc/sudoers.d/powermetrics</code></div>
                        {% endif %}
                        {% endif %}
                        <label class="bn-switch">
                            <input type="checkbox" name="card" checked>
                            <span class="bn-switch-track"></span>
                            <span class="bn-switch-label">Generate shareable card</span>
                        </label>
                        <label class="bn-switch">
                            <input type="checkbox" name="share" checked>
                            <span class="bn-switch-track"></span>
                            <span class="bn-switch-label">Share to community</span>
                            <span class="bn-switch-hint">api.asiai.dev</span>
                        </label>
                    </div>
                </div>
            </details>

            <!-- footer -->
            <div class="bn-footer">
                <button type="submit" id="bench-submit" class="bn-run-btn"
                        {% if bench_running %}disabled{% endif %}>
                    <svg width="10" height="11" viewBox="0 0 11 12" fill="none" aria-hidden="true"><path d="M1 1.3v9.4c0 .8.9 1.3 1.6.9l7.4-4.7c.6-.4.6-1.4 0-1.8L2.6.4C1.9 0 1 .5 1 1.3Z" fill="currentColor"/></svg>
                    <span id="bench-submit-label">Run throughput bench</span>
                </button>
                <span class="bn-est" id="std-est"></span>
                <button type="button" class="bn-ghost-btn" id="std-reset">Reset</button>
            </div>
        </form>
    </div>

    <!-- ══ Mode form (agentic / burst / code / language / instruct / thinking) ══ -->
    <div class="bn-form-card" id="mode-form-card" style="display: none;">
        <form id="mode-form">
            <!-- 01 TARGET -->
            <section class="bn-section">
                <div class="bn-section-head">
                    <span class="bn-step">01</span>
                    <span class="bn-section-title">Target</span>
                    <span class="bn-section-hint">engine &middot; model &middot; runs</span>
                </div>

                <div class="bn-field">
                    <span class="bn-label">Engine (one target)</span>
                    <div class="bn-engine-grid" id="mode-engine-cards">
                        {% for engine in engines %}
                        <button type="button" class="bn-engine-face{% if loop.first %} active{% endif %}"
                                data-engine="{{ engine.name }}">
                            <span class="bn-engine-row">
                                <span class="fl-dot sm {{ 'st-running' if engine.reachable else 'st-stopped' }}"></span>
                                <span class="bn-engine-name">{{ engine.name }}</span>
                            </span>
                            <span class="bn-engine-meta">{% if engine.reachable %}serving &middot; {{ engine.models | length }} model{{ 's' if engine.models | length != 1 }}{% else %}unreachable{% endif %}</span>
                        </button>
                        {% endfor %}
                    </div>
                    {% if not engines %}
                    <p class="bn-empty">No engines detected</p>
                    {% endif %}
                    {# Kept for the form contract (name="mode_engine"); driven by the cards. #}
                    <select name="mode_engine" class="bn-visually-hidden" id="mode-engine" tabindex="-1" aria-hidden="true">
                        {% for engine in engines %}
                        <option value="{{ engine.name }}">{{ engine.name }}{% if not engine.reachable %} (unreachable){% endif %}</option>
                        {% endfor %}
                    </select>
                </div>

                <div class="bn-target-grid">
                    <div class="bn-field">
                        <span class="bn-label">Model</span>
                        <select name="mode_model" class="bn-select" id="mode-model-select">
                            <option value="" selected>(auto — first loaded model)</option>
                            <option value="__custom__">Custom…</option>
                        </select>
                        <input type="text" class="bn-input" id="mode-model-custom"
                               placeholder="model name" style="margin-top: 0.5rem; display: none;">
                        <div class="bn-note">blank = first loaded model &middot; installed models load on first request</div>
                    </div>
                    <div class="bn-field">
                        <span class="bn-label">Runs</span>
                        <div class="bn-seg" id="mode-runs-seg">
                            <button type="button" data-val="1" class="active">1</button>
                            <button type="button" data-val="3">3</button>
                            <button type="button" data-val="5">5</button>
                            <button type="button" data-val="10">10</button>
                        </div>
                        {# Kept for the form contract (name="mode_runs"); driven by the segments. #}
                        <input type="range" name="mode_runs" class="bn-visually-hidden" min="1" max="10" value="1"
                               id="mode-runs-range" tabindex="-1" aria-hidden="true">
                        <div class="bn-note bn-mono" id="mode-runs-ci-note"></div>
                    </div>
                </div>
            </section>

            <!-- 02 OPTIONS (per-type groups) -->
            <section class="bn-section">
                <div class="bn-section-head">
                    <span class="bn-step">02</span>
                    <span class="bn-section-title">Options</span>
                    <span class="bn-section-type" id="mode-options-type"></span>
                </div>

                <!-- agentic -->
                <div class="bn-field mode-group" data-modes="agentic" style="display: none;">
                    <div class="bn-opt-row" style="align-items: center;">
                        <label class="bn-switch">
                            <input type="checkbox" name="agentic_skip_long">
                            <span class="bn-switch-track"></span>
                            <span class="bn-switch-label">Skip long phases</span>
                        </label>
                        <span class="bn-switch-hint">prefix-cache reuse &middot; cold &rarr; warm &middot; verdict + reuse % stamped on the card</span>
                    </div>
                </div>

                <!-- burst -->
                <div class="bn-field mode-group" data-modes="burst" style="display: none;">
                    <div class="bn-opt-row">
                        <div class="bn-w-190">
                            <span class="bn-label">Burst sizes <span style="text-transform: none; letter-spacing: 0;">(comma-separated)</span></span>
                            <input type="text" name="burst_sizes" class="bn-input" value="30,60">
                        </div>
                        <div class="bn-w-170">
                            <span class="bn-label">Max tokens per call</span>
                            <input type="number" name="burst_max_tokens" class="bn-input" value="64" min="1" max="4096">
                        </div>
                    </div>
                    <div class="bn-note bn-mono">p50 / p95 / p99 per size &middot; aggregate tok/s</div>
                </div>

                <!-- code -->
                <div class="bn-field mode-group" data-modes="code" style="display: none;">
                    <span class="bn-label">Suites</span>
                    <div class="bn-suites">
                        {% for suite in code_suites %}
                        <label class="bn-suite">
                            <input type="checkbox" name="code_suites" value="{{ suite }}"
                                   {% if suite in ('tool-call', 'recovery', 'thinking') %}checked{% endif %}>
                            <span class="bn-suite-box"></span>
                            <span class="bn-suite-label">{{ suite }}</span>
                        </label>
                        {% endfor %}
                    </div>
                    <div class="bn-note">deterministic gates run always &middot; LLM judge optional (Advanced)</div>
                </div>

                <!-- language -->
                <div class="bn-field mode-group" data-modes="language" style="display: none;">
                    <div class="bn-opt-row">
                        <div class="bn-w-190">
                            <span class="bn-label">Language</span>
                            <select name="language" class="bn-select">
                                {% for lang in languages %}
                                <option value="{{ lang }}">{{ lang }}</option>
                                {% endfor %}
                            </select>
                        </div>
                        <div class="bn-field">
                            <span class="bn-label">Suites</span>
                            <div class="bn-suites">
                                {% for suite in language_suites %}
                                <label class="bn-suite">
                                    <input type="checkbox" name="language_suites" value="{{ suite }}"
                                           {% if suite != 'fluency' %}checked{% endif %}>
                                    <span class="bn-suite-box"></span>
                                    <span class="bn-suite-label">{{ suite }}</span>
                                    {% if suite == 'fluency' %}<span class="bn-suite-note">needs judge</span>{% endif %}
                                </label>
                                {% endfor %}
                            </div>
                        </div>
                    </div>
                </div>

                <!-- instruct -->
                <div class="bn-field mode-group" data-modes="instruct" style="display: none;">
                    <span class="bn-label">Scenarios <span style="text-transform: none; letter-spacing: 0;">(none = default set)</span></span>
                    <div class="bn-suites">
                        {% for scenario in instruct_scenarios %}
                        <label class="bn-suite">
                            <input type="checkbox" name="instruct_scenarios" value="{{ scenario }}">
                            <span class="bn-suite-box"></span>
                            <span class="bn-suite-label">{{ scenario }}</span>
                        </label>
                        {% endfor %}
                    </div>
                </div>

                <!-- thinking-ablation has no extra options -->
                <div class="bn-field mode-group" data-modes="thinking-ablation" style="display: none;">
                    <div class="bn-note bn-mono" style="margin-top: 0;">thinking on/off ablation on a tool-call-stress load &middot; no extra options</div>
                </div>
            </section>

            <!-- 03 ADVANCED -->
            <details class="bn-adv" id="mode-advanced">
                <summary>
                    <span class="bn-step bn-step-muted">03</span>
                    <span class="bn-section-title">Advanced</span>
                    <span class="bn-adv-note" id="mode-adv-note">judge: off &middot; sampling: engine defaults</span>
                    <span class="bn-adv-chevron">&blacktriangleright;</span>
                </summary>
                <div class="bn-adv-body">
                    <div class="bn-adv-col mode-group" data-modes="code language" style="display: none;">
                        <span class="bn-label">LLM judge &middot; optional</span>
                        <input type="text" name="judge_url" class="bn-input"
                               placeholder="judge URL — blank = deterministic checks only">
                        <input type="text" name="judge_model" class="bn-input" placeholder="judge model">
                        <div class="bn-key-note">
                            <svg width="11" height="12" viewBox="0 0 10 12" fill="none" aria-hidden="true"><rect y="5" width="10" height="7" rx="1.5" fill="#52525b"/><path d="M2.5 5V3.5a2.5 2.5 0 0 1 5 0V5" stroke="#52525b" stroke-width="1.4" fill="none"/></svg>
                            <span>API key is read from the server env (<code>ASIAI_JUDGE_API_KEY</code>) &mdash; never sent from this form. Web judges must target loopback.</span>
                        </div>
                    </div>
                    <div class="bn-adv-col">
                        <span class="bn-label">Sampling params</span>
                        <div class="bn-presets" id="mode-presets">
                            <button type="button" class="bn-preset active" data-preset="">Engine defaults</button>
                            <button type="button" class="bn-preset" data-preset='{"temperature": 0, "seed": 42}'>Deterministic</button>
                            <button type="button" class="bn-preset" data-preset='{"temperature": 0.9, "top_p": 0.95}'>Creative</button>
                            <button type="button" class="bn-preset" data-preset='{"chat_template_kwargs": {"enable_thinking": false}}'>Thinking off</button>
                        </div>
                        <input type="text" name="mode_extra_body" class="bn-input" id="mode-extra-body"
                               placeholder='{"chat_template_kwargs": {"enable_thinking": false}}'>
                        <div class="bn-note">extra_body &mdash; optional JSON forwarded to the engine</div>
                    </div>
                </div>
            </details>

            <!-- footer -->
            <div class="bn-footer">
                <button type="submit" id="mode-submit" class="bn-run-btn"
                        {% if bench_running %}disabled{% endif %}>
                    <svg width="10" height="11" viewBox="0 0 11 12" fill="none" aria-hidden="true"><path d="M1 1.3v9.4c0 .8.9 1.3 1.6.9l7.4-4.7c.6-.4.6-1.4 0-1.8L2.6.4C1.9 0 1 .5 1 1.3Z" fill="currentColor"/></svg>
                    <span id="mode-submit-label">Run bench</span>
                </button>
                <span class="bn-est" id="mode-est"></span>
                <button type="button" class="bn-ghost-btn" id="mode-reset">Reset</button>
            </div>
        </form>
    </div>

    <!-- Status panel (idle / running / completed / error) -->
    <div class="bn-status" id="status-card">
        <div id="bench-status">
            {% if bench_running %}
            <div class="bn-status-row is-running">
                <span class="bn-dot-live"></span>
                <span class="bn-status-title">Running</span>
                <span class="bn-status-meta">benchmark in progress&hellip;</span>
            </div>
            {% else %}
            <div class="bn-status-row">
                <span class="bn-dot-idle"></span>
                <span class="bn-status-title">Ready</span>
                <span class="bn-status-meta">waiting &mdash; Quick Bench or Run bench &middot; progress + live log appear here</span>
            </div>
            {% endif %}
        </div>
    </div>

    <!-- Results area (mode report / standard results table) -->
    <div id="bench-results"></div>

    <!-- Charts -->
    <div class="card-grid-2" id="bench-charts-row" style="display: none;">
        <div class="card">
            <h3 class="card-title">Throughput (tok/s)</h3>
            <div id="bench-tok-chart" class="chart-container"></div>
        </div>
        <div class="card">
            <h3 class="card-title">Time to First Token (ms)</h3>
            <div id="bench-ttft-chart" class="chart-container"></div>
        </div>
    </div>

    <!-- Share section (populated by JS after a standard bench completes) -->
    <div id="bench-share-section" style="display: none;"></div>
</div>

<!-- Right rail -->
<aside class="bn-rail">
    <div class="bn-rail-panel">
        <div class="bn-rail-title">Engines</div>
        <div class="bn-rail-rows">
            {% for engine in engines %}
            <div class="bn-rail-row">
                <span class="fl-dot sm {{ 'st-running' if engine.reachable else 'st-stopped' }}"></span>
                <span class="bn-rail-engine">{{ engine.name }}</span>
                <span class="bn-rail-meta">{% if engine.reachable %}{{ engine.models | length }} model{{ 's' if engine.models | length != 1 }}{% else %}offline{% endif %}</span>
            </div>
            {% else %}
            <p class="bn-empty">No engines detected</p>
            {% endfor %}
        </div>
    </div>
    <div class="bn-rail-panel">
        <div class="bn-rail-head">
            <span class="bn-rail-title">Card conditions</span>
            <span class="bn-cond-badge" id="bn-cond-badge">auto</span>
        </div>
        <div class="bn-cond-rows" id="bn-cond-rows"></div>
        <div class="bn-rail-note">Stamped on every shareable card &mdash; a number never travels without its conditions.</div>
    </div>
</aside>
</div>
{% endblock %}

{% block scripts %}
{# Engine → models data for the mode model picker. |tojson inside a JSON
   script block only — NEVER inside an HTML attribute (it does not escape
   double quotes; see the 1.23.0 leaderboard chip regression). #}
<script type="application/json" id="mode-model-data">{{ engines | tojson }}</script>
<script src="/static/bench.js?v=1.0.0"></script>
{% endblock %}