{% extends "base.html" %}
{% block title %}LLM Evaluations - Admin - craft-dashboard{% endblock %}
{% block head %}
<script src="/static/vendor/chart.umd.min.js"></script>
{% endblock %}
{% block content %}
<h2>Admin</h2>
{% include "admin/partials/tabs.html" %}

<div class="row--25-75">
  <div class="col">
    <h3>LLM token usage</h3>
    <p>Token consumption by the LLM evaluation pipeline.</p>
    <div style="font-size: 0.875rem; display: flex; flex-direction: column; gap: 0.25rem; margin-top: 1rem;">
      <a href="/issues?llm_status=no_llm&state=open">View issues with no LLM data</a>
      <a href="/issues?llm_status=partial_llm&state=open">View issues with incomplete LLM data</a>
    </div>
  </div>
  <div class="col">
    <h4>Last 7 days</h4>
    <table>
      <thead>
        <tr>
          <th>Evaluations</th>
          <th>Total Tokens</th>
          <th>Input Tokens</th>
          <th>Output Tokens</th>
          <th>Embedding Tokens</th>
          <th>Avg Tokens per Eval</th>
        </tr>
      </thead>
      <tbody>
        <tr>
          <td>{{ "{:,}".format(recent_evaluations) }}</td>
          <td>{{ "{:,}".format(recent_tokens) }}</td>
          <td>{{ "{:,}".format(recent_prompt_tokens) }}</td>
          <td>{{ "{:,}".format(recent_completion_tokens) }}</td>
          <td>{{ "{:,}".format(recent_embedding_tokens) }}</td>
          <td>{{ "{:,.0f}".format(recent_tokens / recent_evaluations) if recent_evaluations > 0 else "N/A" }}</td>
        </tr>
      </tbody>
    </table>
    <h4>Lifetime</h4>
    <table>
      <thead>
        <tr>
          <th>Evaluations</th>
          <th>Total Tokens</th>
          <th>Input Tokens</th>
          <th>Output Tokens</th>
          <th>Embedding Tokens</th>
          <th>Avg Tokens per Eval</th>
        </tr>
      </thead>
      <tbody>
        <tr>
          <td>{{ "{:,}".format(total_evaluations) }}</td>
          <td>{{ "{:,}".format(total_tokens) }}</td>
          <td>{{ "{:,}".format(total_prompt_tokens) }}</td>
          <td>{{ "{:,}".format(total_completion_tokens) }}</td>
          <td>{{ "{:,}".format(total_embedding_tokens) }}</td>
          <td>{{ "{:,.0f}".format(total_tokens / total_evaluations) if total_evaluations > 0 else "N/A" }}</td>
        </tr>
      </tbody>
    </table>
  </div>
</div>

<hr class="p-rule" />

<div class="row--25-75">
  <div class="col">
    <h3>LLM evaluation service</h3>
    <p>Status of the continuous <code>evaluate</code> worker, and its recent output.</p>
    {% set llm_status_labels = {
      "running": "Running",
      "stalled": "Stalled",
      "stalled_quota": "Stalled (quota reached)",
      "unknown": "Unknown",
    } %}
    <div class="status-grid">
      <div class="status-card">
        <h4>Status</h4>
        <p>
          <span class="system-status-indicator" data-llm-service-status="{{ llm_service_status.status }}">
            <span class="system-status-dot status-dot--{{ llm_service_status.status }}"></span>
            <span>{{ llm_status_labels.get(llm_service_status.status, llm_service_status.status) }}</span>
          </span>
        </p>
        {% if llm_service_status.status == "stalled_quota" %}
        <p class="status-card-subtext">Resumes {{ llm_service_status.quota_resume_at | local_datetime }}</p>
        {% endif %}
      </div>
      <div class="status-card">
        <h4>Last poll</h4>
        <p>{{ llm_service_status.last_poll_at | local_datetime("Never") }}</p>
      </div>
      <div class="status-card">
        <h4>Last result</h4>
        <p>{{ llm_service_status.last_result_at | local_datetime("Never") }}</p>
      </div>
    </div>
    <h4 style="margin-top: 1rem;">Issues needing evaluations</h4>
    <table>
      <thead>
        <tr>
          <th>Never evaluated</th>
          <th>Eval version outdated</th>
          <th>Content changed since last eval</th>
        </tr>
      </thead>
      <tbody>
        <tr>
          <td>{{ "{:,}".format(outdated_evaluation_counts.never_evaluated) }}</td>
          <td>{{ "{:,}".format(outdated_evaluation_counts.version_outdated) }}</td>
          <td>{{ "{:,}".format(outdated_evaluation_counts.content_changed) }}</td>
        </tr>
      </tbody>
    </table>
  </div>
  <div class="col">
    <h4>Daily evaluations (last 14 days)</h4>
    {% if llm_daily_stats %}
    <table>
      <thead>
        <tr>
          <th>Date</th>
          <th>Evaluations</th>
        </tr>
      </thead>
      <tbody>
        {% for day in llm_daily_stats %}
        <tr>
          <td>{{ day.date }}</td>
          <td>{{ "{:,}".format(day.count) }}</td>
        </tr>
        {% endfor %}
      </tbody>
    </table>
    {% else %}
    <p>No evaluations recorded yet.</p>
    {% endif %}
    <h4 style="margin-top: 1rem;">Recent evaluations</h4>
    {% include "admin/_recent_evaluations_rows.html" %}
  </div>
</div>

<hr class="p-rule" />

<div class="row">
  <div class="col-12">
    <h3>Eval queue depth over time</h3>
    <p>
      Pending issues waiting for evaluation, sampled when workers poll
      <code>/api/eval/next</code>. Only records data while an evaluation worker is actively running.
    </p>
    {% if llm_queue_depth_history %}
    <div style="position: relative; width: 100%; height: 320px;">
      <canvas id="queue-depth-chart" style="width: 100%; height: 100%;"></canvas>
    </div>
    <script>
      (function () {
        const rawTimestamps = {{ llm_queue_depth_history | map(attribute="captured_at") | map("string") | list | tojson }};
        const labels = rawTimestamps.map(function (ts) {
          const d = new Date(ts);
          return isNaN(d) ? ts.slice(0, 10) : d.toLocaleDateString(undefined, { month: 'short', day: 'numeric', year: 'numeric' });
        });
        const pending = {{ llm_queue_depth_history | map(attribute="pending_count") | list | tojson }};
        new Chart(document.getElementById("queue-depth-chart"), {
          type: "line",
          data: {
            labels: labels,
            datasets: [
              {
                label: "Pending",
                data: pending,
                borderColor: "#c7162b",
                backgroundColor: "rgba(199, 22, 43, 0.1)",
                tension: 0.2,
                pointRadius: 0,
              },
            ],
          },
          options: {
            responsive: true,
            maintainAspectRatio: false,
            interaction: { mode: "index", intersect: false },
            plugins: {
              tooltip: {
                callbacks: {
                  title: function (items) {
                    if (!items.length) return "";
                    const d = new Date(rawTimestamps[items[0].dataIndex]);
                    return isNaN(d) ? rawTimestamps[items[0].dataIndex] : d.toLocaleString();
                  },
                },
              },
            },
            scales: {
              x: { ticks: { maxTicksLimit: 8 } },
              y: {
                type: "linear",
                display: true,
                beginAtZero: true,
                title: { display: true, text: "Pending" },
              },
            },
          },
        });
      })();
    </script>
    {% else %}
    <p>No queue depth samples recorded yet.</p>
    {% endif %}
  </div>
</div>

<hr class="p-rule" />

<div class="row">
  <div class="col-12">
    <h3>Daily invalidations by signal</h3>
    {% if commit_scan_summary %}
    {% if commit_scan_summary.warn %}
    <div class="p-notification--caution">
      <div class="p-notification__content">
        <p class="p-notification__message">
          Commit scanner invalidated
          <strong>{{ commit_scan_summary.rolling_total }}</strong> issues in the
          last 7 days, above the configured warning threshold of
          {{ commit_scan_summary.warn_threshold }}. Review the signal mix below
          and consider lowering K / raising the similarity threshold.
        </p>
      </div>
    </div>
    {% else %}
    <p>
      Rolling 7-day total:
      <strong>{{ commit_scan_summary.rolling_total }}</strong> invalidations.
    </p>
    {% endif %}
    {% endif %}
    <p>
      Issues marked for re-evaluation per day by commit scanner signal type.
      Because an issue can match multiple signals in a single pass, the chart
      stacks signal matches, while the headline total above counts unique issues.
    </p>
    {% if commit_scan_history %}
    {% set total_inval = commit_scan_history | sum(attribute="qualified_ref") + commit_scan_history | sum(attribute="path") + commit_scan_history | sum(attribute="semantic") + commit_scan_history | sum(attribute="bare_ref") + commit_scan_history | sum(attribute="launchpad") %}
    {% if total_inval == 0 %}
    <p class="u-text--muted">
      <em>No invalidations were triggered across any signals in this period.</em>
    </p>
    {% endif %}
    <div style="position: relative; width: 100%; height: 320px;">
      <canvas id="invalidations-chart" style="width: 100%; height: 100%;"></canvas>
    </div>
    <script>
      (function () {
        const labels = {{ commit_scan_history | map(attribute="day") | list | tojson }};
        new Chart(document.getElementById("invalidations-chart"), {
          type: "bar",
          data: {
            labels: labels,
            datasets: [
              {
                label: "Qualified ref",
                data: {{ commit_scan_history | map(attribute="qualified_ref") | list | tojson }},
                backgroundColor: "#0e8420",
              },
              {
                label: "Path",
                data: {{ commit_scan_history | map(attribute="path") | list | tojson }},
                backgroundColor: "#c7162b",
              },
              {
                label: "Semantic",
                data: {{ commit_scan_history | map(attribute="semantic") | list | tojson }},
                backgroundColor: "#0072d9",
              },
              {
                label: "Bare ref",
                data: {{ commit_scan_history | map(attribute="bare_ref") | list | tojson }},
                backgroundColor: "#7c4dff",
              },
              {
                label: "Launchpad",
                data: {{ commit_scan_history | map(attribute="launchpad") | list | tojson }},
                backgroundColor: "#e95420",
              },
            ],
          },
          options: {
            responsive: true,
            maintainAspectRatio: false,
            scales: {
              x: { stacked: true, ticks: { maxTicksLimit: 14 } },
              y: { stacked: true, beginAtZero: true },
            },
          },
        });
      })();
    </script>
    {% else %}
    <p>No commit scan runs recorded yet.</p>
    {% endif %}
  </div>
</div>

<hr class="p-rule" />

<div class="row">
  <div class="col-12">
    <h3>Evaluation API Tokens</h3>
    <p>Registered API tokens used for evaluation workers and background runs.</p>
    {% if eval_tokens %}
    <table>
      <thead>
        <tr>
          <th>ID</th>
          <th>Name</th>
          <th>Prefix</th>
          <th>Permissions</th>
          <th>Status</th>
          <th>Evaluations</th>
          <th>Last Used</th>
          <th>Action</th>
        </tr>
      </thead>
      <tbody>
        {% for token in eval_tokens %}
        <tr>
          <td><code>{{ token.id }}</code></td>
          <td>{{ token.name }}</td>
          <td><code>{{ token.token_prefix }}</code></td>
          <td>{{ token.permissions | join(", ") }}</td>
          <td>
            {% if token.is_active %}
            <span class="p-status-label--positive">Active</span>
            {% else %}
            <span class="p-status-label--negative">Revoked</span>
            {% endif %}
          </td>
          <td style="text-align: right;">{{ "{:,}".format(token_stats.get(token.id, 0)) }}</td>
          <td>{{ token.last_used_at | local_datetime if token.last_used_at else "Never" }}</td>
          <td>
            {% if token.is_active %}
            <form method="POST" action="/admin/keys/revoke" style="margin: 0;" onsubmit="return confirm('Revoke evaluation token {{ token.id }}?');">
              <input type="hidden" name="token_id" value="{{ token.id }}">
              <button type="submit" class="p-button--negative p-button--inline u-no-margin--bottom" style="font-size: 0.75rem; padding: 0.25rem 0.5rem;">Revoke</button>
            </form>
            {% else %}
            <span class="u-text--muted">—</span>
            {% endif %}
          </td>
        </tr>
        {% endfor %}
      </tbody>
    </table>
    {% else %}
    <p>No evaluation tokens found.</p>
    {% endif %}
  </div>
</div>
{% endblock %}