{{ ui_lang.judge_heading|default('LLM-Judge ↔ Human Alignment') }}

{{ ui_lang.judge_intro|default("How well does the configured LLM judge agree with human gold labels? Cohen's κ and the confusion matrix are computed over instances that have both a human label and a judge verdict. Inspect disagreements, edit the judge rubric, and re-run to calibrate.") }} {{ ui_lang.judge_current_prompt_version|default('Current prompt version:') }} {{ report.prompt_version or "—" }}.

{% if report.prompt_versions %}

{{ ui_lang.judge_prompt_versions|default('Prompt versions') }}

{% if report.kappa_trend %} {% set t = report.kappa_trend %}
{{ ui_lang.judge_mean_kappa_drift|default('Mean κ drift:') }} {{ t.direction }} {{ '%.3f'|format(t.first) }} → {{ '%.3f'|format(t.last) }} ({{ '%+.3f'|format(t.delta) }} {{ (ui_lang.judge_over_versions|default('over {n} versions'))|replace('{n}', t.series|length|string) }})
{% endif %}
{% endif %} {% if report.eval_cards %}

{{ ui_lang.judge_eval_cards|default('Judge eval cards') }} {{ ui_lang.judge_bias_robustness|default('(bias & robustness)') }}

{{ ui_lang.judge_eval_cards_intro|default('Beyond agreement (κ): does the judge favor longer outputs, is its confidence calibrated, is it order-robust? A portable certificate to ship with the eval.') }}

{% for schema, card in report.eval_cards.items() %}
{{ schema }} {{ card.verdict }}
κ {{ '%.3f'|format(card.agreement.kappa) if card.agreement.kappa is not none else '—' }} {% if card.verbosity and card.verbosity.length_bias_excess is not none %} {{ ui_lang.judge_length_bias|default('length bias') }} {{ '%+.0f'|format(card.verbosity.length_bias_excess) }} {{ ui_lang.judge_chars|default('chars') }} {% endif %} {% if card.calibration and card.calibration.ece is not none %} ECE {{ '%.2f'|format(card.calibration.ece) }} {% endif %}
{% if card.concerns %}
    {% for c in card.concerns %}
  • {{ c }}
    • {% endfor %}
{% endif %}
{% endfor %}
{% endif %}

{{ ui_lang.judge_autocalibrate|default('Auto-calibrate') }}

{{ ui_lang.judge_autocalibrate_intro_1|default('Close the loop automatically: the instances where a human') }} {{ ui_lang.judge_corrected|default('corrected') }} {{ ui_lang.judge_autocalibrate_intro_2|default('the judge (human label ≠ judge label) are its most informative examples.') }} {{ ui_lang.judge_autocalibrate_intro_3|default('POST to') }} /admin/api/judge-alignment/autocalibrate {{ ui_lang.judge_autocalibrate_intro_4|default('to re-run the judge with those corrections injected as few-shot examples (leakage-guarded — an instance never sees its own correction), creating a new prompt version whose mean κ is compared against the baseline. Optional body:') }} {"max_corrections": 5, "max_per_schema": N}. {{ ui_lang.judge_autocalibrate_intro_5|default('The new version appears in') }} {{ ui_lang.judge_prompt_versions|default('Prompt versions') }} {{ ui_lang.judge_above|default('above.') }}

{% if not report.per_schema %}

{{ ui_lang.judge_empty_1|default('No judge predictions yet. Configure') }} judge_alignment {{ ui_lang.judge_empty_2|default('and POST to') }} /admin/api/judge-alignment/run {{ ui_lang.judge_empty_3|default('to generate verdicts.') }}

{% endif %} {% for schema, r in report.per_schema.items() %}
{% set band = 'band-none' if r.kappa is none else ('band-low' if r.kappa < 0.2 else ('band-mid' if r.kappa < 0.6 else 'band-high')) %}

{{ schema }}

{{ r.interpretation }}
{{ ui_lang.judge_cohens_kappa|default("Cohen's κ") }} {{ '%.3f'|format(r.kappa) if r.kappa is not none else '—' }}
{{ ui_lang.judge_agreement|default('Agreement') }} {{ (r.agreement_rate * 100)|round(1) }}%
{{ ui_lang.judge_compared_n|default('Compared (n)') }} {{ r.n }}
{{ ui_lang.judge_disagreements|default('Disagreements') }} {{ r.disagreements|length }}
{% if r.confusion %}
{{ ui_lang.judge_confusion_label|default('Confusion (rows = human, columns = judge)') }}
{% set cols = [] %} {% for h, js in r.confusion.items() %}{% for j in js.keys() %}{% if j not in cols %}{% set _ = cols.append(j) %}{% endif %}{% endfor %}{% endfor %}
{% for c in cols %}{% endfor %} {% for h, js in r.confusion.items() %} {% for c in cols %} {% endfor %} {% endfor %}
{{ ui_lang.judge_confusion_caption|default('Confusion matrix for') }} {{ schema }}: {{ ui_lang.judge_confusion_caption_2|default('rows are human gold labels, columns are judge labels.') }}
{{ ui_lang.judge_human_vs_judge|default('human ＼ judge') }}{{ c }}
{{ h }}{{ js.get(c, 0) }}
{% endif %} {% if r.disagreements %}
{{ ui_lang.judge_disagreements|default('Disagreements') }}
{% for d in r.disagreements %} {% endfor %}
{{ ui_lang.judge_col_instance|default('Instance') }}{{ ui_lang.judge_col_human|default('Human') }}{{ ui_lang.judge_col_judge|default('Judge') }}{{ ui_lang.judge_col_conf|default('Conf.') }}{{ ui_lang.judge_col_reasoning|default('Judge reasoning') }}{{ ui_lang.judge_col_actions|default('Actions') }}
{{ d.instance_id }} {{ d.human_label }} {{ d.judge_label }} {{ '%.2f'|format(d.judge_confidence) if d.judge_confidence is not none else '—' }} {{ d.reasoning }} {{ ui_lang.judge_open_adjudication|default('Adjudicate') }}
{% endif %}
{% endfor %}