{{ ui_lang.iaa_title|default('Inter-Annotator Agreement') }}

← {{ ui_lang.iaa_admin_dashboard|default('Admin Dashboard') }}

{{ ui_lang.iaa_overlap_sample|default('Overlap sample') }} — {{ report.n_overlap_items }} item{{ '' if report.n_overlap_items == 1 else 's' }} {{ ui_lang.iaa_at_cap|default('at cap.') }} {{ ui_lang.iaa_meta_scored|default('Each schema is scored with the metrics appropriate to its annotation type.') }}

{% if not report.schemas %}

{{ ui_lang.iaa_empty_state|default('No overlap-sample items have reached their cap yet. Once the sample items have been annotated by the configured number of annotators, IAA will appear here.') }}

{% endif %} {% for schema_name, schema in report.schemas.items() %}

{{ schema_name }}{{ schema.kind }} · {{ schema.annotation_type }}

{# Scales, and which metrics belong to them, are classified in potato/server_utils/iaa/presentation.py. Doing it here meant the page inferred what a metric was from its value, which is how a count of skipped items came to be labelled "weak agreement". #} {% for scale in schema.scales %} {% if scale == 'kappa' %}

{{ ui_lang.iaa_scale_kappa_label|default('κ-family scale (κ, α, γ, σ):') }} <0.2 {{ ui_lang.iaa_band_poor|default('poor') }} · 0.2–0.4 {{ ui_lang.iaa_band_fair|default('fair') }} · 0.4–0.6 {{ ui_lang.iaa_band_moderate|default('moderate') }} · 0.6–0.8 {{ ui_lang.iaa_band_substantial|default('substantial') }} · >0.8 {{ ui_lang.iaa_band_strong|default('strong') }}.

{% elif scale == 'correlation' %}

{{ ui_lang.iaa_scale_correlation_label|default('Correlation / ICC:') }} {{ ui_lang.iaa_scale_correlation_note|default('ranges −1 to 1; closer to 1 is stronger agreement.') }}

{% elif scale == 'raw' %}

{{ ui_lang.iaa_scale_raw_label|default('Raw agreement (% agreement, Jaccard, F1):') }} {{ ui_lang.iaa_scale_raw_note|default('0 to 1; no chance correction.') }}

{% elif scale == 'span' %}

{{ ui_lang.iaa_scale_span_label|default('Span F1 / token κ:') }} {{ ui_lang.iaa_scale_span_note_pre|default('0 to 1 (κ can be negative);') }} span_f1_partial {{ ui_lang.iaa_scale_span_note_post|default('is the conventional reporting metric.') }}

{% elif scale == 'lower' %}

{{ ui_lang.iaa_scale_lower_label|default('Error and offset metrics (MAE, RMSE, mean offset):') }} {{ ui_lang.iaa_scale_lower_note|default('lower is better; 0 means perfect agreement. Not colour-banded, because the strong/weak bands would be inverted.') }}

{% elif scale == 'distribution' %}

{{ ui_lang.iaa_scale_distribution_label|default('Kolmogorov–Smirnov statistic:') }} {{ ui_lang.iaa_scale_distribution_note|default('0 to 1; how far the within-item offset distribution sits from the between-item one. A separation measure, not an agreement coefficient.') }}

{% elif scale == 'coverage' %}

{{ ui_lang.iaa_scale_coverage_label|default('Coverage:') }} {{ ui_lang.iaa_scale_coverage_note|default('0 to 1; the share of the task that was answered at all. Not agreement — full coverage says everyone answered, not that they answered the same. Read it as the denominator behind the scores above.') }}

{% endif %} {% endfor %}
{% for row in schema.rows %} {% if row.name not in ('n_items', 'n_annotators', 'n_aligned_items') %} {% endif %} {% endfor %}
{{ ui_lang.iaa_th_metric|default('Metric') }}{{ ui_lang.iaa_th_value|default('Value') }}
{{ row.name }} {% if row.is_text %}{{ row.display }}{% elif row.display %}{{ row.display }}{% endif %} {% if row.band %}{% set band_label = ui_lang.iaa_band_strong|default('strong') if row.band == 'strong' else ui_lang.iaa_band_weak|default('weak') %} — {{ band_label }} {{ ui_lang.iaa_agreement|default('agreement') }}{% endif %} {% if row.context %}{{ row.context }}{% endif %} {% if row.note %}{{ row.note }}{% endif %}
{{ schema.n_items }} {{ ui_lang.iaa_items|default('items') }} {% if schema.n_annotators is not none %} · {{ schema.n_annotators }} {{ ui_lang.iaa_annotators|default('annotators') }}{% endif %} {% if schema.n_aligned_items is not none %} · {{ schema.n_aligned_items }} {{ ui_lang.iaa_fully_aligned_items|default('fully-aligned items') }}{% endif %}
{% if schema.sweep %} {# The sweep is the finding, not an appendix: agreement flat across the parameter means annotators identify the same instant, and agreement that appears only at the loosest setting means the most anyone can claim is that something is wrong somewhere. Neither is visible from a single headline number. #}

{{ ui_lang.iaa_sweep_title|default('Sensitivity to the matching window') }}

{{ ui_lang.iaa_sweep_note|default('Every measure below depends on how far apart two marks may be and still count as the same event. The highlighted row is the headline setting; read down a column to see whether agreement survives a tighter window.') }}

{% for column in schema.sweep.columns %}{% endfor %} {% for row in schema.sweep.rows %} {% for cell in row.cells %} {% endfor %} {% endfor %}
{{ schema.sweep.parameter_label }}{{ column }}
{% if row.parameter is number %}{{ "%g"|format(row.parameter) }}{% else %}{{ row.parameter }}{% endif %} {% if row.is_headline %}{{ ui_lang.iaa_sweep_headline|default('headline') }}{% endif %} {{ cell.display }}{% if cell.footnote %}{{ cell.footnote }} — {{ cell.note }}{% endif %}
{% if schema.sweep.footnotes %}
    {% for note in schema.sweep.footnotes %}
  1. {{ note }}
    2. {% endfor %}
{% endif %} {% endif %}
{% endfor %}
{{ ui_lang.iaa_per_item_breakdown|default('Per-item breakdown') }} ({{ report['items']|length }} {{ ui_lang.iaa_items|default('items') }})
{% for iid, item in report['items'].items() %} {% endfor %}
{{ ui_lang.iaa_th_instance|default('Instance') }}{{ ui_lang.iaa_th_cap|default('Cap') }}{{ ui_lang.iaa_th_annotators|default('Annotators') }}
{{ iid }} {{ item.cap }} {{ item.annotators|join(', ') }}