tooltangle report

{{ report.model }} · {{ report.created_at[:16]|replace("T", " ") }} UTC

{{ "%.0f"|format(accuracy) }}%

of {{ "{:,}".format(report.evaluated) }} test messages went to the right tool.

95% interval {{ "%.0f"|format(low) }}–{{ "%.0f"|format(high) }}%

{{ report.tools|length }}tools
≈ {{ "{:,}".format(report.definition_tokens) }}tokens on every request
{{ report.failed }}requests failed

Findings{{ report.findings|length }}

{{ {"error": "Error", "warn": "Warning", "info": "Info"}[finding.severity] }}

{{ finding.title }}{% if finding.detail %}{{ finding.detail }}{% endif %}
{% for example in finding.examples %}
{{ example }}
{% endfor %}
Info
Nothing to report.

{% if matrix.labels %}

Confusion matrix

Rows are the tool a message was written for, columns the tool the model called.

{% for label in matrix.labels %}{% endfor %} {% for expected in matrix.labels %} {% for picked in matrix.labels %} {% set count = matrix.cells.get(expected ~ "|" ~ picked, 0) %} {% if not count %} {% elif expected == picked %} {% else %} {% endif %} {% endfor %} {% endfor %}

	{{ label }}
{{ expected }}		{{ count }}	{{ count }}

correct {% for step in range(1, 6) %}{% endfor %} fewer to more confusions

{% endif %}

Tools{{ tools|length }}

{% for key, tool in tools %} {% endfor %}

Tool	Accuracy	Precision	Messages
`{{ key }}` {{ tool.description or "no description" }}	{% if tool.accuracy is not none %} {{ "%.0f%%"\|format(tool.accuracy * 100) }} {% if tool.accuracy < 0.8 %}below 80%{% endif %} {% else %}–{% endif %}	{{ "%.0f%%"\|format(tool.precision * 100) if tool.precision is not none else "–" }}	{{ tool.queries }}

Messages{{ report.queries|length }}

{{ row.text }}

for {{ row.tool or "no tool" }} {% if row.error %} · request failed {% else %} · called {{ row.picked|pick }} {% endif %} {% if row.accepted|length > 1 %}· also fine: {{ row.accepted|reject("equalto", row.tool)|join(", ") }}{% endif %} {{ row.split }}