"""Offline analysis only. Reads frozen inputs; writes separate analysis artifacts."""
import collections
import hashlib
import json
import math
from pathlib import Path
import re
import statistics as st
import sys
sys.path.insert(0, str(Path(__file__).resolve().parents[5]))
from benchmarks.financial_dependency_suite.heldout.aggregate import summarize

OUT = Path(__file__).resolve().parent
BASE = OUT.parents[1]
ROOT = BASE.parents[2]
RESULTS = BASE / "results" / OUT.name
METHODS = ("bge_google", "bge_omneum")
files = [RESULTS / name for name in ("manifest.json", "summary.json", "results.jsonl")]
before = {str(p.relative_to(ROOT)): hashlib.sha256(p.read_bytes()).hexdigest() for p in files}
# These annotations apply only to the reviewed immutable result bytes.
assert before[str(files[2].relative_to(ROOT))] == "40876dfc228a91acf8af7d6fee66d7c2f1930ed7a8cdbe7015b2a7ee4e3d3594"
rows = [json.loads(line) for line in files[2].read_text().splitlines()]
manifest = json.loads(files[0].read_text())
cases = sorted({r["case_id"] for r in rows})
by = {(r["case_id"], r["trial"], r["method"]): r for r in rows}
assert len(rows) == len(by) == 480 and len(cases) == 24
truths = {c: json.loads((BASE / "cases" / c / "ground_truth.json").read_text()) for c in cases}
inputs = {c: json.loads((BASE / "cases" / c / "case.json").read_text()) for c in cases}


def norm(s):
    return " ".join(re.sub(r"[^\w\s]", " ", s.lower()).split())


def manual(r):
    c, m, t = r["case_id"], r["method"], r["trial"]
    if c == "support_stale_runbook":
        return "generator_refusal", "Explicit Anthropic stop_reason=refusal; empty content; no answer to assess."
    if c == "research_independent_false_consensus" and m == "bge_omneum":
        return "substantively_incorrect", "Reports 240 occupied homes; census reference is 216. Caveats acknowledge missing census evidence but still give the wrong answer."
    if c == "support_resolution_conflict" and m == "bge_google" and t == 5:
        return "incomplete", "Correct rollback action and retained TLS verification, but explicitly cannot identify tb17; selected context lacks that revision."
    if r["answer_score"] == "unscored":
        reason = {
            "coding_package_fix_lineage": "Correct bounding and eviction fix with reuse enabled; inflected/rephrased wording fails exact required substrings.",
            "multi_agent_independent_error": "Correct 120-second delay; normalized 120-second is not required substring 120 seconds.",
            "support_resolution_conflict": "Correct tb18-to-tb17 rollback and keeping upstream_tls_verify enabled; wording lacks required substring verification.",
        }[c]
        return "correct", reason
    return "correct", "Recorded correct retained; detailed manual adjudication targets every unscored answer and failure."


audit = []
for r in rows:
    status, reason = manual(r)
    truth = truths[r["case_id"]]
    if r["answer_score"] != "generation_failure":
        s = norm(r["answer"])
        rule = truth["scoring"]
        score = ("incorrect" if any(norm(v) and norm(v) in s for v in rule["forbidden"])
                 else "correct" if all(norm(v) and norm(v) in s for v in rule["required"])
                 else "unscored")
        assert score == r["answer_score"]
    audit.append({"case_id": r["case_id"], "method": r["method"], "trial": r["trial"],
                  "recorded_score": r["answer_score"], "manual_status": status,
                  "reason": reason, "reference_answer": truth["answer"],
                  "selected": r["selected"], "answer": r["answer"],
                  "failure_type": r.get("failure_type"),
                  "evidence": r["selection"]["selected"], "expected_supporting_sources": truth["supporting_sources"], "generator_diagnostics": r["generation"]["response_diagnostics"]})

checks = collections.Counter()
contract = manifest["checkpoint_contract"]
assert contract["version"] == 1 and contract["trials"] == manifest["trials"] == 10
for key in ("inputs", "models", "estimator", "packages", "python", "selection_pipeline"):
    assert contract[key] == manifest[key]
assert [m["name"] for m in contract["methods"]] == list(METHODS)
assert contract["generator_provider"] == "anthropic"
assert contract["models"] == {"bge_google": "gemini-3.8-flash", "bge_omneum": "Omneum", "generator": "claude-sonnet-5-5"}
assert set(cases) == {Path(p).parent.name for p in manifest["inputs"]}
assert [(r["case_id"],r["method"],r["trial"]) for r in rows] == [(c,m,t) for c in cases for m in METHODS for t in range(1,11)]
checks["checkpoint_contract_and_complete_schedule"] = 1
summary = summarize(rows)
assert summary == json.loads(files[1].read_text())
(OUT / "summary.json").write_text(json.dumps(summary, indent=2) + "\n")
checks["regenerated_summary_matches_saved"] = 1
same_set = same_order = 0
for c in cases:
    for t in range(1, 11):
        a, o = (by[c, t, m] for m in METHODS)
        assert a["selection"]["bge_relevance"] == o["selection"]["bge_relevance"]
        checks["identical_paired_bge_results"] += 1
        same_set += set(a["selected"]) == set(o["selected"])
        same_order += a["selected"] == o["selected"]
        for r in (a, o):
            s = r["selection"]
            short = s["bge_relevance"]["selected"]
            texts = {d["source_id"]: d["text"] for d in inputs[c]["candidates"]}
            assert all(d["text"] == texts[d["source"]] for d in short + s["selected"])
            assert r["selected"] == [d["source"] for d in s["selected"]]
            assert r["answer"] == r["generation"]["text"]
            assert s["context_budget"] == inputs[c]["context_budget"]
            assert s["bge_latency_seconds"] == s["bge_relevance"]["latency_seconds"]
            if r["method"] == "bge_google":
                assert s["evaluator_model"] == "gemini-3.8-flash" and s["evaluator_provider"] == "google"
                assert s["parse_error"] is None
                assert json.loads(s["raw_output"])["selected_sources"] == r["selected"]
                u = s["response_diagnostics"]["usage_metadata"]
                assert u["prompt_token_count"] == s["evaluator_input_tokens"]
                assert u["candidates_token_count"] == s["evaluator_output_tokens"]
                assert u["reasoning_tokens"] == u["thoughts_token_count"]
                assert u["total_token_count"] == u["prompt_token_count"] + u["candidates_token_count"] + u["thoughts_token_count"]
            if r["answer_score"] == "generation_failure":
                assert r["failure_type"] == "refusal" and not r["answer"]
                assert r["generation"]["response_diagnostics"]["stop_reason"] == "refusal"
            checks["text_usage_and_response_contracts"] += 1
            assert hashlib.sha256(json.dumps(short, sort_keys=True, separators=(",", ":")).encode()).hexdigest() == s["bge_shortlist_hash"]
            checks["shortlist_hashes"] += 1
            assert len(short) == min(len(inputs[c]["candidates"]), 2 * inputs[c]["context_budget"])
            assert len(r["selected"]) == len(set(r["selected"])) == inputs[c]["context_budget"]
            assert set(r["selected"]) <= {d["source"] for d in short}
            checks["selection_contracts"] += 1
            context = "\n\n".join(f"Source: {d['source']}\n{d['text']}" for d in s["selected"])
            prompt = f"Answer the question using only the sources below.\n\nQuestion:\n\n{inputs[c]['question']}\n\nSources:\n\n{context}\n\nGive a concise answer and identify which sources support it.\n"
            payload = {"provider": "anthropic", "model": "claude-sonnet-5-5", "prompt": prompt,
                       "max_tokens": 1024, "response_schema": None}
            h = hashlib.sha256(json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode()).hexdigest()
            assert h == r["generation"].get("request_hash", r.get("generation_request_hash"))
            checks["generation_request_hashes"] += 1
            assert r["generation"]["provider"] == "anthropic" and r["generation"]["model"] == "claude-sonnet-5-5"
            judge = s["evaluator_latency_seconds"] if r["method"] == "bge_google" else s["omneum_latency_seconds"]
            assert math.isclose(s["latency_seconds"], s["bge_latency_seconds"] + judge)
            assert math.isclose(r["end_to_end_latency_seconds"], s["latency_seconds"] + r["generation"]["latency_seconds"])
            checks["latency_reconciliations"] += 1
            expected = set(truths[c]["supporting_sources"])
            tp = len(set(r["selected"]) & expected)
            score = r["selection_score"]
            assert (tp, len(set(r["selected"]) - expected), len(expected - set(r["selected"]))) == (score["true_positive"], score["false_positive"], score["false_negative"])
            assert math.isclose(score["precision"], tp / len(r["selected"]))
            assert math.isclose(score["recall"], tp / len(expected))
            checks["selection_scores"] += 1
for path, h in manifest["inputs"].items():
    p = Path(path)
    if p.is_absolute():
        # Preserve frozen absolute-path manifest bytes while permitting checkout relocation.
        relative = str(p).split("/benchmarks/financial_dependency_suite/heldout/", 1)[1]
        p = BASE / relative
    else:
        p = ROOT / p
    assert hashlib.sha256(p.read_bytes()).hexdigest() == h
    checks["frozen_input_hashes"] += 1


def percentile(values, q):
    a = sorted(values)
    pos = (len(a) - 1) * q
    lo = int(pos)
    return a[lo] + (a[min(lo + 1, len(a) - 1)] - a[lo]) * (pos - lo)


def metrics(rr):
    scores = collections.Counter(r["answer_score"] for r in rr)
    verified = collections.Counter(manual(r)[0] for r in rr)
    tp, fp, fn = (sum(r["selection_score"][k] for r in rr) for k in ("true_positive", "false_positive", "false_negative"))
    ei = sum(r["selection"]["evaluator_input_tokens"] for r in rr)
    eo = sum(r["selection"]["evaluator_output_tokens"] for r in rr)
    reasoning = sum(r["selection"].get("response_diagnostics", {}).get("usage_metadata", {}).get("thoughts_token_count", 0) for r in rr)
    gi = sum(r["generation"]["input_tokens"] for r in rr)
    go = sum(r["generation"]["output_tokens"] for r in rr)
    return {"n": len(rr), "recorded": dict(scores), "manual": dict(verified),
            "precision": st.mean(r["selection_score"]["precision"] for r in rr),
            "recall": st.mean(r["selection_score"]["recall"] for r in rr),
            "micro_precision": tp / (tp + fp), "micro_recall": tp / (tp + fn),
            "tp": tp, "fp": fp, "fn": fn,
            "selection_p50": st.median(r["selection"]["latency_seconds"] for r in rr),
            "selection_p95": percentile([r["selection"]["latency_seconds"] for r in rr], .95),
            "e2e_p50": st.median(r["end_to_end_latency_seconds"] for r in rr),
            "e2e_p95": percentile([r["end_to_end_latency_seconds"] for r in rr], .95),
            "generator_p50": st.median(r["generation"]["latency_seconds"] for r in rr),
            "google_reasoning": reasoning, "total_api_tokens": ei + eo + reasoning + gi + go, "google_input": ei, "google_output": eo, "anthropic_input": gi, "anthropic_output": go,
            "google_usd": (ei * .75 + (eo + reasoning) * 3.75) / 1e6,
            "anthropic_usd": (gi * 2 + go * 10) / 1e6,
            "total_usd": (ei * .75 + (eo + reasoning) * 3.75 + gi * 2 + go * 10) / 1e6}


aggregates = {m: metrics([r for r in rows if r["method"] == m]) for m in METHODS}
per_case = {c: {m: metrics([r for r in rows if r["case_id"] == c and r["method"] == m]) for m in METHODS} for c in cases}
per_trial = {t: {m: metrics([r for r in rows if r["trial"] == t and r["method"] == m]) for m in METHODS} for t in range(1, 11)}
a, o = (aggregates[m] for m in METHODS)
savings = {"selection_median_percent": 100 * (1 - o["selection_p50"] / a["selection_p50"]),
           "end_to_end_median_percent": 100 * (1 - o["e2e_p50"] / a["e2e_p50"]),
           "api_cost_percent": 100 * (1 - o["total_usd"] / a["total_usd"]),
           "api_token_percent": 100 * (1 - o["total_api_tokens"] / a["total_api_tokens"])}
(OUT / "comparison.json").write_text(json.dumps({"aggregates": aggregates, "per_case": per_case, "per_trial": per_trial,
    "savings": savings, "integrity_checks": checks, "same_selected_set_pairs": same_set,
    "same_order_and_generation_prompt_pairs": same_order, "source_file_sha256": before}, indent=2) + "\n")
(OUT / "answer_audit.json").write_text(json.dumps(audit, indent=2) + "\n")


assert collections.Counter((r['method'], r['case_id']) for r in rows if r['answer_score'] == 'unscored') == {
    ('bge_google', 'coding_package_fix_lineage'): 10,
    ('bge_omneum', 'coding_package_fix_lineage'): 10,
    ('bge_google', 'multi_agent_independent_error'): 4,
    ('bge_google', 'support_resolution_conflict'): 1,
    ('bge_omneum', 'research_independent_false_consensus'): 10,
}
pairs = [(by[c,t,METHODS[0]], by[c,t,METHODS[1]]) for c in cases for t in range(1,11)]
paired_outcomes = collections.Counter('tie' if (manual(x)[0]=='correct') == (manual(y)[0]=='correct') else 'gemini_win' if manual(x)[0]=='correct' else 'omneum_win' for x,y in pairs)
consistency = {m: {'identical_order_cases': sum(len({tuple(r['selected']) for r in rows if r['case_id']==c and r['method']==m})==1 for c in cases),
                    'identical_set_cases': sum(len({tuple(sorted(r['selected'])) for r in rows if r['case_id']==c and r['method']==m})==1 for c in cases)} for m in METHODS}
ceilings = {'case_budget_macro_recall': st.mean(min(1, inputs[c]['context_budget']/len(truths[c]['supporting_sources'])) for c in cases),
            'shortlist_macro_recall': st.mean(min(inputs[c]['context_budget'], len(set(truths[c]['supporting_sources']) & {d['source'] for d in by[c,t,METHODS[0]]['selection']['bge_relevance']['selected']}))/len(truths[c]['supporting_sources']) for c in cases for t in range(1,11))}
faster = {'selection': sum(y['selection']['latency_seconds'] < x['selection']['latency_seconds'] for x,y in pairs),
          'end_to_end': sum(y['end_to_end_latency_seconds'] < x['end_to_end_latency_seconds'] for x,y in pairs)}
comparison = json.loads((OUT/'comparison.json').read_text())
comparison.update(paired_complete_outcomes=paired_outcomes, selection_consistency=consistency, recall_ceilings=ceilings, omneum_faster_pairs=faster,
 pricing={'date_checked': '2026-10-08', 'usd_per_million': {'gemini_input': .75, 'gemini_visible_plus_thinking_output': 3.75, 'sonnet_input':2, 'sonnet_output':10},
 'google_source':'https://ai.google.dev/gemini-api/docs/pricing', 'anthropic_source':'https://platform.claude.com/docs/en/about-claude/pricing', 'basis':'standard paid tier; Gemini introductory rates through 2026-12-31'},
 implementation={'manifest':contract['implementation'], 'checkout_before_analysis':'00101f98f5610c3427fb2a2463a1cf774f9ad1011c2e7e9001dda05fa8552be1'})
(OUT/'comparison.json').write_text(json.dumps(comparison,indent=2)+'\n')
lines = [f'''This offline audit covers 480 saved records: 24 cases × ten trials × two selectors. Gemini 3.8 Flash and Omneum select context after the same BGE shortlist; Claude Sonnet 5.5 independently generates every answer. No model calls, trial reruns, raw-result edits, or methodology changes were made.

Complete-answer accuracy is **Gemini 229/240 (95.42%) versus Omneum 220/240 (91.67%)**, a 3.75 percentage-point observed advantage for Gemini. Omneum reduces median selection latency by {savings['selection_median_percent']:.2f}%, median end-to-end latency by {savings['end_to_end_median_percent']:.2f}%, estimated API cost by {savings['api_cost_percent']:.2f}%, and logged API tokens by {savings['api_token_percent']:.2f}%.

Recorded scores remain frozen: Gemini 215 correct, 15 unscored, ten generation failures; Omneum 210 correct, 20 unscored, ten generation failures. The existing aggregate.summarize infrastructure regenerated summary.json here, exactly matching the experiment's saved summary. Its evaluable_accuracy=100% excludes unscored answers and failures and must not be advertised as overall accuracy.

Adjudication follows the completed Astra comparison: judge the requested answer and essential reference conditions in meaning; credit synonymous/inflected wording; separate a correct action missing a necessary revision as incomplete; classify a conflicting numeric answer as substantively incorrect even with caveats; identify explicit generator refusals separately. Recorded correct answers are retained; this review is not a blinded human panel or an exhaustive ancillary-claim/citation audit. All 35 unscored answers and all 20 failures are individually reproduced below and in answer_audit.json, with references and selected evidence. Complete accuracy uses all 240 attempts, including refusals.

All 20 package answers correctly bound idle connections, evict expired entries, and keep reuse enabled. Gemini Retry-After trials 1, 2, 8, 9 correctly specify a 120-second wait; substring scoring misses the hyphenated wording. Gemini TLS trial 5 correctly rolls back and retains verification but cannot name tb17: incomplete, matching Astra's treatment of missing revision. Giving operational credit to this incomplete answer would yield Gemini 230/240 (95.83%); it is excluded from complete accuracy.

Integrity checks validate all 480 unique case/method/trial keys, the entire saved schedule (all Gemini trials before Omneum within each case), the checkpoint contract's duplicated fields, all 48 input hashes, all 240 identical paired BGE objects (IDs, order, text, scores, latency), 480 shortlist hashes, selection budgets and subset membership, source texts, selection scores, generation request hashes, usage telemetry, and latency sums. There are {same_set} pairs with equal selected sets and {same_order} with equal ordered context and generation prompts. Request hashes reconstruct the common Anthropic model, prompt template, max_tokens=1024, and response_schema=None. Both arms have complete token and latency telemetry. SHA-256 hashes below verify the manifest, results, and original summary were unchanged.

Material accuracy difference: research_independent_false_consensus

Gemini selects census dataset + neighborhood census summary in all ten trials, with one unrelated estimate filling the third slot, and correctly answers 216. Omneum selects analysis + report + publication in all ten and reports 240 rather than the reference 216 occupied homes. Census microdata explicitly distinguishes 216 occupied and 24 vacant addresses out of 240 total units. The true dataset is first in BGE's shortlist (score 0.9885); the census summary is also present. The signed primary record is outside the six-source shortlist, but the two available census sources suffice. This failure occurs in the reliability selection stage rather than initial retrieval.

The wrong sources use independent footprint, map, and facade counts. Independence does not establish that they measured occupancy or the specified census. Omneum removes both census sources despite their availability. Every generated answer notes the missing census confirmation; several nevertheless strengthen 240 using agreement between methods. Trial 10 is more guarded and reports 240 as the sources' figure while saying the requested census figure cannot be confirmed. Under the same Astra criterion it remains a substantive task failure: it substitutes/reports the wrong figure and supplies no correct census answer. It is not an API generator refusal. These caveats are preserved in the appendix rather than erased by the classification.

Current selector code applies reliability filtering to structured claims and provenance, then selects reliable context; Gemini sees shortlist text and source IDs. A preference for independent agreement over task-specific measurement is consistent with these saved selections, but raw evaluation is serialized only as <ContextEvaluation>; internal claim scores are unavailable, so a numerical causal diagnosis cannot be recovered without new execution. No such execution was performed. Dependency correction alone cannot resolve independently produced shared measurement errors. The TLS case offsets one of these ten paired losses for Omneum, leaving {dict(paired_outcomes)} under complete correctness.

Both arms explicitly refuse support_stale_runbook in all ten trials: Anthropic stop_reason=refusal with empty content. Both select the same current_runbook, incident_record, deployment_config, containing the correct POST /admin/keys/refresh action. This is a shared generator refusal, not evidence of a selector error, and diagnostics do not explain the reason. No other generation/selection failures are logged.

Measurement and generalization limits

These are 24 fixed synthetic cases with ten clustered repeats, not 240 independent tasks. The gap is concentrated in one false-consensus case, offset by one incomplete TLS answer; it does not establish general accuracy equivalence, superiority, or noninferiority. Omneum's deterministic selection includes consistently wrong context. Selection precision/recall use frozen source labels: macro averages records; micro pools TP/FP/FN. Useful diagnostic service_state is unlabeled in TLS. The macro case-budget recall ceiling is {ceilings['case_budget_macro_recall']:.2%}; the available-shortlist ceiling is {ceilings['shortlist_macro_recall']:.2%}. Recall is not sufficient-evidence accuracy.

Judge information differs: Omneum uses structured claims/provenance and relevant dependency observations, while Gemini uses text/IDs; the shortlist and final generator controls are paired, but the two selectors have different inputs. No supporting-source labels enter the selector path in current code. The manifest records a dirty checkout at {manifest['git']['commit']} with implementation fingerprint {contract['implementation']}. The pre-analysis checkout fingerprint 00101f98f5610c3427fb2a2463a1cf774f9ad1011c2e7e9001dda05fa8552be1 differs, so current source is explanatory, not a verified execution-time snapshot. The manifest retains its original absolute paths; this script resolves them without modifying it. Server-side defaults and undocumented retries cannot be established from saved request hashes.

Trials are scheduled in arm blocks per case, without randomization/interleaving; latency can reflect temporal provider variation. Selection median includes BGE plus full selector/integration time; Omneum core evaluator median is {st.median(r['selection']['evaluation_latency_seconds'] for r in rows if r['method']=='bge_omneum')*1000:.2f} ms, not total selection time. E2E is selection plus independent generation. All refusals are included. Percentiles use linear interpolation at (n−1)q. Omneum is faster in {faster['selection']}/240 selection pairs and {faster['end_to_end']}/240 end-to-end pairs.

Costs use standard paid-tier USD per million tokens: Gemini input $0.75, output including thinking $3.75 under introductory pricing through December 31, 2026, checked October 8 in [Google pricing](https://ai.google.dev/gemini-api/docs/pricing); Sonnet 5.5 input $2, output $10 in [Anthropic pricing](https://platform.claude.com/docs/en/about-claude/pricing). Rates are frozen in comparison.json and this script for offline reproducibility. Google candidates_token_count excludes thoughts_token_count here: 4,980 visible output + 216,106 reasoning = 221,086 billable output tokens, counted exactly once. Each of 240 Google total_token_count values reconciles to input + visible output + thoughts. The existing summary reports only visible evaluator output; its zero/low output totals do not represent full Google billable output. No methodology code was changed.

These are estimates, not invoices. Raw inference_cost is unavailable. Free-tier eligibility, promotional credits, discounts, cache/region/tier adjustments, local BGE/Omneum compute/hosting, and unlogged transport retries are excluded. Claude reasoning-token metadata is null, not established zero; output_tokens is used as reported. Refusal input usage is included. Estimated total cost is ${a['total_usd']:.6f} versus ${o['total_usd']:.6f}; combined run ${a['total_usd']+o['total_usd']:.6f}. Omneum's generator costs ${o['anthropic_usd']-a['anthropic_usd']:.6f} more, offsetting part of the ${a['google_usd']:.6f} judge savings. Total API token counts combine different provider tokenizers and should not be interpreted as identical text volume.

Reproduce from the repository root with:

```sh
python benchmarks/financial_dependency_suite/heldout/analysis/bge-google-vs-bge-omneum-20261008/analyze_saved.py
```

The script imports only offline aggregation infrastructure, verifies the reviewed result hash and all integrity contracts, and writes analysis artifacts. It does not call providers or modify experiment files. No benchmark methodology changes are proposed as completed work; measurement limitations above should be addressed in a separately authorized future experiment.

Aggregate comparison

| Metric | BGE + Gemini | BGE + Omneum |
|---|---:|---:|''']
for label,key in [('Macro precision','precision'),('Macro recall','recall'),('Micro precision','micro_precision'),('Micro recall','micro_recall'),('Selection median seconds','selection_p50'),('Selection p95 seconds','selection_p95'),('E2E median seconds','e2e_p50'),('E2E p95 seconds','e2e_p95'),('Generation median seconds','generator_p50'),('Gemini input tokens','google_input'),('Gemini visible output tokens','google_output'),('Gemini reasoning tokens','google_reasoning'),('Claude input tokens','anthropic_input'),('Claude output tokens','anthropic_output'),('Total API tokens','total_api_tokens'),('Gemini estimated USD','google_usd'),('Claude estimated USD','anthropic_usd'),('Total estimated USD','total_usd')]:
    lines.append(f'| {label} | {a[key]:.6f} | {o[key]:.6f} |')
lines += ['\nOutcome separation\n','| Outcome | Gemini | Omneum |','|---|---:|---:|']
for status in ('correct','incomplete','substantively_incorrect','generator_refusal'):
    lines.append(f"| {status} | {a['manual'].get(status,0)} | {o['manual'].get(status,0)} |")
lines += ['\nPer-case metrics (each arm has ten trials per case)\n','| Case | Complete G/O | Precision G/O | Recall G/O | Selection median s G/O | E2E median s G/O |','|---|---:|---:|---:|---:|---:|']
for c in cases:
    x,y=(per_case[c][m] for m in METHODS)
    lines.append(f"| {c} | {x['manual'].get('correct',0)}/{y['manual'].get('correct',0)} | {x['precision']:.4f}/{y['precision']:.4f} | {x['recall']:.4f}/{y['recall']:.4f} | {x['selection_p50']:.4f}/{y['selection_p50']:.4f} | {x['e2e_p50']:.4f}/{y['e2e_p50']:.4f} |")
lines += ['\nSelection consistency\n',json.dumps(consistency), '\nSelected tuples and counts\n']
for c in cases:
    lines.append(f'\n**{c}**\n')
    for m in METHODS:
        counts=collections.Counter(tuple(r['selected']) for r in rows if r['case_id']==c and r['method']==m)
        lines.append(m+': '+ '; '.join(f'{list(k)} × {v}' for k,v in counts.items())+'\n')
lines += ['\nIndividual adjudications: all 35 unscored answers and 20 refusals\n']
for record in audit:
    if record['recorded_score']=='correct': continue
    lines += [f"\n**{record['case_id']} / {record['method']} / trial {record['trial']}**\n",f"Recorded: {record['recorded_score']}. Adjudicated: {record['manual_status']}. {record['reason']}\n",f"Reference: {record['reference_answer']}\n",'Selected evidence:\n']
    lines += [f"- {d['source']}: {d['text']}" for d in record['evidence']]
    lines += ['\nSaved answer:\n',record['answer'] or f"Empty. Diagnostics: `{json.dumps(record['generator_diagnostics'])}`",'\n']
lines += ['\nIntegrity checks\n','```json',json.dumps(checks,indent=2),'```','\nUnchanged source artifact SHA-256 hashes\n','```json',json.dumps(before,indent=2),'```']
(OUT/'report.md').write_text('\n'.join(lines)+'\n')
assert before == {str(p.relative_to(ROOT)):hashlib.sha256(p.read_bytes()).hexdigest() for p in files}
print(json.dumps({'output':str(OUT),'integrity_checks':checks,'outcomes':{m:aggregates[m]['manual'] for m in METHODS},'savings':savings,'paired':paired_outcomes},indent=2))
