requests_total=2656 (before the first model request)
jevbench: rows=2282
{"arm": "baseline", "accuracy": 0.6147186147186147, "unread": 23, "ece": 0.22383245784019895, "ece_bin_counts": [{"bin": 0, "n": 0}, {"bin": 1, "n": 0}, {"bin": 2, "n": 0}, {"bin": 3, "n": 0}, {"bin": 4, "n": 2}, {"bin": 5, "n": 12}, {"bin": 6, "n": 13}, {"bin": 7, "n": 15}, {"bin": 8, "n": 20}, {"bin": 9, "n": 146}], "brier": 0.500044064470475, "valid": 208, "flips_vs_baseline": 0, "mcnemar": null}
{"arm": "C_numeric", "accuracy": 0.6883116883116883, "unread": 0, "ece": 0.15081013457436598, "ece_bin_counts": [{"bin": 0, "n": 0}, {"bin": 1, "n": 0}, {"bin": 2, "n": 0}, {"bin": 3, "n": 6}, {"bin": 4, "n": 16}, {"bin": 5, "n": 19}, {"bin": 6, "n": 17}, {"bin": 7, "n": 22}, {"bin": 8, "n": 20}, {"bin": 9, "n": 131}], "brier": 0.4660048604193158, "valid": 231, "flips_vs_baseline": 79, "mcnemar": {"baseline_only_correct": 23, "arm_only_correct": 40, "p_two_sided": 0.04295654552438921}}
{"arm": "letters_rotation", "accuracy": 0.6926406926406926, "unread": 0, "ece": 0.16822620009869632, "ece_bin_counts": [{"bin": 0, "n": 0}, {"bin": 1, "n": 0}, {"bin": 2, "n": 0}, {"bin": 3, "n": 4}, {"bin": 4, "n": 9}, {"bin": 5, "n": 17}, {"bin": 6, "n": 20}, {"bin": 7, "n": 15}, {"bin": 8, "n": 28}, {"bin": 9, "n": 138}], "brier": 0.4533354382431312, "valid": 231, "flips_vs_baseline": 54, "mcnemar": {"baseline_only_correct": 12, "arm_only_correct": 30, "p_two_sided": 0.007915897334896727}}
{"arm": "label_swap_agreement", "agreement": 0.6493506493506493, "n": 231, "per_call_accuracy": {"letters": 0.6926406926406926, "swap": 0.6060606060606061}}
{"arm": "label_swap", "accuracy": 0.6926406926406926, "unread": 0, "ece": 0.16952954188835392, "ece_bin_counts": [{"bin": 0, "n": 0}, {"bin": 1, "n": 0}, {"bin": 2, "n": 0}, {"bin": 3, "n": 4}, {"bin": 4, "n": 7}, {"bin": 5, "n": 19}, {"bin": 6, "n": 19}, {"bin": 7, "n": 15}, {"bin": 8, "n": 28}, {"bin": 9, "n": 139}], "brier": 0.45752999678828876, "valid": 231, "flips_vs_baseline": 55, "mcnemar": {"baseline_only_correct": 12, "arm_only_correct": 30, "p_two_sided": 0.007915897334896727}}
{"arm": "neutral", "accuracy": 0.7266187050359713, "unread": 0, "ece": 0.1457526419080981, "ece_bin_counts": [{"bin": 0, "n": 0}, {"bin": 1, "n": 0}, {"bin": 2, "n": 0}, {"bin": 3, "n": 1}, {"bin": 4, "n": 9}, {"bin": 5, "n": 9}, {"bin": 6, "n": 12}, {"bin": 7, "n": 15}, {"bin": 8, "n": 8}, {"bin": 9, "n": 85}], "brier": 0.4253381609863711, "valid": 139, "flips_vs_baseline": 36, "mcnemar": {"baseline_only_correct": 6, "arm_only_correct": 24, "p_two_sided": 0.001430906355381012}}
{"arm": "definition_only", "accuracy": 0.6834532374100719, "unread": 0, "ece": 0.1621848659622967, "ece_bin_counts": [{"bin": 0, "n": 0}, {"bin": 1, "n": 0}, {"bin": 2, "n": 0}, {"bin": 3, "n": 4}, {"bin": 4, "n": 6}, {"bin": 5, "n": 12}, {"bin": 6, "n": 12}, {"bin": 7, "n": 10}, {"bin": 8, "n": 13}, {"bin": 9, "n": 82}], "brier": 0.45624885413063926, "valid": 139, "flips_vs_baseline": 45, "mcnemar": {"baseline_only_correct": 10, "arm_only_correct": 22, "p_two_sided": 0.050102459732443094}}
{"arm": "letters_rotation_mean", "accuracy": 0.6796536796536796, "unread": 0, "ece": 0.13196068608502903, "ece_bin_counts": [{"bin": 0, "n": 0}, {"bin": 1, "n": 0}, {"bin": 2, "n": 4}, {"bin": 3, "n": 12}, {"bin": 4, "n": 13}, {"bin": 5, "n": 27}, {"bin": 6, "n": 25}, {"bin": 7, "n": 12}, {"bin": 8, "n": 17}, {"bin": 9, "n": 121}], "brier": 0.42000153451080485, "valid": 231, "flips_vs_baseline": 61, "mcnemar": {"baseline_only_correct": 18, "arm_only_correct": 33, "p_two_sided": 0.04887389231827033}}
{"arm": "letters_rotation", "rotation": 0, "flips_vs_baseline": 54, "n": 231}
{"arm": "letters_rotation", "rotation": 1, "flips_vs_baseline": 69, "n": 231}
{"arm": "letters_rotation", "rotation": 2, "flips_vs_baseline": 58, "n": 157}
{"arm": "letters_rotation", "rotation": 3, "flips_vs_baseline": 48, "n": 142}
{"arm": "letters_rotation", "rotation": 4, "flips_vs_baseline": 18, "n": 72}
{"arm": "letters_rotation", "rotation": 5, "flips_vs_baseline": 8, "n": 16}
governance: rows=660
{"arm": "baseline", "accuracy": 0.6727272727272727, "unread": 0, "ece": 0.2008406671462888, "ece_bin_counts": [{"bin": 0, "n": 0}, {"bin": 1, "n": 0}, {"bin": 2, "n": 0}, {"bin": 3, "n": 0}, {"bin": 4, "n": 0}, {"bin": 5, "n": 2}, {"bin": 6, "n": 7}, {"bin": 7, "n": 6}, {"bin": 8, "n": 6}, {"bin": 9, "n": 34}], "brier": null, "valid": 55, "flips_vs_baseline": 0, "mcnemar": null}
{"arm": "C_numeric", "accuracy": 0.8727272727272727, "unread": 0, "ece": 0.08729434725043334, "ece_bin_counts": [{"bin": 0, "n": 0}, {"bin": 1, "n": 0}, {"bin": 2, "n": 0}, {"bin": 3, "n": 0}, {"bin": 4, "n": 2}, {"bin": 5, "n": 3}, {"bin": 6, "n": 6}, {"bin": 7, "n": 8}, {"bin": 8, "n": 11}, {"bin": 9, "n": 25}], "brier": null, "valid": 55, "flips_vs_baseline": 14, "mcnemar": {"baseline_only_correct": 1, "arm_only_correct": 12, "p_two_sided": 0.00341796875}}
{"arm": "letters_rotation", "accuracy": 0.7090909090909091, "unread": 0, "ece": 0.07112859478261813, "ece_bin_counts": [{"bin": 0, "n": 0}, {"bin": 1, "n": 0}, {"bin": 2, "n": 0}, {"bin": 3, "n": 0}, {"bin": 4, "n": 6}, {"bin": 5, "n": 13}, {"bin": 6, "n": 17}, {"bin": 7, "n": 9}, {"bin": 8, "n": 4}, {"bin": 9, "n": 6}], "brier": null, "valid": 55, "flips_vs_baseline": 24, "mcnemar": {"baseline_only_correct": 0, "arm_only_correct": 2, "p_two_sided": 0.5}}
{"arm": "label_swap_agreement", "agreement": 0.43636363636363634, "n": 55, "per_call_accuracy": {"letters": 0.6909090909090909, "swap": 0.6909090909090909}}
{"arm": "label_swap", "accuracy": 0.6909090909090909, "unread": 0, "ece": 0.08179252416651596, "ece_bin_counts": [{"bin": 0, "n": 0}, {"bin": 1, "n": 0}, {"bin": 2, "n": 0}, {"bin": 3, "n": 0}, {"bin": 4, "n": 5}, {"bin": 5, "n": 13}, {"bin": 6, "n": 16}, {"bin": 7, "n": 11}, {"bin": 8, "n": 4}, {"bin": 9, "n": 6}], "brier": null, "valid": 55, "flips_vs_baseline": 26, "mcnemar": {"baseline_only_correct": 1, "arm_only_correct": 2, "p_two_sided": 1.0}}
{"arm": "neutral", "accuracy": 0.7090909090909091, "unread": 0, "ece": 0.08409413430644241, "ece_bin_counts": [{"bin": 0, "n": 0}, {"bin": 1, "n": 0}, {"bin": 2, "n": 0}, {"bin": 3, "n": 0}, {"bin": 4, "n": 6}, {"bin": 5, "n": 12}, {"bin": 6, "n": 18}, {"bin": 7, "n": 9}, {"bin": 8, "n": 4}, {"bin": 9, "n": 6}], "brier": null, "valid": 55, "flips_vs_baseline": 24, "mcnemar": {"baseline_only_correct": 0, "arm_only_correct": 2, "p_two_sided": 0.5}}
{"arm": "definition_only", "accuracy": 0.6363636363636364, "unread": 0, "ece": 0.07170138895052547, "ece_bin_counts": [{"bin": 0, "n": 0}, {"bin": 1, "n": 0}, {"bin": 2, "n": 0}, {"bin": 3, "n": 1}, {"bin": 4, "n": 6}, {"bin": 5, "n": 9}, {"bin": 6, "n": 12}, {"bin": 7, "n": 23}, {"bin": 8, "n": 4}, {"bin": 9, "n": 0}], "brier": null, "valid": 55, "flips_vs_baseline": 37, "mcnemar": {"baseline_only_correct": 3, "arm_only_correct": 1, "p_two_sided": 0.625}}
{"arm": "negated", "paired_polarity": "affirmed", "mapped_answer_flips": 10, "paired_n": 55}
{"arm": "negated", "accuracy": 0.4909090909090909, "unread": 0, "ece": 0.4442054482006314, "ece_bin_counts": [{"bin": 0, "n": 0}, {"bin": 1, "n": 0}, {"bin": 2, "n": 0}, {"bin": 3, "n": 0}, {"bin": 4, "n": 0}, {"bin": 5, "n": 2}, {"bin": 6, "n": 3}, {"bin": 7, "n": 2}, {"bin": 8, "n": 5}, {"bin": 9, "n": 43}], "brier": null, "valid": 55, "flips_vs_baseline": 55, "mcnemar": {"baseline_only_correct": 11, "arm_only_correct": 1, "p_two_sided": 0.00634765625}}
{"arm": "affirmed", "paired_polarity": "negated", "mapped_answer_flips": 10, "paired_n": 55}
{"arm": "affirmed", "accuracy": 0.6363636363636364, "unread": 0, "ece": 0.32315990734928907, "ece_bin_counts": [{"bin": 0, "n": 0}, {"bin": 1, "n": 0}, {"bin": 2, "n": 0}, {"bin": 3, "n": 0}, {"bin": 4, "n": 0}, {"bin": 5, "n": 0}, {"bin": 6, "n": 0}, {"bin": 7, "n": 4}, {"bin": 8, "n": 3}, {"bin": 9, "n": 48}], "brier": null, "valid": 55, "flips_vs_baseline": 55, "mcnemar": {"baseline_only_correct": 5, "arm_only_correct": 3, "p_two_sided": 0.7265625}}
{"arm": "letters_rotation_mean", "accuracy": 0.7454545454545455, "unread": 0, "ece": 0.10440492469289751, "ece_bin_counts": [{"bin": 0, "n": 0}, {"bin": 1, "n": 0}, {"bin": 2, "n": 0}, {"bin": 3, "n": 7}, {"bin": 4, "n": 5}, {"bin": 5, "n": 10}, {"bin": 6, "n": 8}, {"bin": 7, "n": 15}, {"bin": 8, "n": 6}, {"bin": 9, "n": 4}], "brier": null, "valid": 55, "flips_vs_baseline": 4, "mcnemar": {"baseline_only_correct": 0, "arm_only_correct": 4, "p_two_sided": 0.125}}
{"arm": "neutral_vs_definition_only", "paired_n": 55, "mapped_answer_flips": 14}
{"arm": "letters_rotation", "rotation": 0, "flips_vs_baseline": 24, "n": 55}
{"arm": "letters_rotation", "rotation": 1, "flips_vs_baseline": 14, "n": 55}
{"arm": "letters_rotation", "rotation": 2, "flips_vs_baseline": 10, "n": 55}
EXIT 0
