== e2_latency.json ==
  ok  e2_latency.json.load_amortisation.steady_ms_per_request ordered min<=p50<=p95<=max  [6797.590 9757.844 10716.275 10825.337 n=5]
  ok  e2_latency.json.load_amortisation.steady_ms_per_request spread  [n=5]
  ok  e2_latency.json.model_load ordered min<=p50<=p95<=max  [734.869 855.004 983.163 1005.676 n=5]
  ok  e2_latency.json.model_load spread  [n=5]
  ok  e2_latency.json.per_question[0].ms ordered min<=p50<=p95<=max  [5370.510 6141.113 6419.314 6443.273 n=5]
  ok  e2_latency.json.per_question[0].ms spread  [n=5]
  ok  e2_latency.json.per_question[0].total_ms ordered min<=p50<=p95<=max  [5384.100 6186.524 6432.712 6456.749 n=5]
  ok  e2_latency.json.per_question[0].total_ms spread  [n=5]
  ok  e2_latency.json.per_question[1].ms ordered min<=p50<=p95<=max  [7271.997 8542.577 11221.168 11854.647 n=5]
  ok  e2_latency.json.per_question[1].ms spread  [n=5]
  ok  e2_latency.json.per_question[1].total_ms ordered min<=p50<=p95<=max  [7284.326 8555.137 11308.168 11952.331 n=5]
  ok  e2_latency.json.per_question[1].total_ms spread  [n=5]
  ok  e2_latency.json.per_question[2].ms ordered min<=p50<=p95<=max  [11099.255 14923.724 18782.525 19735.482 n=5]
  ok  e2_latency.json.per_question[2].ms spread  [n=5]
  ok  e2_latency.json.per_question[2].total_ms ordered min<=p50<=p95<=max  [11245.202 15140.764 18914.935 19844.692 n=5]
  ok  e2_latency.json.per_question[2].total_ms spread  [n=5]
  ok  e2_latency.json.prefill[0].ms ordered min<=p50<=p95<=max  [25893.846 31706.761 41915.468 43494.498 n=5]
  ok  e2_latency.json.prefill[0].ms spread  [n=5]
  ok  e2_latency.json.prefill[0].tok_per_s ordered min<=p50<=p95<=max  [5.886 8.074 9.579 9.887 n=5]
  ok  e2_latency.json.prefill[0].tok_per_s spread  [n=5]
  ok  e2_latency.json.prefill[1].ms ordered min<=p50<=p95<=max  [227275.077 248300.702 268261.935 268466.515 n=5]
  ok  e2_latency.json.prefill[1].ms spread  [n=5]
  ok  e2_latency.json.prefill[1].tok_per_s ordered min<=p50<=p95<=max  [7.629 8.248 8.948 9.011 n=5]
  ok  e2_latency.json.prefill[1].tok_per_s spread  [n=5]
  ok  e2_latency.json.warm_cache.prefill_ms ordered min<=p50<=p95<=max  [0.000 0.000 0.000 0.000 n=5]
  ok  e2_latency.json.warm_cache.prefill_ms spread  [n=5]
  ok  e2_latency.json.warm_cache.questions_ms ordered min<=p50<=p95<=max  [7577.175 7978.160 8459.720 8486.949 n=5]
  ok  e2_latency.json.warm_cache.questions_ms spread  [n=5]
  ok  e2_latency.json.wave_scaling[0].ms ordered min<=p50<=p95<=max  [4080.342 7594.447 12573.907 13439.045 n=5]
  ok  e2_latency.json.wave_scaling[0].ms spread  [n=5]
  ok  e2_latency.json.wave_scaling[1].ms ordered min<=p50<=p95<=max  [6587.373 7394.049 13879.688 15377.806 n=5]
  ok  e2_latency.json.wave_scaling[1].ms spread  [n=5]
  ok  e2_latency.json.wave_scaling[2].ms ordered min<=p50<=p95<=max  [10392.902 14711.329 33216.033 37444.471 n=5]
  ok  e2_latency.json.wave_scaling[2].ms spread  [n=5]
  ok  e2_latency.json.wave_scaling[3].ms ordered min<=p50<=p95<=max  [13694.008 14632.800 81802.283 97687.586 n=5]
  ok  e2_latency.json.wave_scaling[3].ms spread  [n=5]
  ok  e2_latency.json.wave_scaling[4].ms ordered min<=p50<=p95<=max  [18354.013 24073.554 201117.425 211755.717 n=5]
  ok  e2_latency.json.wave_scaling[4].ms spread  [n=5]
  ok  e2_latency.json.wave_scaling[5].ms ordered min<=p50<=p95<=max  [20380.661 21513.931 111521.437 133939.670 n=5]
  ok  e2_latency.json.wave_scaling[5].ms spread  [n=5]
  ok  e2_latency.json.wave_scaling[6].ms ordered min<=p50<=p95<=max  [19297.427 114289.027 178195.216 191017.310 n=5]
  ok  e2_latency.json.wave_scaling[6].ms spread  [n=5]
  ok  e2_latency.json.wave_scaling[7].ms ordered min<=p50<=p95<=max  [34147.662 41486.986 43192.379 43431.414 n=5]
  ok  e2_latency.json.wave_scaling[7].ms spread  [n=5]
  ok  e2_latency.json.wave_scaling[8].ms ordered min<=p50<=p95<=max  [43790.522 46950.933 48982.510 49282.973 n=5]
  ok  e2_latency.json.wave_scaling[8].ms spread  [n=5]
  ok  e2_latency.json.wave_scaling[9].ms ordered min<=p50<=p95<=max  [28174.696 44425.097 51586.516 53084.336 n=5]
  ok  e2_latency.json.wave_scaling[9].ms spread  [n=5]
  ok  e2_latency.json.wave_scaling[10].ms ordered min<=p50<=p95<=max  [68307.479 203447.621 486197.718 529448.001 n=5]
  ok  e2_latency.json.wave_scaling[10].ms spread  [n=5]
  ok  e2_latency.json.wave_scaling[11].ms ordered min<=p50<=p95<=max  [58750.020 66541.337 77050.610 77961.817 n=5]
  ok  e2_latency.json.wave_scaling[11].ms spread  [n=5]
  ok  e2_latency.json.wave_scaling[12].ms ordered min<=p50<=p95<=max  [48643.622 64177.134 77981.744 78406.651 n=5]
  ok  e2_latency.json.wave_scaling[12].ms spread  [n=5]
  ok  e2_latency.json.wave_scaling[13].ms ordered min<=p50<=p95<=max  [51373.442 86845.894 251232.297 290696.123 n=5]
  ok  e2_latency.json.wave_scaling[13].ms spread  [n=5]
  ok  e2_latency.json.wave_scaling[14].ms ordered min<=p50<=p95<=max  [68556.840 71118.109 94824.842 99912.936 n=5]
  ok  e2_latency.json.wave_scaling[14].ms spread  [n=5]
  ok  e2_latency.json.wave_scaling[15].ms ordered min<=p50<=p95<=max  [59677.483 72488.106 110387.467 111333.938 n=5]
  ok  e2_latency.json.wave_scaling[15].ms spread  [n=5]
  ok  latency load_amortisation: one_shot == serve + model_load  [10612.848 vs 10612.848]
  ok  latency saved == model_load
  ok  latency wave_accounting
  ok  latency wave scaling has 16 rows  [n=16]
  ok  latency ws[16] > ws[1]  [72488 > 7594]
 FAIL latency ws[2] > ws[1]  [7394 > 7594]
== e2_quality.json ==
  ok  quality 60 items  [{'choice': [24, 18], 'score': [18, 4], 'noul': [18, 16]}]
  ok  quality per_type[choice] counts match items  [items 18/24 vs report 18/24]
  ok  quality per_type[score] counts match items  [items 4/18 vs report 4/18]
  ok  quality per_type[noul] counts match items  [items 16/18 vs report 16/18]
  ok  quality overall correct == sum of items
  ok  quality agreement == correct/n
  ok  quality per-item probabilities sum to 1
== e2_calibration.json ==
  ok  e2_calibration.json.coverage ordered min<=p50<=p95<=max  [0.001 0.313 0.628 0.673 n=60]
  ok  e2_calibration.json.coverage spread  [n=60]
  ok  calibration entropy: ECE == sum over bins  [0.243452658487176 vs 0.243452658487176]
  ok  calibration entropy: bins n sum == items  [60 vs 60]
  ok  calibration margin: ECE == sum over bins  [0.262119570348292 vs 0.262119570348292]
  ok  calibration margin: bins n sum == items  [60 vs 60]
  ok  calibration normalized_peak: ECE == sum over bins  [0.250639111562424 vs 0.250639111562424]
  ok  calibration normalized_peak: bins n sum == items  [60 vs 60]
  ok  calibration agreement == correct/n
== e2_determinism.json / e2_throughput.json / e2_qwen_*.json ==
  ok  determinism cpu identical
  ok  e2_throughput.json.backends[0].decision_ms ordered min<=p50<=p95<=max  [7385.773 8010.987 9096.384 9297.459 n=5]
  ok  e2_throughput.json.backends[0].decision_ms spread  [n=5]
  ok  e2_throughput.json.backends[0].decision_tok_per_s ordered min<=p50<=p95<=max  [0.860 0.999 1.066 1.083 n=5]
  ok  e2_throughput.json.backends[0].decision_tok_per_s spread  [n=5]
  ok  e2_throughput.json.backends[0].load_ms ordered min<=p50<=p95<=max  [801.194 802.017 1131.200 1188.889 n=5]
  ok  e2_throughput.json.backends[0].load_ms spread  [n=5]
  ok  e2_throughput.json.backends[0].prefill[0].ms ordered min<=p50<=p95<=max  [37381.036 37892.409 38711.047 38717.307 n=5]
  ok  e2_throughput.json.backends[0].prefill[0].ms spread  [n=5]
  ok  e2_throughput.json.backends[0].prefill[0].tok_per_s ordered min<=p50<=p95<=max  [6.612 6.756 6.848 6.848 n=5]
  ok  e2_throughput.json.backends[0].prefill[0].tok_per_s spread  [n=5]
  ok  e2_throughput.json.backends[0].prefill_tok_per_s ordered min<=p50<=p95<=max  [6.612 6.756 6.848 6.848 n=5]
  ok  e2_throughput.json.backends[0].prefill_tok_per_s spread  [n=5]
  ok  throughput cpu: prefill_tok_per_s == prefill[0]
  ok  throughput vulkan has a reason
  ok  throughput cuda has a reason
  ok  e2_qwen_latency.json.load_amortisation.steady_ms_per_request ordered min<=p50<=p95<=max  [1254.625 1343.235 1510.566 1520.539 n=5]
  ok  e2_qwen_latency.json.load_amortisation.steady_ms_per_request spread  [n=5]
  ok  e2_qwen_latency.json.model_load ordered min<=p50<=p95<=max  [992.566 1571.673 2518.973 2645.827 n=5]
  ok  e2_qwen_latency.json.model_load spread  [n=5]
  ok  e2_qwen_latency.json.per_question[0].ms ordered min<=p50<=p95<=max  [768.123 833.886 874.365 876.112 n=5]
  ok  e2_qwen_latency.json.per_question[0].ms spread  [n=5]
  ok  e2_qwen_latency.json.per_question[0].total_ms ordered min<=p50<=p95<=max  [778.892 839.541 880.025 881.822 n=5]
  ok  e2_qwen_latency.json.per_question[0].total_ms spread  [n=5]
  ok  e2_qwen_latency.json.per_question[1].ms ordered min<=p50<=p95<=max  [1087.908 1125.088 1439.058 1478.421 n=5]
  ok  e2_qwen_latency.json.per_question[1].ms spread  [n=5]
  ok  e2_qwen_latency.json.per_question[1].total_ms ordered min<=p50<=p95<=max  [1111.838 1131.205 1447.269 1487.053 n=5]
  ok  e2_qwen_latency.json.per_question[1].total_ms spread  [n=5]
  ok  e2_qwen_latency.json.per_question[2].ms ordered min<=p50<=p95<=max  [1836.712 2221.644 2543.329 2571.118 n=5]
  ok  e2_qwen_latency.json.per_question[2].ms spread  [n=5]
  ok  e2_qwen_latency.json.per_question[2].total_ms ordered min<=p50<=p95<=max  [1843.144 2229.491 2560.347 2578.718 n=5]
  ok  e2_qwen_latency.json.per_question[2].total_ms spread  [n=5]
  ok  e2_qwen_latency.json.prefill[0].ms ordered min<=p50<=p95<=max  [8526.632 9443.904 10391.614 10598.579 n=5]
  ok  e2_qwen_latency.json.prefill[0].ms spread  [n=5]
  ok  e2_qwen_latency.json.prefill[0].tok_per_s ordered min<=p50<=p95<=max  [24.154 27.107 29.896 30.024 n=5]
  ok  e2_qwen_latency.json.prefill[0].tok_per_s spread  [n=5]
  ok  e2_qwen_latency.json.prefill[1].ms ordered min<=p50<=p95<=max  [50525.492 55240.973 67101.257 69941.665 n=5]
  ok  e2_qwen_latency.json.prefill[1].ms spread  [n=5]
  ok  e2_qwen_latency.json.prefill[1].tok_per_s ordered min<=p50<=p95<=max  [29.282 37.074 40.215 40.534 n=5]
  ok  e2_qwen_latency.json.prefill[1].tok_per_s spread  [n=5]
  ok  e2_qwen_latency.json.prefill[2].ms ordered min<=p50<=p95<=max  [216356.185 250811.546 321099.395 322341.620 n=5]
  ok  e2_qwen_latency.json.prefill[2].ms spread  [n=5]
  ok  e2_qwen_latency.json.prefill[2].tok_per_s ordered min<=p50<=p95<=max  [25.414 32.662 37.619 37.863 n=5]
  ok  e2_qwen_latency.json.prefill[2].tok_per_s spread  [n=5]
  ok  e2_qwen_latency.json.warm_cache.prefill_ms ordered min<=p50<=p95<=max  [0.000 0.000 0.000 0.000 n=5]
  ok  e2_qwen_latency.json.warm_cache.prefill_ms spread  [n=5]
  ok  e2_qwen_latency.json.warm_cache.questions_ms ordered min<=p50<=p95<=max  [955.888 1281.379 1784.839 1821.991 n=5]
  ok  e2_qwen_latency.json.warm_cache.questions_ms spread  [n=5]
  ok  e2_qwen_latency.json.wave_scaling[0].ms ordered min<=p50<=p95<=max  [448.219 653.667 921.289 966.161 n=5]
  ok  e2_qwen_latency.json.wave_scaling[0].ms spread  [n=5]
  ok  e2_qwen_latency.json.wave_scaling[1].ms ordered min<=p50<=p95<=max  [1301.385 1478.926 1663.013 1667.214 n=5]
  ok  e2_qwen_latency.json.wave_scaling[1].ms spread  [n=5]
  ok  e2_qwen_latency.json.wave_scaling[2].ms ordered min<=p50<=p95<=max  [2100.313 2198.045 3101.855 3184.668 n=5]
  ok  e2_qwen_latency.json.wave_scaling[2].ms spread  [n=5]
  ok  e2_qwen_latency.json.wave_scaling[3].ms ordered min<=p50<=p95<=max  [2399.344 3244.997 3634.245 3713.203 n=5]
  ok  e2_qwen_latency.json.wave_scaling[3].ms spread  [n=5]
  ok  e2_qwen_latency.json.wave_scaling[4].ms ordered min<=p50<=p95<=max  [3045.215 3766.558 4147.747 4169.800 n=5]
  ok  e2_qwen_latency.json.wave_scaling[4].ms spread  [n=5]
  ok  e2_qwen_latency.json.wave_scaling[5].ms ordered min<=p50<=p95<=max  [4377.850 4703.782 5049.311 5105.823 n=5]
  ok  e2_qwen_latency.json.wave_scaling[5].ms spread  [n=5]
  ok  e2_qwen_latency.json.wave_scaling[6].ms ordered min<=p50<=p95<=max  [4086.272 4991.353 5337.537 5369.841 n=5]
  ok  e2_qwen_latency.json.wave_scaling[6].ms spread  [n=5]
  ok  e2_qwen_latency.json.wave_scaling[7].ms ordered min<=p50<=p95<=max  [5171.169 6989.073 7656.278 7819.743 n=5]
  ok  e2_qwen_latency.json.wave_scaling[7].ms spread  [n=5]
  ok  e2_qwen_latency.json.wave_scaling[8].ms ordered min<=p50<=p95<=max  [7194.017 9248.717 22929.696 26253.310 n=5]
  ok  e2_qwen_latency.json.wave_scaling[8].ms spread  [n=5]
  ok  e2_qwen_latency.json.wave_scaling[9].ms ordered min<=p50<=p95<=max  [6540.778 8401.411 10535.499 10896.590 n=5]
  ok  e2_qwen_latency.json.wave_scaling[9].ms spread  [n=5]
  ok  e2_qwen_latency.json.wave_scaling[10].ms ordered min<=p50<=p95<=max  [11463.999 12875.971 14191.363 14197.734 n=5]
  ok  e2_qwen_latency.json.wave_scaling[10].ms spread  [n=5]
  ok  e2_qwen_latency.json.wave_scaling[11].ms ordered min<=p50<=p95<=max  [10736.038 13948.431 16604.623 16918.372 n=5]
  ok  e2_qwen_latency.json.wave_scaling[11].ms spread  [n=5]
  ok  e2_qwen_latency.json.wave_scaling[12].ms ordered min<=p50<=p95<=max  [9312.274 11112.752 15014.153 15972.972 n=5]
  ok  e2_qwen_latency.json.wave_scaling[12].ms spread  [n=5]
  ok  e2_qwen_latency.json.wave_scaling[13].ms ordered min<=p50<=p95<=max  [9790.845 13288.185 16031.074 16507.120 n=5]
  ok  e2_qwen_latency.json.wave_scaling[13].ms spread  [n=5]
  ok  e2_qwen_latency.json.wave_scaling[14].ms ordered min<=p50<=p95<=max  [12072.694 12306.517 15985.169 16595.219 n=5]
  ok  e2_qwen_latency.json.wave_scaling[14].ms spread  [n=5]
  ok  e2_qwen_latency.json.wave_scaling[15].ms ordered min<=p50<=p95<=max  [11687.492 13108.831 19455.936 20721.094 n=5]
  ok  e2_qwen_latency.json.wave_scaling[15].ms spread  [n=5]
  ok  qwen quality 28/60  [28]

163 checks, 1 problems
  PROBLEM: latency ws[2] > ws[1]: 7394 > 7594
