framework,version,device,op_name,kernel_source,batch_size,isl,num_heads,num_key_value_heads,head_dim,beam_width,attn_dtype,kv_cache_dtype,step,latency
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,float16,1,0.04941760003566742
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,fp8,1,0.039419201016426084
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,float16,3,0.04938240051269531
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,fp8,3,0.0396016001701355
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,float16,7,0.04934079945087433
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,fp8,7,0.03920640051364899
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,float16,15,0.049409601092338565
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,fp8,15,0.039113599061965945
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,fp8,31,0.039175999164581296
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,float16,63,0.053478401899337766
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,float16,127,0.055415999889373777
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,float16,255,0.08639680147171021
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,fp8,1023,0.16013920307159424
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,float16,31,0.05115519762039185
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,float16,2047,0.5986847877502441
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,float16,1,0.010356800258159637
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,float16,511,0.1583359956741333
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,float16,1023,0.3026432037353516
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,float16,3,0.010412800312042236
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,float16,7,0.010412800312042236
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,fp8,1,0.008379200100898742
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,float16,15,0.010425599664449692
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,fp8,15,0.008420799672603608
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,float16,31,0.010492800176143647
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,fp8,31,0.008347199857234954
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,float16,63,0.010390400141477584
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,fp8,63,0.008364800363779068
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,float16,127,0.010414399951696397
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,fp8,127,0.008452799916267396
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,float16,255,0.010424000024795533
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,fp8,255,0.00841279998421669
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,float16,511,0.010414399951696397
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,fp8,511,0.008441600203514098
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,float16,1023,0.011307200044393539
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,fp8,1023,0.00857120007276535
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,float16,2047,0.013070400059223174
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,fp8,2047,0.01048159971833229
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,float16,1,0.010375999659299851
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,fp8,1,0.008316799998283386
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,float16,3,0.010371199995279311
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,fp8,3,0.008355200290679932
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,float16,7,0.01032480001449585
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,fp8,7,0.008350399881601333
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,float16,15,0.010375999659299851
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,fp8,15,0.008342400193214417
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,float16,31,0.010409600287675857
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,fp8,31,0.008326400071382523
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,float16,63,0.01034879982471466
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,fp8,63,0.008329600095748901
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,float16,127,0.010390400141477584
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,fp8,127,0.008352000266313553
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,float16,255,0.01040479987859726
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,fp8,255,0.008366400003433227
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,float16,511,0.01048159971833229
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,fp8,511,0.008355200290679932
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,float16,1023,0.012503999471664428
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,fp8,1023,0.010380800068378448
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,float16,2047,0.030840000510215758
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,fp8,2047,0.012492799758911132
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,float16,1,0.02893120050430298
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,fp8,1,0.022804799675941467
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,float16,3,0.02892639935016632
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,fp8,2047,0.3111488103866577
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,fp8,3,0.022780799865722658
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,fp8,7,0.022705599665641785
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,float16,15,0.028923198580741882
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,fp8,15,0.02285439968109131
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,fp8,63,0.03933280110359192
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,float16,31,0.02892799973487854
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,float16,63,0.029023998975753786
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,fp8,127,0.04114879965782166
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,fp8,31,0.02279839962720871
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,fp8,63,0.02282879948616028
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,float16,255,0.04997119903564453
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,fp8,127,0.022912000119686127
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,fp8,255,0.028409600257873535
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,float16,127,0.03391039967536926
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,fp8,255,0.048019200563430786
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,fp8,511,0.04745439887046814
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,fp8,511,0.08506399989128113
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,float16,511,0.08619999885559082
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,float16,2047,0.3047760009765625
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,float16,1023,0.16002240180969238
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,fp8,1023,0.08546559810638428
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,fp8,1,0.008441600203514098
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,float16,7,0.010480000078678131
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,fp8,2047,0.16223679780960082
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,float16,1,0.010427200049161912
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,fp8,7,0.008417599648237229
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,float16,3,0.010744000226259232
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,fp8,3,0.008478400111198426
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,fp8,3,0.008371199667453765
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,fp8,15,0.008364800363779068
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,fp8,63,0.008459199965000153
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,fp8,31,0.008387199789285659
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,float16,15,0.010790400207042694
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,fp8,7,0.008436799794435502
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,float16,7,0.028937599062919615
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,float16,63,0.011020799726247787
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,float16,127,0.010520000010728836
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,float16,511,0.013574400544166565
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,fp8,127,0.008580800145864487
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,float16,255,0.010419200360774993
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,fp8,1023,0.012511999905109405
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,fp8,255,0.00840959995985031
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,fp8,1,0.07173439860343933
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,fp8,511,0.010502400249242783
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,float16,1023,0.029812800884246825
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,float16,2047,0.04795520007610321
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,fp8,2047,0.02789919972419739
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,float16,1,0.08847039937973022
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,float16,3,0.08836479783058167
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,fp8,3,0.07174720168113709
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,float16,7,0.08841120004653931
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,fp8,7,0.07154240012168885
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,float16,15,0.09075040221214295
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,fp8,15,0.07173439860343933
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,float16,31,0.09106879830360412
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,fp8,31,0.07192959785461425
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,float16,63,0.09250879883766175
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,fp8,63,0.07202720046043395
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,float16,127,0.09651520252227783
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,fp8,127,0.07234879732131957
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,float16,255,0.1599311947822571
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,fp8,255,0.08648800253868102
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,float16,511,0.30240159034729003
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,fp8,511,0.15962079763412476
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,float16,1,0.1664047956466675
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,fp8,1,0.13549599647521973
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,float16,3,0.1667695999145508
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,float16,31,0.010468800365924836
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,fp8,1023,0.30904159545898435
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,float16,1023,0.5904784202575684
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,float16,31,0.17050399780273437
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,fp8,3,0.13492159843444823
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,fp8,7,0.13465280532836915
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,float16,7,0.17046879529953002
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,float16,15,0.1706079959869385
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,fp8,63,0.13596960306167602
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,fp8,15,0.13557440042495728
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,fp8,31,0.13567520380020143
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,64,128,1,float16,float16,1,0.32635200023651123
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,float16,63,0.17041759490966796
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,float16,127,0.1784208059310913
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,64,128,1,float16,fp8,1,0.2622783899307251
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,fp8,255,0.1636672019958496
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,64,128,1,float16,float16,3,0.32774078845977783
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,64,128,1,float16,fp8,7,0.262609601020813
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,64,128,1,float16,fp8,15,0.2628799915313721
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,64,128,1,float16,float16,31,0.3277055978775024
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,64,128,1,float16,fp8,31,0.26246559619903564
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,64,128,1,float16,float16,63,0.3279776096343994
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,64,128,1,float16,fp8,63,0.26251199245452883
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,64,128,1,float16,float16,127,0.3427567958831787
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,float16,1,0.014502400159835815
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,64,128,1,float16,fp8,127,0.2645776033401489
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,fp8,3,0.010623999685049058
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,float16,7,0.014480000734329224
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,fp8,1,0.010836800187826156
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,fp8,7,0.010704000294208527
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,float16,3,0.014473600685596466
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,fp8,15,0.010411199927330018
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,fp8,31,0.010436800122261048
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,float16,63,0.014504000544548035
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,float16,15,0.014528000354766845
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,fp8,63,0.010502400249242783
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,float16,31,0.01451839953660965
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,float16,255,0.3055871963500977
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,64,128,1,float16,fp8,3,0.2627023935317993
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,float16,255,0.014508800208568573
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,fp8,255,0.010478399693965912
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,float16,511,0.031088000535964964
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,float16,1023,0.049358400702476504
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,float16,127,0.014534400403499603
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,float16,2047,0.08750720024108886
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,fp8,2047,0.047932800650596616
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,64,128,1,float16,float16,7,0.3279023885726929
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,fp8,511,0.014478400349617004
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,64,128,1,float16,float16,15,0.327783989906311
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,64,128,1,float16,fp8,1,0.5170688152313232
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,64,128,1,float16,float16,1,0.645369577407837
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,64,128,1,float16,fp8,3,0.5186255931854248
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,64,128,1,float16,float16,3,0.6433616161346436
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,64,128,1,float16,fp8,7,0.5178847789764405
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,64,128,1,float16,float16,7,0.6441567897796631
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,64,128,1,float16,float16,15,0.645091199874878
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,64,128,1,float16,fp8,31,0.5165631771087646
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,64,128,1,float16,fp8,15,0.5182096004486084
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,64,128,1,float16,fp8,63,0.5171055793762207
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,64,128,1,float16,float16,31,0.6429599761962891
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,64,128,1,float16,float16,63,0.6447824001312256
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,fp8,127,0.13598239421844482
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,fp8,127,0.011446399986743927
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,64,128,1,float16,fp8,1,1.0293168067932128
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,64,128,1,float16,float16,1,1.2814703941345216
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,fp8,1023,0.02881920039653778
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,64,128,1,float16,float16,3,1.2849535942077637
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,float16,1,0.01870879977941513
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,fp8,1,0.014627200365066529
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,64,128,1,float16,fp8,3,1.0285663604736328
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,float16,3,0.018716800212860107
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,fp8,3,0.014672000706195832
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,64,128,1,float16,float16,7,1.2829999923706055
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,float16,7,0.018680000305175783
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,fp8,7,0.01459999978542328
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,float16,15,0.01870719939470291
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,fp8,15,0.014502400159835815
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,float16,31,0.018624000251293182
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,64,128,1,float16,float16,15,1.2833231925964355
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,64,128,1,float16,fp8,15,1.027896022796631
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,fp8,31,0.014508800208568573
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,64,128,1,float16,fp8,7,1.0273183822631835
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,float16,63,0.018676799535751343
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,fp8,127,0.01464959979057312
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,float16,127,0.018590399622917177
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,64,128,1,float16,fp8,31,1.0263824462890625
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,fp8,63,0.014556799829006196
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,float16,511,0.04949280023574829
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,fp8,511,0.028726398944854736
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,float16,1023,0.08607360124588012
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,float16,2047,0.15784800052642822
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,float16,1,0.039150398969650266
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,fp8,1,0.03100320100784302
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,fp8,3,0.03097119927406311
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,float16,7,0.039087998867034915
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,fp8,7,0.030982398986816408
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,float16,15,0.0390720009803772
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,fp8,15,0.031043198704719544
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,float16,31,0.03902879953384399
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,fp8,31,0.030947199463844298
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,float16,63,0.041193601489067075
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,fp8,63,0.03102560043334961
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,float16,127,0.04345119893550873
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,fp8,127,0.03293280005455017
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,float16,255,0.06700320243835449
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,fp8,255,0.038222399353981015
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,float16,511,0.12121759653091431
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,fp8,511,0.06668959856033325
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,fp8,255,0.014524799585342408
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,float16,255,0.03102239966392517
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,64,128,1,float16,float16,31,1.2818703651428223
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,fp8,2047,0.0867680013179779
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,fp8,1023,0.12302080392837525
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,fp8,1023,0.048876801133155824
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,float16,1,0.010496000200510025
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,float16,1023,0.22821600437164308
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,float16,3,0.03915840089321136
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,float16,2047,0.44524478912353516
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,float16,15,0.010486400127410889
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,float16,7,0.010398399829864503
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,float16,3,0.010447999835014344
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,float16,63,0.010507199913263321
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,fp8,1,0.008486399799585343
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,float16,127,0.0104032002389431
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,fp8,3,0.008460800349712371
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,fp8,127,0.008489599823951722
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,float16,255,0.010529600083827972
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,float16,511,0.01051200032234192
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,fp8,511,0.010175999999046326
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,float16,1023,0.012511999905109405
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,fp8,1023,0.010473600029945374
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,float16,2047,0.014504000544548035
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,fp8,2047,0.010468800365924836
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,float16,1,0.008463999629020691
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,fp8,2047,0.23621759414672852
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,fp8,31,0.008668799698352814
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,fp8,1,0.008055999875068665
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,fp8,3,0.00835999995470047
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,fp8,7,0.009196799993515015
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,float16,3,0.0083856001496315
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,float16,15,0.008968000113964082
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,float16,7,0.008436799794435502
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,float16,31,0.008764799684286118
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,float16,63,0.00897279977798462
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,fp8,63,0.008337599784135818
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,float16,127,0.00904960036277771
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,fp8,31,0.007947199791669846
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,float16,31,0.01043040007352829
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,fp8,127,0.008345600217580795
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,float16,255,0.00843520015478134
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,fp8,255,0.00836160033941269
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,fp8,511,0.008343999832868576
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,float16,1023,0.012427199631929398
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,fp8,1023,0.01040000021457672
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,fp8,63,0.00865280032157898
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,float16,2047,0.018900799751281738
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,fp8,2047,0.012547199428081513
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,float16,1,0.02460159957408905
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,float16,3,0.024532799422740937
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,float16,7,0.024294400215148927
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,fp8,3,0.018691200017929076
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,fp8,7,0.01858399957418442
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,float16,15,0.02300640046596527
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,fp8,15,0.0187376007437706
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,float16,31,0.022878399491310118
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,fp8,31,0.019023999571800232
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,float16,63,0.022977599501609804
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,float16,127,0.02683199942111969
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,fp8,127,0.01886879950761795
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,float16,255,0.039350399374961854
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,fp8,255,0.008441600203514098
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,fp8,255,0.02062080055475235
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,fp8,15,0.00841120034456253
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,float16,511,0.0664143979549408
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,fp8,511,0.0384335994720459
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,float16,1023,0.1209231972694397
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,fp8,1023,0.06630560159683227
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,float16,2047,0.22944159507751466
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,fp8,2047,0.12389440536499023
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,fp8,7,0.008347199857234954
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,float16,1,0.010395199805498124
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,fp8,1,0.008470399677753449
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,float16,3,0.010398399829864503
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,fp8,3,0.00843840017914772
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,fp8,15,0.008372800052165985
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,float16,7,0.010428799688816071
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,fp8,7,0.008441600203514098
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,float16,15,0.01051200032234192
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,float16,31,0.010390400141477584
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,fp8,15,0.00835840031504631
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,fp8,31,0.008441600203514098
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,fp8,63,0.008475200086832047
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,float16,127,0.01048320010304451
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,fp8,255,0.008433599770069123
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,float16,511,0.012531200051307678
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,fp8,511,0.010409600287675857
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,float16,511,0.010462400317192078
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,float16,1023,0.02143999934196472
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,fp8,1023,0.012558400630950928
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,float16,2047,0.04253279864788055
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,fp8,2047,0.019913600385189058
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,fp8,1,0.018692800402641298
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,float16,1,0.0683359980583191
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,fp8,1,0.055555200576782225
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,fp8,3,0.055561602115631104
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,float16,7,0.06826559901237488
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,fp8,7,0.055524802207946776
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,fp8,15,0.055550402402877806
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,fp8,63,0.018848000466823576
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,float16,31,0.07158719897270202
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,float16,63,0.0720416009426117
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,fp8,31,0.055542397499084475
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,fp8,63,0.05606560111045837
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,fp8,127,0.05731679797172547
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,fp8,255,0.06834880113601685
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,float16,127,0.07435680031776429
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,float16,255,0.12444959878921509
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,float16,511,0.22986240386962892
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,fp8,511,0.12181119918823242
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,fp8,1023,0.23539040088653565
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,fp8,127,0.008419200032949447
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,float16,255,0.010390400141477584
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,fp8,1,0.10377919673919678
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,float16,3,0.1275167942047119
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,float16,1,0.1274896025657654
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,fp8,3,0.10284479856491088
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,float16,7,0.12948960065841675
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,fp8,7,0.10340800285339355
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,float16,1023,0.44526238441467286
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,fp8,15,0.10295519828796387
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,fp8,31,0.10475039482116699
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,fp8,63,0.1046288013458252
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,float16,3,0.06797279715538025
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,float16,15,0.12991039752960204
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,float16,15,0.06992800235748291
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,fp8,127,0.10478559732437134
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,float16,255,0.23186719417572021
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,48,48,128,1,float16,float16,1,0.24550399780273438
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,48,48,128,1,float16,fp8,1,0.1989184021949768
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,fp8,255,0.12585599422454835
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,48,48,128,1,float16,fp8,3,0.19896639585494996
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,48,48,128,1,float16,float16,3,0.24827198982238768
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,48,48,128,1,float16,float16,7,0.24794559478759765
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,48,48,128,1,float16,fp8,7,0.19925279617309571
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,48,48,128,1,float16,fp8,15,0.19960960149765014
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,48,48,128,1,float16,float16,15,0.2482912063598633
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,48,48,128,1,float16,float16,31,0.24797921180725097
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,48,48,128,1,float16,fp8,31,0.19925919771194459
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,48,48,128,1,float16,fp8,63,0.1995568037033081
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,48,48,128,1,float16,float16,63,0.2485935926437378
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,float16,1,0.012452799826860428
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,fp8,1,0.010411199927330018
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,float16,3,0.012491200119256973
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,48,48,128,1,float16,float16,127,0.2583168029785156
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,fp8,3,0.010409600287675857
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,float16,7,0.012451200187206269
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,48,48,128,1,float16,fp8,127,0.20110878944396973
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,float16,15,0.012542399764060973
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,fp8,15,0.010401599854230881
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,fp8,7,0.010411199927330018
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,fp8,31,0.010387200117111205
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,float16,31,0.012451200187206269
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,fp8,63,0.010417599976062775
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,float16,127,0.01255200058221817
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,fp8,127,0.010406400263309478
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,fp8,255,0.010396800190210342
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,float16,511,0.02080159932374954
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,fp8,511,0.012457600235939026
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,float16,1023,0.03914720118045807
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,fp8,1023,0.018164800107479097
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,float16,63,0.010409600287675857
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,float16,2047,0.06577759981155396
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,fp8,2047,0.03829759955406189
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,48,48,128,1,float16,fp8,1,0.3901103973388672
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,48,48,128,1,float16,float16,1,0.48535680770874023
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,float16,31,0.1296239972114563
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,float16,127,0.13558720350265502
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,48,48,128,1,float16,fp8,3,0.3912992000579834
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,48,48,128,1,float16,float16,3,0.48584961891174316
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,48,48,128,1,float16,float16,7,0.48420162200927735
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,float16,63,0.13144479990005492
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,48,48,128,1,float16,fp8,7,0.39126560688018797
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,48,48,128,1,float16,fp8,15,0.3913520097732544
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,48,48,128,1,float16,float16,15,0.48461599349975587
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,48,48,128,1,float16,float16,31,0.4845119953155518
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,48,48,128,1,float16,fp8,31,0.3896944046020508
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,48,48,128,1,float16,float16,63,0.4856704235076904
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,48,48,128,1,float16,fp8,63,0.39052639007568357
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,48,48,128,1,float16,float16,1,0.9632160186767578
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,48,48,128,1,float16,fp8,1,0.7739280223846435
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,48,48,128,1,float16,float16,3,0.9632431983947753
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,float16,63,0.012451200187206269
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,48,48,128,1,float16,fp8,3,0.7743360042572022
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,48,48,128,1,float16,float16,7,0.964140796661377
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,float16,1,0.016542400419712066
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,48,48,128,1,float16,fp8,7,0.7740287780761719
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,float16,3,0.016521599888801575
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,48,48,128,1,float16,float16,15,0.9620287895202637
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,fp8,3,0.013761599361896516
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,fp8,7,0.013577599823474885
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,48,48,128,1,float16,fp8,31,0.7729040145874023
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,float16,15,0.016569599509239197
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,fp8,15,0.013327999413013459
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,float16,31,0.016548800468444824
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,float16,7,0.016631999611854555
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,fp8,31,0.013471999764442444
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,float16,63,0.01656000018119812
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,fp8,63,0.012503999471664428
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,fp8,127,0.013473600149154663
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,float16,255,0.021814399957656862
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,fp8,255,0.012555199861526489
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,float16,511,0.040587198734283444
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,fp8,511,0.01903360038995743
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,fp8,1023,0.03830400109291077
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,float16,2047,0.12409600019454955
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,fp8,2047,0.06628000140190124
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,float16,1,0.033327999711036685
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,float16,3,0.03331199884414673
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,float16,255,0.012438400089740754
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,fp8,1,0.013865600526332855
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,float16,7,0.03327040076255798
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,float16,15,0.03314880132675171
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,48,48,128,1,float16,fp8,15,0.7726880073547363
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,float16,31,0.033062401413917544
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,float16,127,0.01652639955282211
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,48,48,128,1,float16,float16,31,0.9632783889770508
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,float16,63,0.034995201230049136
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,float16,1023,0.06833440065383911
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,float16,127,0.037520000338554384
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,float16,255,0.05746080279350281
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,fp8,1,0.026923200488090514
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,float16,511,0.10261280536651611
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,fp8,511,0.05737280249595642
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,float16,1023,0.19273920059204103
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,fp8,3,0.027592000365257264
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,fp8,1023,0.1042672038078308
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,fp8,7,0.02771199941635132
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,float16,1,0.008856000006198883
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,fp8,2047,0.19896479845046997
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,float16,3,0.008441600203514098
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,fp8,15,0.027647998929023743
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,fp8,31,0.026819199323654175
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,float16,7,0.009380800276994705
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,float16,15,0.008446399867534638
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,fp8,63,0.026895999908447266
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,fp8,127,0.028923198580741882
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,float16,31,0.008472000062465668
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,fp8,255,0.03315359950065613
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,float16,63,0.008720000088214875
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,float16,127,0.009491200000047684
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,fp8,1,0.00841279998421669
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,float16,255,0.008419200032949447
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,fp8,255,0.008408000320196151
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,float16,511,0.008451200276613235
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,fp8,3,0.008387199789285659
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,fp8,511,0.008363199979066848
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,float16,1023,0.010491199791431427
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,fp8,1023,0.008446399867534638
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,float16,2047,0.012433599680662155
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,fp8,15,0.008379200100898742
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,fp8,2047,0.010366400331258773
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,float16,1,0.008425600081682205
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,fp8,1,0.008339200168848038
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,float16,3,0.008432000130414962
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,fp8,3,0.007840000092983246
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,fp8,7,0.008417599648237229
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,fp8,7,0.006822399795055389
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,float16,15,0.00843999981880188
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,fp8,15,0.0069760002195835115
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,fp8,31,0.008228799700737
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,float16,63,0.00841279998421669
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,fp8,63,0.008408000320196151
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,fp8,31,0.00843999981880188
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,fp8,63,0.008337599784135818
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,float16,127,0.008372800052165985
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,float16,255,0.008451200276613235
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,fp8,255,0.008459199965000153
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,float16,511,0.010375999659299851
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,fp8,511,0.008392000198364257
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,float16,1023,0.012460800260305405
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,fp8,1023,0.0094480000436306
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,float16,2047,0.01660960018634796
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,fp8,2047,0.012534399330615998
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,float16,1,0.02070239931344986
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,float16,3,0.020721599459648132
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,fp8,1,0.01658879965543747
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,fp8,3,0.01660960018634796
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,float16,7,0.02067359983921051
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,fp8,7,0.016625599563121797
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,float16,15,0.020678399503231047
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,fp8,15,0.016630400717258454
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,float16,31,0.020683200657367708
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,fp8,31,0.016633599996566772
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,float16,63,0.020769600570201874
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,fp8,63,0.016739200055599212
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,float16,127,0.02274080067873001
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,fp8,127,0.017052799463272095
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,float16,255,0.035041600465774536
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,float16,511,0.057529598474502563
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,fp8,511,0.03346880078315735
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,float16,1023,0.10259679555892945
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,fp8,1023,0.05758399963378906
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,float16,2047,0.193012797832489
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,fp8,2047,0.10526399612426758
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,float16,7,0.008366400003433227
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,float16,1,0.01043360009789467
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,float16,31,0.008452799916267396
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,float16,3,0.01043040007352829
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,fp8,3,0.00835679993033409
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,fp8,7,0.008451200276613235
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,float16,15,0.010390400141477584
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,fp8,127,0.008408000320196151
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,fp8,15,0.008441600203514098
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,fp8,127,0.008376000076532364
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,float16,31,0.010491199791431427
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,float16,63,0.010424000024795533
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,fp8,63,0.008428800106048583
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,float16,127,0.01037440001964569
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,fp8,127,0.008395200222730636
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,fp8,31,0.008436799794435502
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,float16,255,0.010515200346708298
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,fp8,255,0.008395200222730636
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,fp8,511,0.010384000092744827
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,float16,1023,0.01701119989156723
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,fp8,1023,0.012548799812793731
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,float16,2047,0.04091359972953797
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,fp8,2047,0.018648000061511995
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,float16,1,0.05811039805412292
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,fp8,1,0.047270399332046506
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,float16,3,0.058303999900817874
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,fp8,3,0.04729120135307312
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,float16,7,0.058427202701568606
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,fp8,7,0.04727360010147095
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,fp8,255,0.018374399840831758
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,float16,31,0.061054402589797975
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,float16,15,0.058248001337051394
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,fp8,31,0.047307199239730834
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,float16,63,0.06158080101013184
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,fp8,63,0.04824959933757782
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,fp8,127,0.0493120014667511
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,float16,255,0.10285760164260864
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,fp8,255,0.058550399541854856
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,float16,511,0.1930608034133911
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,fp8,511,0.10353599786758423
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,fp8,1,0.008342400193214417
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,float16,1023,0.3727231979370117
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,float16,7,0.01043040007352829
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,float16,1,0.10692800283432007
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,float16,2047,0.37249600887298584
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,float16,3,0.10713920593261719
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,fp8,1,0.08732479810714722
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,float16,7,0.10736479759216308
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,float16,15,0.10956000089645386
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,fp8,15,0.08759520053863526
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,float16,31,0.10955840349197388
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,fp8,31,0.08852159976959229
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,float16,63,0.11034719944000244
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,fp8,63,0.08840960264205933
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,float16,127,0.11496959924697876
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,float16,511,0.012476799637079239
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,fp8,127,0.08872159719467163
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,fp8,255,0.10869439840316772
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,40,40,128,1,float16,float16,1,0.2057487964630127
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,fp8,15,0.04733920097351074
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,40,40,128,1,float16,fp8,1,0.1670624017715454
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,40,40,128,1,float16,float16,3,0.20755839347839355
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,40,40,128,1,float16,fp8,3,0.1664576053619385
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,40,40,128,1,float16,fp8,7,0.16689599752426149
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,40,40,128,1,float16,float16,7,0.20871200561523437
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,40,40,128,1,float16,float16,15,0.20790560245513917
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,40,40,128,1,float16,fp8,15,0.16832640171051025
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,40,40,128,1,float16,float16,31,0.2085200071334839
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,float16,127,0.06372479796409607
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,40,40,128,1,float16,fp8,31,0.16835520267486573
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,40,40,128,1,float16,float16,63,0.20866720676422118
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,40,40,128,1,float16,float16,127,0.21744320392608643
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,40,40,128,1,float16,fp8,127,0.16960639953613282
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,float16,1,0.012439999729394913
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,fp8,1,0.010395199805498124
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,float16,3,0.012451200187206269
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,fp8,3,0.010438399761915207
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,float16,7,0.012428800016641617
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,fp8,1023,0.19804799556732178
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,fp8,7,0.010395199805498124
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,float16,15,0.012300799787044524
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,float16,31,0.010921599715948105
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,fp8,31,0.01037919968366623
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,fp8,15,0.010395199805498124
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,float16,63,0.011001600325107575
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,fp8,63,0.010395199805498124
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,float16,127,0.01242239996790886
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,fp8,127,0.010353600233793258
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,float16,255,0.010908800363540649
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,fp8,3,0.08700960278511047
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,fp8,255,0.010414399951696397
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,float16,511,0.016499200463294984
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,fp8,511,0.012459199875593185
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,float16,1023,0.03687199950218201
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,fp8,1023,0.016624000668525696
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,fp8,7,0.08712800145149231
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,float16,2047,0.061654400825500486
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,40,40,128,1,float16,fp8,1,0.32577760219573976
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,40,40,128,1,float16,float16,1,0.40551037788391114
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,40,40,128,1,float16,float16,3,0.4055791854858398
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,40,40,128,1,float16,fp8,3,0.32665600776672366
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,40,40,128,1,float16,fp8,7,0.32728800773620603
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,40,40,128,1,float16,float16,7,0.40555357933044434
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,40,40,128,1,float16,float16,15,0.4056992053985596
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,float16,255,0.19465600252151488
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,40,40,128,1,float16,fp8,15,0.3277103900909424
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,40,40,128,1,float16,fp8,31,0.3267024040222168
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,40,40,128,1,float16,float16,31,0.40570878982543945
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,40,40,128,1,float16,float16,63,0.4057663917541504
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,40,40,128,1,float16,float16,1,0.8053839683532715
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,40,40,128,1,float16,fp8,1,0.6458928108215332
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,40,40,128,1,float16,float16,3,0.804355239868164
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,40,40,128,1,float16,fp8,3,0.6459280014038086
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,40,40,128,1,float16,fp8,63,0.1682800054550171
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,40,40,128,1,float16,fp8,7,0.6456192016601563
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,40,40,128,1,float16,float16,7,0.8046704292297363
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,float16,1,0.014473600685596466
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,40,40,128,1,float16,fp8,15,0.6457600116729736
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,fp8,1,0.012444800138473511
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,40,40,128,1,float16,float16,15,0.803059196472168
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,fp8,3,0.012542399764060973
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,float16,7,0.014508800208568573
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,fp8,7,0.012503999471664428
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,float16,15,0.014504000544548035
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,fp8,15,0.012454400211572647
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,fp8,2047,0.03501439988613129
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,float16,31,0.014496000111103058
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,fp8,31,0.012441600114107132
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,float16,63,0.014483200013637542
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,float16,127,0.014499199390411378
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,fp8,127,0.012457600235939026
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,float16,255,0.014681600034236908
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,fp8,255,0.012462399899959564
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,fp8,511,0.016630400717258454
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,float16,511,0.03497759997844696
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,float16,1023,0.05768479704856873
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,fp8,1023,0.03320159912109375
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,40,40,128,1,float16,float16,31,0.8031599998474122
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,float16,2047,0.1042672038078308
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,fp8,2047,0.05755680203437805
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,float16,3,0.028832000494003297
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,float16,7,0.028854399919509888
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,fp8,3,0.02288320064544678
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,float16,15,0.02880159914493561
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,fp8,7,0.022889600694179536
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,fp8,15,0.022742399573326112
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,float16,31,0.028860801458358766
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,fp8,31,0.0227743998169899
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,fp8,63,0.02277279943227768
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,40,40,128,1,float16,fp8,63,0.3264591932296753
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,float16,127,0.032204800844192506
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,float16,255,0.04735040068626404
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,fp8,255,0.028835201263427736
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,fp8,127,0.02316640019416809
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,float16,511,0.08411359786987305
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,fp8,511,0.04776000082492828
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,float16,1023,0.15606720447540284
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,fp8,1023,0.08611199855804444
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,fp8,2047,0.16125439405441283
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,float16,2047,0.30027360916137696
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,float16,3,0.014646400511264802
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,float16,1,0.008347199857234954
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,40,40,128,1,float16,fp8,31,0.6449584007263184
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,fp8,63,0.012432000041007996
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,fp8,4095,0.3131824016571045
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,float16,3,0.008444800227880477
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,float16,4095,0.5910399913787842
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,float16,7,0.00844319984316826
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,float16,15,0.00846880003809929
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,float16,1,0.02884480059146881
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,float16,31,0.008462399989366532
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,fp8,1,0.022735999524593355
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,float16,63,0.008423999696969987
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,float16,63,0.028803199529647827
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,float16,127,0.008422400057315826
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,fp8,1,0.008416000008583068
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,float16,255,0.008393599838018417
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,fp8,255,0.008404800295829773
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,float16,511,0.00841279998421669
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,fp8,511,0.00835679993033409
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,fp8,3,0.00843999981880188
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,float16,1023,0.010452800244092942
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,fp8,15,0.008364800363779068
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,fp8,1023,0.008454400300979614
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,fp8,2047,0.009399999678134919
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,fp8,7,0.008390399813652038
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,float16,4095,0.012508800625801087
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,fp8,4095,0.01048320010304451
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,float16,1,0.01029599979519844
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,float16,3,0.008643200248479843
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,fp8,1,0.008392000198364257
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,fp8,3,0.008425600081682205
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,fp8,31,0.008371199667453765
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,float16,7,0.00897120013833046
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,fp8,7,0.008347199857234954
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,float16,15,0.00902239978313446
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,float16,31,0.008764799684286118
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,fp8,15,0.00843840017914772
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,float16,63,0.009161599725484849
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,fp8,31,0.008422400057315826
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,fp8,63,0.008401600271463394
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,float16,127,0.010428799688816071
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,fp8,127,0.008392000198364257
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,float16,255,0.010283199697732925
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,fp8,255,0.008408000320196151
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,float16,511,0.010294400155544281
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,fp8,511,0.008363199979066848
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,fp8,1023,0.008432000130414962
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,float16,2047,0.01252640038728714
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,fp8,63,0.0084927998483181
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,fp8,4095,0.012535999715328216
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,float16,1,0.018697600066661834
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,fp8,1,0.014567999541759491
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,float16,3,0.018671999871730804
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,fp8,3,0.014577600359916686
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,float16,7,0.01858399957418442
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,fp8,7,0.014587199687957764
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,float16,15,0.01873439997434616
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,fp8,15,0.01449279934167862
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,float16,31,0.018719999492168425
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,fp8,31,0.014580799639225006
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,float16,63,0.018595199286937713
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,fp8,63,0.014563199877738953
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,float16,127,0.018681600689888
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,fp8,127,0.014563199877738953
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,float16,255,0.028916800022125246
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,fp8,255,0.014556799829006196
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,float16,511,0.047537600994110106
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,fp8,511,0.028784000873565675
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,fp8,1023,0.04735040068626404
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,float16,2047,0.15575519800186158
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,fp8,2047,0.08635039925575257
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,float16,4095,0.30197279453277587
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,float16,1,0.008455999940633774
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,fp8,4095,0.16211999654769899
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,float16,2047,0.01066880002617836
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,fp8,1,0.008348800241947174
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,fp8,3,0.00843840017914772
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,float16,7,0.00843999981880188
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,fp8,7,0.00840959995985031
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,float16,15,0.008379200100898742
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,fp8,15,0.008419200032949447
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,float16,31,0.008539199829101562
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,fp8,31,0.00841120034456253
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,float16,63,0.0087567999958992
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,fp8,63,0.008345600217580795
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,float16,1023,0.010766399651765823
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,fp8,127,0.0083856001496315
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,fp8,2047,0.010409600287675857
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,float16,4095,0.029110398888587952
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,float16,255,0.008486399799585343
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,fp8,255,0.00835840031504631
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,float16,511,0.010427200049161912
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,fp8,127,0.008348800241947174
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,float16,1023,0.012486399710178375
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,float16,2047,0.028814399242401124
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,fp8,1023,0.010428799688816071
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,fp8,2047,0.012451200187206269
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,float16,4095,0.047116801142692566
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,fp8,4095,0.027772799134254456
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,float16,1,0.04836640059947968
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,fp8,1,0.03931039869785309
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,float16,3,0.04816960096359253
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,float16,7,0.0483951985836029
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,fp8,7,0.03918560147285462
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,float16,15,0.04772000014781952
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,fp8,15,0.03936159908771515
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,float16,31,0.049132800102233885
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,fp8,31,0.03946560025215149
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,float16,63,0.05148320198059082
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,fp8,63,0.03919839859008789
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,float16,1023,0.08409439921379089
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,float16,127,0.05341280102729797
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,fp8,127,0.041198399662971494
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,float16,255,0.08422240018844604
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,fp8,255,0.048135998845100406
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,float16,511,0.15607359409332275
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,fp8,511,0.0843168020248413
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,float16,1023,0.30070559978485106
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,fp8,1023,0.16019519567489623
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,float16,1,0.0876416027545929
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,fp8,1,0.0718991994857788
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,float16,2047,0.5887839794158936
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,float16,3,0.0871295988559723
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,fp8,2047,0.3116048097610474
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,float16,7,0.08736799955368042
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,fp8,7,0.07198079824447631
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,float16,15,0.08999040126800537
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,fp8,15,0.07189440131187438
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,float16,31,0.09037920236587524
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,fp8,31,0.07194719910621643
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,fp8,3,0.07192000150680541
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,fp8,63,0.07221760153770447
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,float16,127,0.008852799981832504
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,fp8,127,0.07234240174293519
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,fp8,511,0.008433599770069123
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,float16,63,0.09036160111427308
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,float16,255,0.1579327940940857
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,fp8,255,0.08710399866104127
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,float16,127,0.09361600279808044
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,fp8,3,0.03917120099067688
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,fp8,1,0.13550080060958863
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,float16,511,0.3015072107315063
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,float16,3,0.16674400568008424
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,fp8,3,0.13539520502090455
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,float16,7,0.16835999488830566
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,fp8,511,0.15911840200424193
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,fp8,7,0.1356943964958191
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,fp8,15,0.13580000400543213
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,float16,31,0.16854560375213623
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,fp8,63,0.13577280044555665
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,float16,127,0.17571680545806884
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,fp8,31,0.13616960048675536
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,float16,255,0.3046816110610962
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,fp8,127,0.13659839630126952
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,fp8,255,0.16776959896087645
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,fp8,1,0.00841120034456253
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,float16,1,0.010419200360774993
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,float16,3,0.010425599664449692
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,fp8,3,0.008452799916267396
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,float16,7,0.010416000336408614
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,fp8,7,0.008428800106048583
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,float16,15,0.010449600219726563
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,fp8,15,0.008417599648237229
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,float16,31,0.01043360009789467
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,fp8,31,0.008505599945783615
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,float16,63,0.01037440001964569
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,fp8,63,0.008476799726486206
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,float16,127,0.010489600151777268
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,fp8,127,0.008459199965000153
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,fp8,255,0.00843840017914772
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,float16,511,0.012459199875593185
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,float16,3,0.008436799794435502
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,fp8,511,0.01048320010304451
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,float16,1023,0.028804799914360045
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,fp8,1023,0.013254399597644805
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,float16,2047,0.04736000001430511
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,float16,4095,0.08461920022964478
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,fp8,4095,0.0478736013174057
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,32,128,1,float16,float16,1,0.3254431962966919
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,32,128,1,float16,float16,3,0.3271647930145264
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,float16,1,0.16642080545425414
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,32,128,1,float16,fp8,3,0.2630543947219849
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,32,128,1,float16,float16,7,0.3268752098083496
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,float16,15,0.16858079433441162
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,32,128,1,float16,fp8,7,0.26444640159606936
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,float16,63,0.16896320581436158
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,32,128,1,float16,fp8,15,0.26283040046691897
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,32,128,1,float16,float16,15,0.3265696048736572
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,32,128,1,float16,float16,31,0.32671360969543456
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,32,128,1,float16,float16,63,0.3263999938964844
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,32,128,1,float16,fp8,63,0.2632767915725708
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,32,128,1,float16,fp8,31,0.2634063959121704
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,32,128,1,float16,float16,127,0.33855040073394777
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,32,128,1,float16,fp8,127,0.2645312070846558
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,32,128,1,float16,fp8,1,0.518400001525879
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,32,128,1,float16,float16,1,0.6451424121856689
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,float16,255,0.010420800000429154
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,32,128,1,float16,fp8,3,0.5192048072814941
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,32,128,1,float16,float16,7,0.6448512077331543
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,32,128,1,float16,fp8,7,0.5186287879943847
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,32,128,1,float16,float16,3,0.645359992980957
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,32,128,1,float16,float16,15,0.6447840213775635
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,32,128,1,float16,fp8,15,0.5181183815002441
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,float16,1,0.012891200184822083
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,fp8,1,0.01050880029797554
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,float16,3,0.012639999389648438
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,32,128,1,float16,float16,31,0.6454559803009033
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,fp8,3,0.010460799932479859
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,32,128,1,float16,fp8,31,0.5179599761962891
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,float16,7,0.012534399330615998
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,fp8,7,0.010475199669599533
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,float16,15,0.012652799487113953
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,float16,31,0.01252480000257492
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,32,128,1,float16,fp8,63,0.518616008758545
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,32,128,1,float16,float16,63,0.6451072216033935
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,fp8,31,0.010558400303125381
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,float16,63,0.01255200058221817
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,fp8,127,0.010548800230026245
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,fp8,63,0.010491199791431427
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,float16,255,0.012542399764060973
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,float16,127,0.012555199861526489
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,fp8,255,0.010463999956846238
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,fp8,511,0.014574399590492249
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,float16,1023,0.04758560061454773
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,fp8,2047,0.04755519926548004
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,float16,2047,0.08561919927597046
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,float16,4095,0.1621359944343567
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,float16,1,0.022694399952888487
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,fp8,4095,0.08582720160484314
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,float16,3,0.022759999334812164
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,fp8,1,0.01903519928455353
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,float16,7,0.02281759977340698
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,fp8,3,0.018985599279403687
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,fp8,7,0.019276799261569978
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,float16,15,0.022793599963188173
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,fp8,15,0.019497600197792054
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,float16,31,0.02276639938354492
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,fp8,31,0.018697600066661834
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,float16,63,0.02274879962205887
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,fp8,63,0.018611200153827667
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,fp8,127,0.018612800538539885
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,fp8,2047,0.0287200003862381
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,fp8,255,0.0206496000289917
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,32,128,1,float16,fp8,1,0.26260640621185305
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,fp8,15,0.010515200346708298
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,fp8,511,0.038736000657081604
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,float16,1023,0.11934239864349365
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,fp8,1023,0.06698399782180786
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,float16,511,0.028857600688934327
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,fp8,1023,0.02898240089416504
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,fp8,2047,0.12377279996871948
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,float16,2047,0.22821600437164308
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,float16,1,0.010366400331258773
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,float16,3,0.01037919968366623
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,float16,4095,0.4456416130065918
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,fp8,4095,0.23919360637664794
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,float16,7,0.009444800019264222
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,float16,15,0.010401599854230881
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,float16,31,0.010424000024795533
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,float16,127,0.02483839988708496
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,float16,63,0.010462400317192078
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,float16,255,0.03792479932308197
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,float16,127,0.010385599732398988
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,float16,511,0.06567360162734985
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,float16,255,0.010417599976062775
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,fp8,255,0.008507200330495835
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,fp8,1,0.008436799794435502
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,float16,511,0.010407999902963639
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,fp8,511,0.008591999858617782
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,fp8,7,0.00846560001373291
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,fp8,1023,0.01040000021457672
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,float16,2047,0.012577599287033081
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,float16,4095,0.012542399764060973
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,fp8,15,0.008472000062465668
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,fp8,4095,0.011035200208425522
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,float16,1,0.010387200117111205
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,fp8,1,0.00846719965338707
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,float16,3,0.010419200360774993
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,fp8,3,0.00870719999074936
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,float16,7,0.010094399750232696
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,fp8,7,0.008516799658536911
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,float16,15,0.010288000106811523
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,fp8,15,0.00888800024986267
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,float16,31,0.010427200049161912
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,fp8,31,0.008667200058698653
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,fp8,31,0.008603200316429138
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,float16,63,0.010332799702882766
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,float16,127,0.010478399693965912
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,fp8,63,0.008750399947166443
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,fp8,127,0.008460800349712371
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,fp8,255,0.008448000252246856
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,float16,511,0.010462400317192078
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,fp8,63,0.008488000184297562
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,float16,1023,0.012427199631929398
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,float16,2047,0.012510399520397186
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,fp8,127,0.008376000076532364
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,fp8,2047,0.010500799864530563
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,float16,4095,0.020865599811077117
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,fp8,4095,0.012612800300121307
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,float16,1,0.01653439998626709
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,fp8,1,0.01379680037498474
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,float16,3,0.015992000699043274
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,fp8,3,0.013483199477195739
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,float16,7,0.016340799629688263
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,fp8,7,0.012544000148773193
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,float16,15,0.01652639955282211
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,fp8,15,0.0137472003698349
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,float16,31,0.01510239988565445
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,fp8,31,0.012564800679683685
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,float16,63,0.014804799854755402
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,fp8,63,0.012561599910259246
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,float16,127,0.014707200229167938
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,fp8,127,0.012531200051307678
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,float16,255,0.01870879977941513
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,fp8,255,0.012545600533485413
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,float16,1023,0.010424000024795533
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,fp8,2047,0.01048159971833229
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,float16,511,0.03903680145740509
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,float16,1023,0.06641280055046081
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,fp8,1023,0.03895359933376312
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,float16,2047,0.1220479965209961
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,fp8,2047,0.06826879978179931
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,float16,4095,0.23350241184234619
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,fp8,4095,0.1263808012008667
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,float16,1,0.008441600203514098
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,fp8,511,0.009467200189828873
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,float16,3,0.008337599784135818
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,fp8,3,0.007516799867153168
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,float16,7,0.008441600203514098
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,fp8,7,0.007924800366163253
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,float16,15,0.008425600081682205
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,fp8,15,0.007822400331497193
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,float16,31,0.008352000266313553
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,fp8,31,0.007233600318431854
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,fp8,63,0.008329600095748901
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,float16,127,0.008366400003433227
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,fp8,127,0.008316799998283386
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,float16,255,0.008388800173997879
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,fp8,255,0.008336000144481659
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,float16,511,0.010401599854230881
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,fp8,511,0.008444800227880477
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,float16,1023,0.011407999694347382
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,fp8,1023,0.010361599922180175
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,float16,2047,0.018995200097560883
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,fp8,2047,0.012467200309038163
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,float16,4095,0.037569600343704226
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,fp8,3,0.00846719965338707
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,fp8,4095,0.019308799505233766
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,float16,1,0.03821919858455658
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,fp8,1,0.031017601490020752
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,float16,3,0.03784799873828888
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,fp8,511,0.019079999625682832
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,fp8,3,0.031067198514938353
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,float16,7,0.03795199990272522
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,float16,15,0.03770720064640045
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,fp8,15,0.0308896005153656
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,float16,31,0.0378383994102478
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,fp8,31,0.030980798602104186
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,float16,63,0.03924799859523773
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,fp8,63,0.030900800228118898
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,float16,127,0.041193601489067075
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,fp8,127,0.03261280059814453
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,float16,255,0.0656607985496521
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,fp8,255,0.03842560052871704
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,float16,511,0.11916160583496094
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,fp8,511,0.06610720157623291
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,float16,255,0.010478399693965912
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,float16,1023,0.22770240306854247
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,fp8,1,0.008355200290679932
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,fp8,1023,0.12307679653167725
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,fp8,1023,0.010480000078678131
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,float16,1,0.06814399957656861
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,float16,3,0.06821759939193725
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,fp8,3,0.055550402402877806
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,float16,7,0.06793280243873596
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,fp8,7,0.05573599934577942
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,float16,2047,0.4434192180633545
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,fp8,2047,0.2371920108795166
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,float16,15,0.06834239959716797
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,float16,63,0.008423999696969987
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,float16,31,0.06997759938240052
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,float16,127,0.07241759896278381
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,fp8,63,0.05687040090560913
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,float16,63,0.07044320106506348
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,fp8,255,0.06872640252113342
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,float16,255,0.12104480266571045
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,float16,511,0.22796640396118165
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,fp8,511,0.12256959676742554
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,fp8,1,0.10331039428710938
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,float16,3,0.12742559909820556
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,fp8,3,0.10312479734420776
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,float16,7,0.12760000228881835
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,fp8,7,0.10310560464859009
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,float16,15,0.12935999631881714
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,fp8,15,0.103329598903656
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,float16,31,0.12924959659576415
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,fp8,31,0.10474239587783814
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,float16,63,0.12941919565200805
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,fp8,63,0.10465600490570068
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,fp8,7,0.031108799576759338
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,float16,127,0.1340432047843933
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,fp8,127,0.10538079738616943
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,float16,1,0.009950400143861771
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,fp8,1,0.008403199911117553
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,float16,255,0.23056159019470215
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,float16,3,0.010041599720716476
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,fp8,3,0.00843840017914772
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,float16,7,0.01045759990811348
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,fp8,7,0.008441600203514098
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,float16,15,0.010444799810647965
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,fp8,15,0.008408000320196151
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,fp8,1,0.055448001623153685
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,fp8,31,0.008420799672603608
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,float16,63,0.01043199971318245
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,fp8,15,0.055579197406768796
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,fp8,63,0.008488000184297562
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,float16,127,0.010390400141477584
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,fp8,127,0.008451200276613235
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,float16,255,0.010371199995279311
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,fp8,255,0.008390399813652038
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,fp8,511,0.01043360009789467
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,fp8,31,0.05568159818649292
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,float16,1023,0.02074880003929138
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,fp8,1023,0.012535999715328216
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,float16,2047,0.04119200110435486
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,fp8,2047,0.02054399996995926
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,fp8,4095,0.04273279905319214
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,float16,1,0.12767360210418702
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,24,24,128,1,float16,float16,1,0.24503040313720703
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,24,24,128,1,float16,fp8,1,0.19919359683990479
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,24,24,128,1,float16,float16,3,0.24609920978546143
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,24,24,128,1,float16,float16,7,0.24706881046295165
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,24,24,128,1,float16,fp8,7,0.1990895986557007
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,24,24,128,1,float16,float16,15,0.24744000434875488
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,24,24,128,1,float16,fp8,15,0.20030720233917237
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,fp8,255,0.1276927947998047
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,24,24,128,1,float16,float16,31,0.24762721061706544
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,24,24,128,1,float16,fp8,31,0.1996816039085388
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,float16,31,0.01048159971833229
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,24,24,128,1,float16,float16,63,0.24793760776519774
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,24,24,128,1,float16,fp8,63,0.19990559816360473
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,24,24,128,1,float16,float16,127,0.25651519298553466
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,24,24,128,1,float16,fp8,127,0.20194559097290038
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,float16,511,0.012432000041007996
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,24,24,128,1,float16,float16,1,0.48492159843444826
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,24,24,128,1,float16,fp8,1,0.3912384033203125
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,float16,4095,0.07195360064506531
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,24,24,128,1,float16,float16,3,0.4853407859802246
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,24,24,128,1,float16,fp8,3,0.3915103912353516
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,24,24,128,1,float16,float16,7,0.4851359844207764
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,24,24,128,1,float16,fp8,7,0.3916192054748535
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,24,24,128,1,float16,float16,15,0.4848320007324219
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,24,24,128,1,float16,fp8,31,0.3912751913070679
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,float16,1,0.0106175996363163
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,fp8,1,0.010422399640083313
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,24,24,128,1,float16,float16,63,0.4854591846466064
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,24,24,128,1,float16,fp8,15,0.3912064075469971
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,fp8,3,0.010476800054311753
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,24,24,128,1,float16,float16,31,0.48534560203552246
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,float16,7,0.012334399670362473
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,float16,15,0.010454399883747101
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,float16,31,0.010718400031328202
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,fp8,31,0.010462400317192078
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,fp8,15,0.010411199927330018
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,float16,63,0.011638399958610535
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,fp8,63,0.010539200156927109
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,float16,127,0.01175839975476265
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,fp8,127,0.010395199805498124
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,float16,255,0.01088479980826378
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,fp8,255,0.01040479987859726
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,float16,511,0.018587200343608855
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,fp8,511,0.012521600723266602
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,float16,1023,0.03803359866142273
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,fp8,1023,0.018248000741004945
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,float16,2047,0.06548320055007935
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,fp8,2047,0.038227200508117676
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,float16,1,0.017507199943065644
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,float16,4095,0.12070720195770264
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,fp8,4095,0.06682400107383728
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,float16,3,0.016627199947834015
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,float16,7,0.017867200076580048
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,fp8,7,0.014603200554847717
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,float16,15,0.01788160055875778
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,fp8,15,0.014606399834156037
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,float16,31,0.016700799763202667
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,fp8,31,0.014504000544548035
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,float16,63,0.016777600347995757
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,fp8,63,0.014595200121402741
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,fp8,127,0.057599997520446776
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,float16,127,0.01839359998703003
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,float16,255,0.02826719880104065
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,fp8,255,0.014628799259662628
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,fp8,127,0.014616000652313232
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,float16,511,0.04679040014743805
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,fp8,511,0.028915199637413024
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,float16,1023,0.08276640176773072
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,fp8,1023,0.04836640059947968
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,fp8,2047,0.08760960102081299
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,float16,2047,0.15575679540634155
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,24,24,128,1,float16,fp8,3,0.19928640127182007
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,float16,3,0.010628800094127654
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,fp8,4095,0.1633407950401306
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,fp8,7,0.010465600341558457
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,float16,4095,0.2990463972091675
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,float16,1,0.009398400038480758
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,float16,3,0.008470399677753449
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,float16,8191,0.5899456024169922
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,fp8,8191,0.31848480701446535
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,float16,7,0.008481600135564805
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,float16,15,0.008446399867534638
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,fp8,1,0.01459999978542328
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,fp8,3,0.014891199767589569
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,float16,31,0.008875200152397155
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,float16,63,0.008548799902200699
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,fp8,1,0.008455999940633774
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,fp8,3,0.008462399989366532
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,float16,127,0.0104032002389431
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,fp8,127,0.008353599905967712
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,float16,255,0.010377600044012069
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,fp8,255,0.008406399935483932
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,float16,511,0.010366400331258773
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,fp8,511,0.008425600081682205
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,float16,1023,0.01040000021457672
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,fp8,7,0.00843840017914772
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,fp8,1023,0.008854400366544724
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,float16,2047,0.01037919968366623
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,fp8,2047,0.010420800000429154
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,fp8,4095,0.011416000127792359
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,float16,8191,0.014585599303245544
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,fp8,8191,0.012580800056457519
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,float16,1,0.008448000252246856
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,fp8,31,0.008459199965000153
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,24,24,128,1,float16,fp8,63,0.3912287950515747
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,fp8,1,0.008343999832868576
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,fp8,3,0.008715199679136277
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,fp8,7,0.008337599784135818
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,fp8,63,0.00844319984316826
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,float16,15,0.008408000320196151
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,fp8,15,0.008339200168848038
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,float16,31,0.008377599716186523
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,fp8,31,0.008454400300979614
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,float16,63,0.008457600325345992
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,fp8,63,0.008363199979066848
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,fp8,127,0.008353599905967712
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,float16,255,0.008446399867534638
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,fp8,255,0.008401600271463394
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,float16,511,0.00835679993033409
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,fp8,511,0.008303999900817871
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,float16,1023,0.00865119993686676
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,fp8,1023,0.008406399935483932
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,float16,2047,0.0104032002389431
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,fp8,2047,0.010422399640083313
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,float16,4095,0.012540799379348756
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,fp8,4095,0.010470400005578995
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,float16,8191,0.028772801160812378
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,fp8,8191,0.014422400295734406
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,float16,1,0.0124719999730587
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,fp8,15,0.008380799740552902
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,fp8,1,0.010462400317192078
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,float16,3,0.012518399953842163
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,float16,7,0.01249760016798973
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,fp8,7,0.0105103999376297
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,fp8,3,0.010520000010728836
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,float16,15,0.01244639977812767
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,fp8,15,0.010398399829864503
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,float16,31,0.012548799812793731
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,fp8,31,0.010468800365924836
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,float16,63,0.012465599924325943
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,fp8,63,0.010699199885129929
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,float16,127,0.012428800016641617
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,fp8,127,0.010505600273609162
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,float16,255,0.012510399520397186
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,fp8,255,0.010599999874830245
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,fp8,511,0.014505599439144135
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,fp8,1023,0.028891199827194215
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,float16,1023,0.04723039865493774
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,float16,2047,0.0842191994190216
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,float16,4095,0.15932960510253907
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,fp8,4095,0.08669919967651367
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,float16,8191,0.3103631973266602
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,fp8,8191,0.16498880386352538
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,float16,1,0.008479999750852585
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,float16,4095,0.012428800016641617
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,float16,3,0.008454400300979614
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,fp8,3,0.00833439975976944
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,float16,7,0.008449599891901017
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,fp8,7,0.008321599662303924
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,float16,15,0.008750399947166443
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,float16,3,0.00846880003809929
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,fp8,15,0.008327999711036682
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,float16,31,0.008767999708652496
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,fp8,31,0.008377599716186523
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,float16,7,0.008363199979066848
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,fp8,63,0.00835840031504631
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,float16,127,0.008448000252246856
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,float16,255,0.008564800024032593
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,fp8,255,0.00836160033941269
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,float16,511,0.008491200208663941
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,float16,127,0.008425600081682205
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,fp8,127,0.008420799672603608
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,float16,1023,0.010417599976062775
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,fp8,1023,0.009758400171995163
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,float16,2047,0.012470400333404541
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,fp8,2047,0.010417599976062775
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,float16,4095,0.028679999709129333
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,fp8,4095,0.013728000223636627
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,float16,8191,0.04600479900836944
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,fp8,8191,0.028836798667907716
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,float16,1,0.02698880136013031
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,fp8,1,0.022819200158119203
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,float16,3,0.02691679894924164
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,fp8,3,0.022811199724674224
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,float16,7,0.026902401447296144
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,fp8,7,0.022758400440216063
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,float16,15,0.02710399925708771
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,fp8,15,0.022702400386333466
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,float16,31,0.028041601181030273
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,fp8,31,0.022753599286079406
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,fp8,63,0.022763200104236603
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,float16,127,0.03094080090522766
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,float16,511,0.02879680097103119
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,fp8,127,0.022812800109386445
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,float16,255,0.04695520102977753
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,fp8,255,0.02887359857559204
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,fp8,2047,0.04757919907569885
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,fp8,511,0.047225600481033324
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,fp8,1023,0.08466719985008239
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,float16,1023,0.1556704044342041
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,fp8,2047,0.16136640310287476
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,fp8,1,0.008436799794435502
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,float16,63,0.00843999981880188
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,float16,4095,0.5897200107574463
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,fp8,4095,0.31631999015808104
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,fp8,511,0.008396799862384795
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,fp8,1,0.03927200138568878
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,fp8,3,0.039108800888061526
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,float16,3,0.047775998711586
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,float16,7,0.04762240052223206
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,fp8,7,0.039208000898361205
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,float16,15,0.04745280146598816
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,fp8,15,0.03916960060596466
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,float16,31,0.04873439967632294
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,fp8,31,0.039164799451828006
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,float16,63,0.049414399266242984
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,fp8,63,0.039182400703430174
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,fp8,127,0.04111199975013733
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,float16,255,0.08368800282478332
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,float16,63,0.028403198719024657
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,fp8,255,0.0488864004611969
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,float16,127,0.0514352023601532
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,fp8,511,0.08465440273284912
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,float16,511,0.15576959848403932
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,float16,1,0.08739039897918702
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,float16,1023,0.29915359020233157
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,float16,511,0.08257759809494018
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,fp8,1,0.07190880179405212
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,float16,3,0.087254399061203
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,fp8,1023,0.1590816020965576
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,float16,15,0.08920159935951233
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,fp8,7,0.07191519737243653
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,float16,7,0.087227201461792
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,fp8,3,0.07197920083999634
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,float16,31,0.08859519958496094
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,fp8,31,0.07195199728012085
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,float16,63,0.08884159922599792
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,fp8,63,0.0723360002040863
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,float16,127,0.09231520295143128
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,fp8,127,0.07408159971237183
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,float16,255,0.15637760162353515
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,fp8,255,0.08902879953384399
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,float16,1,0.008339200168848038
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,fp8,1,0.008376000076532364
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,float16,3,0.008452799916267396
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,fp8,3,0.008452799916267396
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,float16,511,0.2998640060424805
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,float16,15,0.008446399867534638
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,float16,31,0.008454400300979614
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,fp8,15,0.00835999995470047
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,float16,7,0.008337599784135818
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,fp8,31,0.008401600271463394
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,float16,63,0.00835679993033409
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,float16,127,0.008441600203514098
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,fp8,127,0.008343999832868576
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,float16,255,0.008446399867534638
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,fp8,255,0.008352000266313553
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,float16,511,0.010480000078678131
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,fp8,63,0.008303999900817871
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,fp8,1023,0.01037919968366623
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,float16,2047,0.027774399518966673
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,fp8,2047,0.012531200051307678
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,float16,4095,0.045598399639129636
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,fp8,4095,0.027880001068115234
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,float16,8191,0.08203359842300414
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,fp8,8191,0.04711839854717255
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,float16,1,0.1662943959236145
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,fp8,1,0.13550879955291747
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,float16,3,0.16621439456939696
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,fp8,15,0.07205600142478943
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,fp8,3,0.13570079803466797
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,float16,7,0.16833280324935912
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,fp8,7,0.13568639755249023
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,float16,2047,0.30002241134643554
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,float16,15,0.1684720039367676
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,fp8,15,0.1364207983016968
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,fp8,31,0.1365023970603943
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,fp8,511,0.15896480083465575
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,float16,63,0.168121600151062
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,float16,31,0.16839840412139892
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,fp8,63,0.1368191957473755
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,fp8,127,0.1377679944038391
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,float16,127,0.17429440021514891
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,float16,1,0.04749920070171356
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,fp8,511,0.00835679993033409
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,float16,1023,0.012441600114107132
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,fp8,255,0.17015360593795775
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,float16,255,0.3035775899887085
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,16,128,1,float16,fp8,1,0.26292479038238525
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,16,128,1,float16,float16,3,0.3259871959686279
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,16,128,1,float16,fp8,3,0.2637343883514404
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,16,128,1,float16,float16,1,0.32482879161834716
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,16,128,1,float16,fp8,7,0.2644912004470825
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,16,128,1,float16,float16,15,0.32613120079040525
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,16,128,1,float16,fp8,15,0.2639359951019287
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,16,128,1,float16,fp8,31,0.2643471956253052
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,16,128,1,float16,float16,31,0.3263423919677734
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,16,128,1,float16,float16,63,0.32647199630737306
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,16,128,1,float16,fp8,63,0.26484479904174807
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,fp8,1,0.008372800052165985
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,float16,3,0.010436800122261048
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,16,128,1,float16,float16,127,0.33794078826904295
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,fp8,3,0.008452799916267396
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,float16,7,0.010478399693965912
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,16,128,1,float16,fp8,127,0.26595520973205566
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,float16,15,0.010500799864530563
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,fp8,15,0.008510400354862214
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,fp8,7,0.008551999926567078
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,fp8,31,0.00840959995985031
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,float16,63,0.01048159971833229
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,fp8,63,0.008449599891901017
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,fp8,127,0.00883679986000061
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,float16,127,0.010420800000429154
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,float16,255,0.01048479974269867
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,fp8,255,0.008476799726486206
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,fp8,511,0.010449600219726563
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,float16,1023,0.02847839891910553
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,fp8,1023,0.012563200294971466
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,float16,2047,0.04591040015220642
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,fp8,2047,0.028798401355743408
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,fp8,4095,0.04772160053253174
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,float16,4095,0.08286399841308593
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,fp8,8191,0.08760640025138855
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,float16,8191,0.15616159439086913
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,float16,1,0.015209600329399109
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,float16,3,0.01499519944190979
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,16,128,1,float16,float16,7,0.3263200044631958
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,float16,7,0.016392000019550323
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,float16,1,0.010500799864530563
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,float16,15,0.015588800609111785
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,float16,31,0.01621119976043701
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,float16,31,0.010367999970912933
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,float16,63,0.015276800096035003
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,float16,511,0.012467200309038163
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,float16,127,0.016126400232315062
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,fp8,7,0.008364800363779068
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,float16,255,0.02043039947748184
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,fp8,1,0.01408960074186325
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,float16,511,0.03874079883098602
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,fp8,511,0.020608000457286835
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,float16,1023,0.06670079827308655
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,fp8,1023,0.03907040059566498
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,fp8,7,0.01356160044670105
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,float16,2047,0.12224800586700439
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,fp8,2047,0.06827200055122376
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,fp8,15,0.012825599312782288
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,fp8,4095,0.12735359668731688
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,float16,4095,0.2328943967819214
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,fp8,31,0.012561599910259246
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,float16,1,0.0083856001496315
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,float16,3,0.008475200086832047
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,fp8,63,0.012716799974441528
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,float16,7,0.010393600165843963
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,fp8,8191,0.24855680465698243
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,float16,8191,0.46139841079711913
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,fp8,127,0.013544000685214996
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,float16,31,0.010419200360774993
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,float16,15,0.009564799815416336
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,float16,63,0.009454400092363358
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,fp8,3,0.01255200058221817
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,float16,127,0.00870240032672882
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,fp8,1,0.008377599716186523
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,fp8,3,0.008446399867534638
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,float16,255,0.00840959995985031
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,fp8,255,0.008427199721336365
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,float16,511,0.009364800155162811
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,fp8,511,0.008419200032949447
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,float16,1023,0.010344000160694122
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,fp8,7,0.008423999696969987
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,fp8,1023,0.008371199667453765
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,fp8,2047,0.010416000336408614
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,float16,2047,0.010480000078678131
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,fp8,15,0.00846560001373291
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,fp8,31,0.008427199721336365
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,float16,4095,0.012531200051307678
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,fp8,4095,0.012095999717712403
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,fp8,63,0.008393599838018417
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,float16,8191,0.014500799775123595
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,fp8,8191,0.01255359947681427
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,float16,1,0.010428799688816071
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,fp8,1,0.0084927998483181
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,fp8,3,0.008414400368928909
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,float16,7,0.010503999888896942
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,fp8,7,0.00843520015478134
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,float16,15,0.0104032002389431
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,fp8,15,0.008441600203514098
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,float16,31,0.0105103999376297
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,fp8,31,0.008479999750852585
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,float16,63,0.010414399951696397
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,fp8,63,0.008470399677753449
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,float16,127,0.010516799986362457
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,fp8,127,0.00862559974193573
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,float16,255,0.010384000092744827
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,fp8,255,0.008451200276613235
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,float16,511,0.009652800112962722
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,float16,1023,0.010427200049161912
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,fp8,127,0.008388800173997879
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,fp8,2047,0.01045759990811348
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,fp8,1023,0.01019200012087822
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,float16,4095,0.013142399489879608
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,float16,8191,0.01908160001039505
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,fp8,8191,0.014497600495815277
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,float16,1,0.010486400127410889
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,fp8,1,0.010454399883747101
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,float16,3,0.010784000158309937
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,fp8,3,0.010409600287675857
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,float16,7,0.011084800213575363
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,fp8,7,0.010382399708032609
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,fp8,15,0.010388799756765366
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,float16,15,0.01120000034570694
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,float16,31,0.011337599903345107
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,fp8,31,0.010411199927330018
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,float16,63,0.01144160032272339
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,fp8,63,0.010398399829864503
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,float16,127,0.012441600114107132
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,fp8,127,0.010412800312042236
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,fp8,255,0.013507199287414551
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,fp8,255,0.01045600026845932
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,float16,511,0.01919199973344803
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,fp8,511,0.01244639977812767
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,float16,1023,0.03863039910793305
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,fp8,1023,0.018588800728321076
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,float16,2047,0.06521919965744019
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,fp8,2047,0.03896479904651642
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,fp8,4095,0.06781280040740967
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,float16,8191,0.23049440383911132
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,fp8,8191,0.126803195476532
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,float16,1,0.010419200360774993
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,fp8,1,0.008448000252246856
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,float16,3,0.010377600044012069
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,float16,3,0.00936800017952919
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,fp8,3,0.008423999696969987
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,float16,7,0.010425599664449692
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,fp8,7,0.008403199911117553
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,float16,15,0.010401599854230881
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,fp8,15,0.009017600119113922
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,float16,31,0.010444799810647965
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,fp8,511,0.0094991996884346
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,fp8,31,0.009172800183296203
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,float16,63,0.010416000336408614
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,fp8,63,0.008486399799585343
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,float16,127,0.01040000021457672
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,fp8,127,0.009734400361776353
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,fp8,4095,0.011073599755764007
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,fp8,255,0.009390400350093841
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,float16,511,0.010505600273609162
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,fp8,511,0.00859839990735054
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,fp8,1023,0.010463999956846238
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,float16,2047,0.012515200674533844
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,fp8,2047,0.01045600026845932
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,float16,4095,0.020414400100708007
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,float16,8191,0.03915199935436249
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,fp8,8191,0.01867839992046356
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,float16,1,0.02274399995803833
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,fp8,1,0.01940159946680069
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,float16,255,0.012436799705028534
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,float16,3,0.022758400440216063
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,fp8,3,0.019543999433517457
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,float16,7,0.022780799865722658
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,fp8,7,0.019431999325752257
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,float16,15,0.022742399573326112
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,fp8,15,0.019412800669670105
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,float16,63,0.022782400250434875
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,fp8,63,0.019299200177192687
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,float16,127,0.02497600018978119
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,float16,4095,0.12004159688949585
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,fp8,127,0.018692800402641298
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,float16,255,0.038868799805641174
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,fp8,255,0.020734399557113647
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,float16,511,0.06567040085792542
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,fp8,511,0.03853279948234558
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,float16,1023,0.11924959421157837
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,float16,2047,0.22772159576416015
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,fp8,2047,0.1249343991279602
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,float16,2047,0.012511999905109405
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,float16,255,0.01043199971318245
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,float16,1023,0.01241919994354248
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,float16,4095,0.4464111804962158
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,fp8,4095,0.012564800679683685
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,fp8,4095,0.24040000438690184
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,float16,3,0.037975999712944034
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,fp8,1,0.03140799999237061
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,float16,7,0.03817760050296783
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,fp8,7,0.031164801120758055
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,fp8,15,0.031376001238822934
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,float16,31,0.038729599118232726
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,fp8,31,0.030982398986816408
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,float16,15,0.038308799266815186
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,float16,31,0.022750400006771088
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,fp8,31,0.01868959963321686
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,float16,127,0.04130240082740784
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,fp8,127,0.03293919861316681
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,float16,255,0.06564639806747437
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,fp8,63,0.0311055988073349
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,fp8,255,0.03938240110874176
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,float16,511,0.11979039907455444
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,float16,1023,0.22778398990631105
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,fp8,1023,0.06620479822158813
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,fp8,511,0.06649760007858277
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,float16,1,0.06819360256195069
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,float16,3,0.06812959909439087
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,fp8,3,0.055764800310134886
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,float16,7,0.0679647982120514
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,fp8,7,0.05574079751968384
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,float16,15,0.06807680130004883
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,fp8,15,0.05578879714012146
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,float16,31,0.06981760263442993
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,fp8,31,0.05578879714012146
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,float16,63,0.07004799842834472
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,fp8,63,0.056867200136184695
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,float16,127,0.0718832015991211
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,fp8,127,0.05767199993133545
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,float16,255,0.12080960273742676
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,float16,1,0.03793280124664307
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,fp8,255,0.06840320229530335
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,float16,1,0.008366400003433227
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,fp8,511,0.12256959676742554
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,fp8,1,0.008345600217580795
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,float16,511,0.22793920040130616
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,fp8,3,0.008348800241947174
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,fp8,3,0.031041601300239564
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,float16,3,0.008433599770069123
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,fp8,7,0.008348800241947174
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,float16,7,0.008425600081682205
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,fp8,15,0.00833280012011528
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,float16,31,0.008316799998283386
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,fp8,31,0.008211199939250947
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,float16,63,0.008353599905967712
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,fp8,63,0.00692640021443367
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,float16,127,0.008369600027799606
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,fp8,127,0.008376000076532364
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,float16,255,0.008336000144481659
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,fp8,255,0.008416000008583068
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,float16,63,0.04050399959087372
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,fp8,511,0.008342400193214417
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,fp8,1023,0.010419200360774993
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,float16,2047,0.018619200587272643
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,fp8,2047,0.012503999471664428
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,float16,4095,0.03730719983577728
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,fp8,4095,0.018606400489807128
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,float16,8191,0.06538559794425965
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,fp8,8191,0.037011200189590455
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,fp8,1,0.10440640449523926
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,float16,1,0.12695839405059814
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,float16,3,0.1271247982978821
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,fp8,1023,0.12328159809112549
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,float16,7,0.12791039943695068
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,fp8,1,0.05564640164375305
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,fp8,7,0.1047327995300293
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,fp8,15,0.10459840297698975
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,float16,15,0.12921279668807983
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,fp8,31,0.1051103949546814
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,float16,63,0.12933919429779053
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,float16,127,0.13348959684371947
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,fp8,127,0.1067088007926941
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,fp8,255,0.12967679500579835
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,float16,255,0.23095359802246093
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,12,12,128,1,float16,float16,1,0.24614880084991456
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,12,12,128,1,float16,fp8,1,0.20115358829498292
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,12,12,128,1,float16,fp8,3,0.20096800327301026
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,float16,15,0.008427199721336365
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,12,12,128,1,float16,float16,7,0.24661760330200194
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,12,12,128,1,float16,fp8,7,0.20092320442199707
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,float16,511,0.009804800152778625
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,float16,1023,0.012076800316572189
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,12,12,128,1,float16,float16,15,0.24670240879058838
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,12,12,128,1,float16,fp8,15,0.20121281147003173
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,12,12,128,1,float16,fp8,31,0.2011023998260498
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,12,12,128,1,float16,float16,63,0.2472127914428711
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,12,12,128,1,float16,fp8,63,0.20132639408111572
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,float16,1,0.010463999956846238
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,12,12,128,1,float16,float16,31,0.24641599655151367
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,12,12,128,1,float16,float16,127,0.2564271926879883
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,fp8,1,0.008423999696969987
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,fp8,3,0.10391199588775635
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,float16,3,0.010385599732398988
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,12,12,128,1,float16,fp8,127,0.20337600708007814
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,float16,7,0.010467199981212616
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,fp8,7,0.00843999981880188
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,float16,15,0.010366400331258773
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,fp8,3,0.00841120034456253
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,float16,31,0.010500799864530563
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,fp8,31,0.008363199979066848
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,float16,63,0.010460799932479859
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,fp8,15,0.008475200086832047
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,fp8,63,0.008483199775218964
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,float16,127,0.010424000024795533
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,fp8,127,0.008430399745702744
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,float16,255,0.010414399951696397
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,fp8,255,0.008457600325345992
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,float16,511,0.012465599924325943
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,fp8,511,0.010470400005578995
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,fp8,1023,0.012606400251388549
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,float16,1023,0.020236800611019134
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,fp8,2047,0.020556800067424774
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,float16,31,0.12893279790878295
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,float16,4095,0.07192479968070983
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,fp8,4095,0.04266240000724793
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,float16,1,0.012511999905109405
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,float16,8191,0.1326799988746643
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,fp8,8191,0.07480000257492066
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,float16,7,0.012459199875593185
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,fp8,7,0.010817600041627884
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,float16,15,0.012444800138473511
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,fp8,15,0.01056160032749176
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,float16,31,0.012436799705028534
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,fp8,31,0.010550399869680404
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,float16,63,0.012444800138473511
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,fp8,63,0.010915199667215348
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,12,12,128,1,float16,float16,3,0.2465264081954956
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,float16,127,0.01249760016798973
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,float16,255,0.012430399656295776
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,fp8,255,0.010473600029945374
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,fp8,127,0.010416000336408614
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,float16,511,0.027350398898124694
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,fp8,511,0.014572800695896148
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,float16,1023,0.04650239944458008
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,fp8,1023,0.02900159955024719
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,float16,2047,0.08312000036239624
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,float16,4095,0.15841439962387086
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,fp8,4095,0.08645280003547669
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,float16,8191,0.30818560123443606
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,float16,2047,0.04118080139160156
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,fp8,63,0.10508960485458374
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,float16,3,0.012544000148773193
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,float16,1,0.008423999696969987
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,fp8,8191,0.16476479768753052
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,fp8,16383,0.32404000759124757
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,float16,16383,0.6078944206237793
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,float16,3,0.00836160033941269
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,float16,7,0.008763200044631958
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,float16,15,0.00918240025639534
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,float16,31,0.008403199911117553
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,fp8,2047,0.04766559898853302
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,float16,63,0.008454400300979614
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,fp8,1,0.008484800159931184
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,float16,127,0.008416000008583068
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,fp8,127,0.008404800295829773
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,fp8,3,0.00835999995470047
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,fp8,255,0.008511999994516373
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,float16,511,0.00864799991250038
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,fp8,511,0.008449599891901017
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,float16,1023,0.00835840031504631
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,fp8,1023,0.008376000076532364
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,fp8,7,0.008364800363779068
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,float16,2047,0.0104032002389431
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,fp8,15,0.008449599891901017
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,fp8,2047,0.010265599936246872
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,fp8,4095,0.010521599650382995
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,float16,8191,0.012571200728416443
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,fp8,8191,0.012547199428081513
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,float16,16383,0.014609600603580474
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,fp8,16383,0.014374400675296783
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,float16,1,0.008425600081682205
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,fp8,1,0.008478400111198426
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,float16,3,0.008448000252246856
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,fp8,3,0.008479999750852585
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,fp8,7,0.008481600135564805
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,float16,15,0.009361600130796432
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,fp8,15,0.008448000252246856
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,float16,31,0.009444800019264222
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,fp8,31,0.008484800159931184
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,float16,63,0.008472000062465668
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,fp8,63,0.008454400300979614
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,float16,127,0.008419200032949447
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,fp8,127,0.008390399813652038
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,float16,255,0.008968000113964082
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,fp8,255,0.008444800227880477
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,float16,511,0.008455999940633774
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,fp8,511,0.008367999643087386
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,float16,1023,0.010401599854230881
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,fp8,1023,0.008472000062465668
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,float16,2047,0.010435199737548828
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,fp8,2047,0.01043360009789467
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,float16,4095,0.012561599910259246
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,fp8,4095,0.011644800007343293
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,float16,8191,0.014601600170135499
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,float16,255,0.008459199965000153
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,fp8,8191,0.012489599734544754
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,float16,16383,0.030551999807357788
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,fp8,16383,0.015508800745010376
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,float16,1,0.010388799756765366
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,fp8,1,0.010633599758148194
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,fp8,1,0.008436799794435502
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,float16,3,0.010447999835014344
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,fp8,3,0.008436799794435502
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,float16,4095,0.010526400059461594
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,float16,7,0.01045759990811348
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,fp8,7,0.00843840017914772
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,float16,15,0.01040000021457672
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,fp8,15,0.008460800349712371
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,float16,31,0.010417599976062775
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,fp8,31,0.008505599945783615
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,float16,63,0.010396800190210342
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,fp8,63,0.008462399989366532
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,fp8,31,0.008444800227880477
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,float16,255,0.01048479974269867
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,float16,511,0.012574400007724761
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,float16,7,0.008550400286912918
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,fp8,511,0.010489600151777268
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,fp8,63,0.008404800295829773
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,float16,1023,0.027687999606132507
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,fp8,1023,0.013260799646377563
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,float16,2047,0.04589279890060425
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,float16,4095,0.08284159898757934
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,fp8,4095,0.04787839949131012
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,float16,8191,0.15563520193099975
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,fp8,8191,0.08920000195503235
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,fp8,16383,0.1714352011680603
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,float16,16383,0.3040816068649292
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,float16,1,0.008352000266313553
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,float16,3,0.008425600081682205
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,fp8,1,0.008414400368928909
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,fp8,3,0.00833439975976944
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,float16,7,0.008491200208663941
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,fp8,7,0.008419200032949447
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,float16,15,0.008470399677753449
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,fp8,15,0.008342400193214417
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,fp8,31,0.008419200032949447
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,float16,63,0.00836160033941269
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,fp8,63,0.008337599784135818
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,float16,127,0.008448000252246856
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,fp8,127,0.008367999643087386
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,float16,255,0.008396799862384795
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,fp8,255,0.008350399881601333
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,float16,511,0.008329600095748901
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,fp8,511,0.00830880030989647
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,fp8,3,0.01048159971833229
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,float16,1023,0.008441600203514098
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,fp8,1023,0.008427199721336365
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,fp8,2047,0.010478399693965912
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,float16,4095,0.01247360035777092
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,float16,127,0.010406400263309478
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,fp8,4095,0.01045759990811348
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,fp8,127,0.008478400111198426
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,float16,8191,0.028228801488876343
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,fp8,255,0.008374399691820144
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,float16,16383,0.04531520009040833
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,fp8,16383,0.02885119915008545
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,float16,1,0.01661760061979294
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,float16,3,0.01663679927587509
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,fp8,3,0.014580799639225006
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,fp8,1,0.014591999351978302
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,float16,7,0.016628800332546233
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,fp8,7,0.014585599303245544
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,fp8,15,0.014604799449443817
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,float16,31,0.017118400335311888
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,fp8,31,0.014630399644374847
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,float16,63,0.017481599748134614
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,float16,127,0.0172447994351387
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,fp8,127,0.014545600116252898
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,float16,255,0.027379199862480164
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,fp8,255,0.014683200418949128
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,float16,511,0.04566720128059387
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,fp8,511,0.0288783997297287
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,float16,1023,0.0821120023727417
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,fp8,1023,0.047670400142669676
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,float16,2047,0.15445760488510132
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,float16,31,0.008419200032949447
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,fp8,4095,0.16627520322799683
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,float16,2047,0.010412800312042236
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,float16,8191,0.5911664009094239
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,fp8,8191,0.014377599954605103
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,float16,1,0.026966398954391478
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,fp8,1,0.022779199481010436
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,float16,3,0.026907199621200563
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,fp8,3,0.022728000581264497
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,fp8,2047,0.02882080078125
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,float16,7,0.026913601160049438
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,fp8,7,0.02279680073261261
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,float16,15,0.016648000478744505
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,float16,15,0.02797119915485382
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,fp8,15,0.02332320064306259
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,fp8,31,0.02279199957847595
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,float16,63,0.02820639908313751
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,fp8,63,0.02285760045051575
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,fp8,63,0.01449120044708252
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,float16,127,0.030876800417900085
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,float16,255,0.04639840126037598
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,fp8,255,0.029265600442886352
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,float16,511,0.08232319951057435
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,fp8,511,0.0475488007068634
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,float16,1023,0.15495840311050416
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,fp8,1023,0.08515520095825195
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,fp8,2047,0.08639680147171021
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,fp8,2047,0.16097760200500488
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,float16,2047,0.2999664068222046
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,float16,1,0.04747200012207031
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,fp8,1,0.03919520080089569
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,fp8,3,0.0391728013753891
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,float16,7,0.047388800978660585
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,fp8,7,0.03918400108814239
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,float16,15,0.04747360050678253
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,fp8,15,0.039155200123786926
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,float16,31,0.04747360050678253
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,fp8,31,0.03924640119075775
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,float16,63,0.04938080012798309
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,fp8,63,0.03936960101127625
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,float16,127,0.05136160254478454
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,fp8,127,0.04110719859600067
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,float16,255,0.0833296000957489
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,fp8,255,0.04936800003051758
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,float16,511,0.15485759973526
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,fp8,511,0.08430399894714355
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,fp8,8191,0.32082719802856446
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,fp8,1023,0.15961120128631592
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,float16,1023,0.2991024017333984
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,float16,1,0.008376000076532364
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,fp8,1,0.008337599784135818
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,fp8,3,0.008377599716186523
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,float16,7,0.008398400247097015
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,fp8,127,0.02311359941959381
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,float16,3,0.00846560001373291
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,float16,15,0.008422400057315826
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,fp8,15,0.008339200168848038
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,fp8,31,0.00835679993033409
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,float16,63,0.00843840017914772
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,float16,127,0.008336000144481659
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,fp8,63,0.008455999940633774
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,float16,255,0.008444800227880477
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,fp8,127,0.008374399691820144
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,fp8,255,0.00835999995470047
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,float16,511,0.008479999750852585
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,fp8,511,0.008444800227880477
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,float16,1023,0.010380800068378448
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,fp8,1023,0.010168000310659408
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,float16,2047,0.01239520013332367
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,fp8,2047,0.01043040007352829
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,float16,4095,0.027459201216697694
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,fp8,4095,0.014401599764823914
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,float16,8191,0.04548160135746002
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,fp8,8191,0.02874400019645691
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,float16,16383,0.08216639757156372
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,fp8,16383,0.046751999855041505
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,float16,1,0.08844000101089478
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,fp8,1,0.0719327986240387
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,float16,4095,0.3008239984512329
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,fp8,3,0.07185440063476563
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,fp8,7,0.07193920016288757
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,float16,7,0.08724319934844971
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,float16,15,0.08840159773826599
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,fp8,15,0.07191680073738098
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,float16,31,0.08860480189323425
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,fp8,31,0.0718447983264923
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,float16,63,0.0889087975025177
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,fp8,63,0.07290239930152893
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,float16,127,0.0924127995967865
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,fp8,127,0.07430880069732666
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,float16,255,0.15688480138778688
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,float16,31,0.0271232008934021
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,float16,511,0.300383996963501
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,fp8,511,0.1594640016555786
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,float16,1,0.16638239622116088
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,fp8,7,0.008448000252246856
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,float16,31,0.008353599905967712
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,fp8,1,0.13563840389251708
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,float16,3,0.1667855978012085
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,fp8,3,0.13558559417724608
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,float16,7,0.16851680278778075
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,float16,15,0.16825599670410157
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,fp8,15,0.13566880226135253
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,float16,31,0.16827199459075928
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,fp8,31,0.1371392011642456
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,float16,63,0.16857759952545165
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,fp8,63,0.13742719888687133
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,float16,127,0.17463840246200563
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,fp8,127,0.1394703984260559
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,float16,1,0.00838399976491928
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,fp8,1,0.00836160033941269
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,float16,3,0.00843999981880188
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,float16,255,0.30402560234069825
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,fp8,3,0.008355200290679932
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,float16,7,0.0083856001496315
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,float16,15,0.008350399881601333
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,fp8,15,0.00843999981880188
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,fp8,7,0.00838399976491928
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,float16,3,0.04801279902458191
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,float16,31,0.008374399691820144
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,fp8,31,0.008408000320196151
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,fp8,63,0.008398400247097015
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,float16,3,0.08842560052871704
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,float16,127,0.008390399813652038
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,fp8,127,0.008320000022649765
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,float16,255,0.008491200208663941
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,fp8,255,0.008382400125265121
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,float16,511,0.00976639986038208
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,fp8,511,0.008376000076532364
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,float16,1023,0.012350399792194367
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,fp8,1023,0.010425599664449692
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,float16,2047,0.026851201057434083
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,fp8,2047,0.012443199753761292
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,float16,4095,0.045347198843955994
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,fp8,4095,0.026918399333953857
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,float16,8191,0.08162879943847656
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,fp8,8191,0.04681600034236908
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,float16,1,0.010372799634933472
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,float16,16383,0.1539680004119873
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,fp8,16383,0.08330079913139343
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,float16,3,0.009857600182294845
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,float16,7,0.010043200105428696
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,fp8,1,0.00843520015478134
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,float16,15,0.010419200360774993
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,fp8,15,0.008417599648237229
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,fp8,255,0.09241600036621093
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,float16,31,0.010440000146627427
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,float16,63,0.010384000092744827
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,fp8,63,0.008408000320196151
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,float16,127,0.010424000024795533
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,fp8,31,0.008495999872684479
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,fp8,127,0.00843840017914772
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,float16,255,0.010395199805498124
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,fp8,255,0.00835679993033409
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,float16,511,0.012508800625801087
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,fp8,511,0.010496000200510025
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,float16,1023,0.02717759907245636
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,fp8,1023,0.01390880048274994
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,fp8,7,0.1354159951210022
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,float16,2047,0.04559679925441742
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,fp8,2047,0.02868640124797821
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,float16,4095,0.08216800093650818
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,float16,8191,0.15603519678115846
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,fp8,4095,0.047142401337623596
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,fp8,8191,0.08422080278396607
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,fp8,255,0.17463359832763672
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,float16,16383,0.30202720165252683
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,float16,1,0.00835999995470047
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,float16,63,0.008367999643087386
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,float16,3,0.008446399867534638
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,float16,32767,0.5934160232543946
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,fp8,32767,0.3109328031539917
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,float16,7,0.008755200356245042
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,float16,15,0.008444800227880477
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,fp8,3,0.00843840017914772
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,fp8,7,0.008417599648237229
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,float16,31,0.008406399935483932
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,float16,63,0.008416000008583068
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,fp8,1,0.008380799740552902
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,float16,127,0.008436799794435502
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,fp8,127,0.008448000252246856
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,float16,255,0.008459199965000153
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,fp8,255,0.008422400057315826
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,float16,511,0.008452799916267396
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,fp8,511,0.00835679993033409
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,fp8,15,0.008433599770069123
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,float16,1023,0.008895999938249587
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,fp8,1023,0.008422400057315826
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,float16,2047,0.010416000336408614
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,float16,4095,0.010486400127410889
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,fp8,4095,0.010494399815797806
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,float16,8191,0.012862400710582733
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,fp8,8191,0.012555199861526489
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,fp8,2047,0.008472000062465668
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,float16,16383,0.016539199650287627
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,fp8,16383,0.014571200311183929
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,float16,32767,0.018592000007629395
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,float16,1,0.00843999981880188
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,fp8,31,0.008427199721336365
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,fp8,1,0.008399999886751174
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,float16,3,0.00838399976491928
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,float16,7,0.00843840017914772
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,fp8,7,0.008483199775218964
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,float16,15,0.008451200276613235
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,fp8,63,0.008455999940633774
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,fp8,15,0.008457600325345992
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,fp8,31,0.008390399813652038
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,float16,31,0.008448000252246856
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,fp8,63,0.008455999940633774
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,float16,127,0.008376000076532364
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,fp8,127,0.00843840017914772
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,float16,255,0.008364800363779068
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,fp8,255,0.008425600081682205
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,float16,511,0.008441600203514098
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,fp8,511,0.008726400136947633
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,float16,1023,0.008417599648237229
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,fp8,1023,0.008423999696969987
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,fp8,2047,0.009880000352859497
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,float16,4095,0.010487999767065048
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,fp8,3,0.008393599838018417
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,fp8,4095,0.010532800108194351
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,float16,8191,0.012561599910259246
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,fp8,8191,0.012460800260305405
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,float16,16383,0.014582400023937226
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,fp8,16383,0.16028959751129152
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,fp8,16383,0.013380800187587739
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,fp8,32767,0.016521599888801575
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,float16,32767,0.03097440004348755
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,fp8,1,0.008353599905967712
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,float16,3,0.00833439975976944
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,fp8,32767,0.01664319932460785
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,fp8,3,0.008369600027799606
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,float16,7,0.008436799794435502
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,fp8,3,0.008395200222730636
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,fp8,7,0.008324799686670303
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,float16,15,0.008406399935483932
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,fp8,15,0.008427199721336365
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,float16,31,0.008382400125265121
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,float16,63,0.008403199911117553
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,fp8,63,0.008347199857234954
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,float16,127,0.008380799740552902
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,fp8,127,0.008350399881601333
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,float16,255,0.008379200100898742
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,fp8,255,0.008313599973917007
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,float16,511,0.009838400036096573
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,fp8,511,0.00846560001373291
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,float16,1023,0.01136159971356392
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,fp8,1023,0.010387200117111205
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,float16,2047,0.026915198564529418
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,fp8,2047,0.012521600723266602
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,float16,4095,0.04527679979801178
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,fp8,4095,0.028303998708724975
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,float16,8191,0.08205119967460632
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,fp8,8191,0.046428799629211426
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,float16,16383,0.15399839878082275
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,fp8,16383,0.08218079805374146
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,fp8,32767,0.15569280385971068
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,float16,32767,0.29895360469818116
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,float16,1,0.00846719965338707
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,float16,3,0.008451200276613235
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,fp8,3,0.00843840017914772
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,fp8,1,0.00843520015478134
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,float16,7,0.008819200098514557
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,fp8,7,0.008475200086832047
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,fp8,15,0.008417599648237229
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,float16,31,0.008921600133180618
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,fp8,7,0.008473599702119828
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,float16,15,0.008851200342178345
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,fp8,31,0.008460800349712371
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,float16,63,0.008433599770069123
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,float16,127,0.008516799658536911
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,fp8,127,0.008422400057315826
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,float16,1,0.008363199979066848
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,fp8,255,0.008347199857234954
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,fp8,511,0.00846880003809929
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,fp8,63,0.0084927998483181
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,fp8,1023,0.00843999981880188
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,float16,2047,0.010436800122261048
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,fp8,2047,0.01048320010304451
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,float16,4095,0.012452799826860428
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,fp8,4095,0.010900799930095673
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,float16,8191,0.01451520025730133
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,fp8,8191,0.012513600289821625
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,float16,16383,0.029262399673461913
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,float16,63,0.00846880003809929
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,fp8,16383,0.015220800042152404
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,float16,32767,0.04779039919376373
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,float16,1,0.0124208003282547
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,fp8,32767,0.030644801259040833
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,fp8,1,0.01053439974784851
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,float16,3,0.012478400021791458
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,float16,7,0.012459199875593185
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,fp8,7,0.010520000010728836
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,float16,15,0.012464000284671784
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,fp8,15,0.01051200032234192
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,float16,2047,0.01037919968366623
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,fp8,31,0.010502400249242783
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,float16,63,0.012439999729394913
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,fp8,63,0.010625600069761276
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,float16,127,0.012436799705028534
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,float16,255,0.012439999729394913
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,fp8,255,0.010595200210809707
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,float16,511,0.027671998739242552
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,fp8,511,0.014571200311183929
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,float16,1023,0.04623199999332428
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,fp8,1023,0.028932800889015196
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,float16,2047,0.08360000252723694
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,fp8,2047,0.04747839868068695
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,float16,4095,0.15767040252685546
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,fp8,4095,0.0843168020248413
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,float16,255,0.008561599999666214
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,float16,511,0.008849599957466125
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,float16,8191,0.3084480047225952
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,float16,1023,0.008448000252246856
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,float16,1,0.016654400527477263
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,fp8,8191,0.15803999900817872
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,fp8,1,0.014519999921321868
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,float16,3,0.01663679927587509
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,fp8,3,0.014556799829006196
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,fp8,31,0.008451200276613235
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,float16,7,0.016579200327396394
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,float16,15,0.016596800088882445
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,fp8,16383,0.30489120483398435
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,fp8,15,0.014547200500965118
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,float16,16383,0.6011055946350098
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,fp8,31,0.014596800506114959
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,float16,31,0.012540799379348756
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,fp8,63,0.014547200500965118
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,float16,31,0.018216000497341157
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,float16,127,0.01775840073823929
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,fp8,127,0.014556799829006196
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,float16,63,0.0176816001534462
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,float16,255,0.027489599585533143
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,fp8,127,0.01069599986076355
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,fp8,255,0.014539200067520141
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,float16,511,0.046137601137161255
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,fp8,511,0.02874400019645691
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,float16,1023,0.08226720094680787
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,fp8,1023,0.04776000082492828
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,float16,1,0.027923199534416198
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,fp8,2047,0.08479040265083312
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,fp8,1,0.02295359969139099
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,float16,3,0.027886399626731874
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,fp8,3,0.023585599660873414
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,fp8,4095,0.15809919834136962
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,float16,4095,0.3002079963684082
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,float16,7,0.027622398734092713
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,fp8,7,0.023316800594329834
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,float16,31,0.02815999984741211
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,fp8,31,0.022814400494098663
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,fp8,15,0.0236272007226944
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,float16,63,0.02836799919605255
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,fp8,63,0.022720000147819518
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,float16,127,0.030313599109649658
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,fp8,127,0.02279199957847595
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,float16,255,0.04581280052661896
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,fp8,255,0.028998398780822755
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,float16,511,0.0821183979511261
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,fp8,511,0.04716959893703461
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,float16,1023,0.15644960403442382
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,fp8,1023,0.08408960103988647
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,float16,1,0.008404800295829773
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,fp8,1,0.008398400247097015
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,float16,3,0.008452799916267396
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,fp8,2047,0.1571984052658081
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,fp8,3,0.008390399813652038
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,fp8,7,0.008603200316429138
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,float16,15,0.008392000198364257
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,fp8,15,0.00838399976491928
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,float16,31,0.008432000130414962
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,fp8,31,0.008403199911117553
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,float16,63,0.0083856001496315
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,float16,127,0.008342400193214417
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,fp8,63,0.00843840017914772
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,float16,255,0.008401600271463394
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,fp8,255,0.008374399691820144
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,float16,511,0.008414400368928909
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,fp8,511,0.008446399867534638
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,float16,1023,0.008804800361394883
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,fp8,1023,0.00843999981880188
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,float16,2047,0.010451199859380722
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,fp8,2047,0.01043040007352829
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,float16,4095,0.012476799637079239
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,fp8,4095,0.010436800122261048
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,float16,8191,0.027351999282836915
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,float16,15,0.02756800055503845
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,fp8,8191,0.013391999900341034
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,float16,16383,0.045419201254844666
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,fp8,16383,0.028803199529647827
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,float16,32767,0.08208159804344177
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,fp8,32767,0.047336000204086306
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,float16,1,0.048235198855400084
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,fp8,1,0.03975200057029724
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,float16,3,0.04776319861412048
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,fp8,3,0.039531201124191284
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,float16,7,0.04809280037879944
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,fp8,7,0.03930720090866089
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,float16,15,0.047751998901367186
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,float16,2047,0.29990720748901367
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,fp8,15,0.039505600929260254
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,float16,31,0.04859200119972229
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,float16,63,0.049351999163627626
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,fp8,63,0.04009920060634613
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,fp8,31,0.03982079923152924
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,float16,7,0.008430399745702744
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,float16,127,0.05135679841041565
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,fp8,127,0.04132319986820221
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,float16,255,0.08245919942855835
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,fp8,255,0.049267199635505673
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,fp8,511,0.08439520001411438
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,float16,511,0.15547360181808473
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,float16,1,0.08781920075416565
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,float16,1023,0.30064640045166013
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,fp8,1,0.07270079851150513
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,float16,3,0.08784639835357666
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,fp8,7,0.014563199877738953
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,float16,7,0.0875823974609375
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,fp8,3,0.07277759909629822
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,fp8,127,0.008449599891901017
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,float16,15,0.08884800076484681
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,fp8,15,0.07241439819335938
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,fp8,31,0.07324320077896118
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,float16,63,0.08889920115470887
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,fp8,63,0.07409759759902954
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,float16,127,0.09226880073547364
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,float16,2047,0.1557711958885193
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,fp8,127,0.07540959715843201
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,float16,255,0.15685759782791137
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,fp8,255,0.09115359783172608
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,float16,1,0.008462399989366532
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,fp8,1,0.008457600325345992
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,float16,511,0.30009119510650634
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,float16,3,0.008363199979066848
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,float16,7,0.00835999995470047
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,fp8,3,0.008390399813652038
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,fp8,7,0.008366400003433227
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,float16,15,0.008476799726486206
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,fp8,15,0.008348800241947174
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,float16,31,0.00843999981880188
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,fp8,31,0.00843999981880188
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,float16,63,0.008457600325345992
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,fp8,63,0.008347199857234954
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,float16,127,0.008420799672603608
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,fp8,127,0.008878400176763534
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,float16,255,0.00844319984316826
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,fp8,255,0.008350399881601333
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,float16,511,0.00856959968805313
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,fp8,511,0.008430399745702744
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,float16,1023,0.010460799932479859
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,fp8,1023,0.009798400104045868
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,float16,2047,0.012449599802494049
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,fp8,2047,0.01040479987859726
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,float16,4095,0.028295999765396117
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,fp8,4095,0.013798399269580841
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,float16,8191,0.04599039852619171
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,fp8,8191,0.028835201263427736
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,float16,16383,0.08222560286521911
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,fp8,16383,0.0463456004858017
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,float16,32767,0.1545248031616211
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,fp8,1023,0.15747040510177612
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,1,0.0084927998483181
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,3,0.00841120034456253
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,fp8,7,0.0727407991886139
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,fp8,32767,0.08300480246543884
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,fp8,3,0.010507199913263321
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,1,0.00835999995470047
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,float16,31,0.0891152024269104
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,3,0.008427199721336365
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,31,0.00841279998421669
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,63,0.008446399867534638
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,15,0.008339200168848038
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,127,0.00843520015478134
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,31,0.008396799862384795
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,127,0.008446399867534638
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,255,0.008414400368928909
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,511,0.008508799970149994
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,1023,0.012462399899959564
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,1023,0.010952000319957734
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,2047,0.026984000205993654
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,2047,0.01247360035777092
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,4095,0.04520800113677979
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,4095,0.028424000740051268
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,fp8,511,0.15927040576934814
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,8191,0.08041279911994934
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,8191,0.04617440104484558
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,16383,0.08252800107002259
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,32767,0.2966543912887573
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,32767,0.1562191963195801
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,7,0.008321599662303924
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,15,0.00843520015478134
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,1,0.008374399691820144
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,7,0.008321599662303924
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,255,0.008457600325345992
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,511,0.009996800124645234
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,7,0.008414400368928909
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,16383,0.1525871992111206
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,15,0.008374399691820144
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,1,0.008377599716186523
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,3,0.008459199965000153
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,31,0.008366400003433227
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,31,0.008479999750852585
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,63,0.008436799794435502
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,7,0.008423999696969987
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,63,0.008491200208663941
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,127,0.00843520015478134
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,127,0.008452799916267396
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,255,0.008558399975299835
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,255,0.008448000252246856
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,511,0.008988799899816513
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,511,0.008457600325345992
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,1023,0.008476799726486206
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,1023,0.008436799794435502
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,2047,0.009164799749851228
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,4095,0.010516799986362457
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,4095,0.010412800312042236
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,8191,0.012558400630950928
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,8191,0.01162080019712448
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,16383,0.016616000235080718
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,16383,0.016444799304008485
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,32767,0.018651199340820313
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,32767,0.017407999932765962
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,65535,0.020716799795627593
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,65535,0.5821167945861816
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,65535,0.018680000305175783
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,1,0.008444800227880477
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,65535,0.3030384063720703
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,3,0.008427199721336365
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,1,0.008763200044631958
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,15,0.00844319984316826
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,7,0.008463999629020691
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,7,0.008448000252246856
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,15,0.008448000252246856
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,3,0.008403199911117553
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,31,0.008396799862384795
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,63,0.008433599770069123
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,63,0.008408000320196151
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,127,0.008417599648237229
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,255,0.008428800106048583
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,255,0.008393599838018417
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,511,0.00841120034456253
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,511,0.008446399867534638
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,1023,0.008451200276613235
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,1023,0.0083856001496315
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,2047,0.010451199859380722
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,2047,0.010345599800348281
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,4095,0.010441599786281586
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,4095,0.010486400127410889
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,8191,0.014327999949455262
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,8191,0.012529599666595458
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,16383,0.016568000614643096
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,16383,0.014590400457382201
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,32767,0.01865600049495697
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,32767,0.01661120057106018
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,65535,0.034980800747871396
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,65535,0.018638400733470915
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,1,0.008408000320196151
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,2047,0.010369600355625152
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,3,0.008355200290679932
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,3,0.008433599770069123
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,7,0.008436799794435502
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,7,0.008382400125265121
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,63,0.008316799998283386
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,15,0.008502399921417237
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,3,0.008470399677753449
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,15,0.008476799726486206
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,31,0.008457600325345992
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,63,0.008457600325345992
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,31,0.00846719965338707
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,63,0.00843520015478134
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,127,0.008448000252246856
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,127,0.008488000184297562
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,255,0.00846719965338707
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,255,0.008363199979066848
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,511,0.008454400300979614
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,511,0.008355200290679932
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,15,0.008457600325345992
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,1023,0.010299199819564819
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,31,0.008352000266313553
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,2047,0.010507199913263321
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,4095,0.02704319953918457
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,4095,0.012664000689983367
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,8191,0.045403200387954715
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,8191,0.028516799211502075
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,16383,0.08142399787902832
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,16383,0.04706720113754272
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,32767,0.15337280035018921
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,32767,0.0824832022190094
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,1,0.008376000076532364
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,1,0.00835679993033409
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,65535,0.29685440063476565
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,65535,0.15576640367507935
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,7,0.008455999940633774
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,3,0.008441600203514098
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,7,0.008460800349712371
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,15,0.00843999981880188
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,15,0.008436799794435502
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,31,0.008430399745702744
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,31,0.008449599891901017
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,63,0.0084927998483181
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,63,0.008428800106048583
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,127,0.008422400057315826
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,127,0.008454400300979614
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,255,0.00841279998421669
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,255,0.008371199667453765
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,511,0.008350399881601333
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,1023,0.008459199965000153
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,1023,0.008377599716186523
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,2047,0.010427200049161912
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,2047,0.010286399722099304
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,4095,0.010473600029945374
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,4095,0.012028799951076507
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,8191,0.012537600100040435
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,8191,0.012443199753761292
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,16383,0.014686399698257446
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,16383,0.014472000300884247
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,1,0.008430399745702744
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,32767,0.016612799465656282
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,65535,0.04983679950237274
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,65535,0.031769600510597226
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,float16,1,0.00947519987821579
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,fp8,1,0.008499199897050858
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,float16,3,0.009860800206661224
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,fp8,3,0.008364800363779068
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,float16,7,0.010344000160694122
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,fp8,7,0.008455999940633774
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,float16,15,0.010375999659299851
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,fp8,15,0.008683200180530547
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,float16,31,0.010440000146627427
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,2047,0.012542399764060973
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,fp8,31,0.008817599713802337
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,127,0.008449599891901017
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,float16,63,0.010427200049161912
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,fp8,63,0.008556800335645676
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,float16,127,0.010416000336408614
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,fp8,127,0.0084927998483181
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,fp8,255,0.008428800106048583
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,float16,511,0.012478400021791458
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,fp8,511,0.010491199791431427
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,float16,1023,0.02696320116519928
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,fp8,1023,0.013009600341320038
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,float16,2047,0.04602720141410828
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,fp8,2047,0.02879360020160675
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,fp8,4095,0.04766559898853302
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,float16,4095,0.08166720271110535
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,3,0.00835679993033409
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,float16,8191,0.15410399436950684
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,float16,16383,0.29938559532165526
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,fp8,16383,0.16048959493637086
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,32767,0.03087199926376343
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,fp8,32767,0.3104127883911133
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,float16,32767,0.5918735980987548
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,float16,1,0.01255359947681427
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,fp8,1,0.010424000024795533
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,float16,7,0.012462399899959564
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,float16,3,0.012545600533485413
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,fp8,3,0.011515200138092041
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,float16,15,0.012433599680662155
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,fp8,15,0.011009600013494492
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,fp8,31,0.01043040007352829
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,float16,63,0.012465599924325943
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,fp8,63,0.010425599664449692
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,fp8,127,0.010420800000429154
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,float16,255,0.012435200065374375
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,1023,0.01040000021457672
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,fp8,255,0.010785599797964096
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,float16,511,0.027702400088310243
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,float16,1023,0.04548960030078888
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,float16,255,0.010443200170993806
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,fp8,1023,0.028838399052619933
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,float16,2047,0.08322399854660034
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,fp8,2047,0.047152000665664676
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,float16,4095,0.15752639770507812
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,fp8,4095,0.0839407980442047
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,fp8,8191,0.08538079857826233
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,float16,1,0.01719679981470108
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,fp8,8191,0.15714240074157715
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,float16,8191,0.30526080131530764
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,fp8,1,0.0149167999625206
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,float16,3,0.017472000420093538
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,fp8,3,0.014571200311183929
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,fp8,7,0.014975999295711518
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,float16,15,0.016651199758052827
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,fp8,15,0.015412800014019012
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,float16,31,0.016646400094032288
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,fp8,31,0.014550399780273438
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,float16,63,0.016681599617004394
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,fp8,63,0.014590400457382201
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,float16,127,0.016672000288963318
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,fp8,127,0.014548799395561219
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,float16,255,0.026907199621200563
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,fp8,255,0.014502400159835815
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,float16,511,0.04546400010585785
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,fp8,511,0.02871359884738922
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,float16,1023,0.08190720081329346
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,fp8,1023,0.04766719937324524
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,fp8,7,0.01080000028014183
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,float16,31,0.012547199428081513
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,float16,2047,0.1542448043823242
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,fp8,2047,0.08524159789085388
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,float16,127,0.012436799705028534
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,1,0.00891520008444786
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,1,0.008457600325345992
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,3,0.008427199721336365
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,3,0.008390399813652038
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,7,0.00904799997806549
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,float16,4095,0.29734880924224855
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,7,0.00843520015478134
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,fp8,4095,0.15915999412536622
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,15,0.009027200192213059
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,15,0.008455999940633774
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,63,0.008377599716186523
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,127,0.008955200016498566
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,31,0.008454400300979614
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,fp8,511,0.014476799964904785
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,127,0.008504000306129456
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,255,0.009385599941015243
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,255,0.00833119973540306
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,511,0.008417599648237229
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,1023,0.01037440001964569
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,1023,0.008454400300979614
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,2047,0.010473600029945374
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,2047,0.010420800000429154
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,4095,0.012503999471664428
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,4095,0.010983999818563461
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,8191,0.014542399346828461
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,8191,0.012457600235939026
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,16383,0.030118399858474733
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,16383,0.01459999978542328
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,32767,0.04801599979400635
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,32767,0.030851200222969055
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,65535,0.0842464029788971
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,65535,0.04905279874801636
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,float16,1,0.02755360007286072
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,fp8,1,0.024439999461174013
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,float16,3,0.02766079902648926
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,fp8,3,0.024035200476646423
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,float16,7,0.02787039875984192
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,fp8,7,0.024350400269031524
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,float16,15,0.027011200785636902
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,fp8,15,0.024372799694538115
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,float16,31,0.027063998579978942
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,float16,7,0.017105600237846373
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,fp8,31,0.024512000381946564
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,float16,63,0.026841598749160766
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,float16,127,0.030844798684120177
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,fp8,63,0.024718399345874786
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,fp8,127,0.02447360008955002
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,float16,255,0.0463344007730484
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,fp8,255,0.02951360046863556
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,float16,511,0.0821615993976593
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,fp8,511,0.04773440062999725
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,fp8,1023,0.0844752013683319
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,float16,1023,0.15382080078125
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,float16,1,0.04802719950675964
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,fp8,2047,0.15763200521469117
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,float16,3,0.04813440144062042
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,fp8,1,0.041176000237464906
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,fp8,3,0.041222399473190306
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,float16,7,0.04757600128650665
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,31,0.008436799794435502
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,float16,15,0.0473904013633728
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,63,0.008367999643087386
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,float16,31,0.04822239875793457
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,511,0.008995199948549271
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,float16,63,0.04939199984073639
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,fp8,31,0.04130400121212006
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,float16,127,0.05051519870758057
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,fp8,63,0.04116320013999939
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,fp8,127,0.043268799781799316
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,float16,255,0.08219839930534363
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,fp8,255,0.051451200246810914
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,float16,511,0.1540976047515869
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,fp8,511,0.08556640148162842
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,1,0.00846560001373291
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,1,0.008387199789285659
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,fp8,1023,0.15825120210647584
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,float16,1023,0.2970144033432007
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,3,0.0083856001496315
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,7,0.00838399976491928
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,7,0.008347199857234954
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,15,0.008488000184297562
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,15,0.008376000076532364
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,31,0.00846560001373291
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,511,0.00844319984316826
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,31,0.008329600095748901
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,63,0.008433599770069123
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,127,0.00835840031504631
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,63,0.00843999981880188
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,255,0.00843840017914772
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,255,0.00843520015478134
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,511,0.008380799740552902
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,511,0.008416000008583068
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,1023,0.009110400080680847
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,1023,0.008414400368928909
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,4095,0.012542399764060973
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,2047,0.010377600044012069
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,4095,0.010686399787664414
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,8191,0.02744320034980774
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,8191,0.014552000164985656
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,16383,0.04541760087013245
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,float16,2047,0.299017596244812
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,32767,0.08184000253677368
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,32767,0.04696959853172302
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,65535,0.1537951946258545
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,65535,0.08542079925537109
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,1,0.00841279998421669
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,fp8,7,0.04116159975528717
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,3,0.0083856001496315
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,7,0.008452799916267396
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,3,0.008388800173997879
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,15,0.008374399691820144
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,127,0.008454400300979614
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,31,0.008428800106048583
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,16383,0.028934401273727418
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,63,0.008406399935483932
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,fp8,15,0.041198399662971494
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,127,0.00841279998421669
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,2047,0.010475199669599533
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,255,0.008433599770069123
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,1,0.008454400300979614
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,511,0.00843999981880188
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,63,0.008475200086832047
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,511,0.008363199979066848
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,1023,0.010527999699115753
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,2047,0.012503999471664428
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,2047,0.010524799674749374
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,4095,0.02807359993457794
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,3,0.008417599648237229
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,4095,0.014596800506114959
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,8191,0.04534560143947601
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,8191,0.028828799724578857
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,7,0.008395200222730636
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,16383,0.08177599906921387
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,16383,0.04660159945487976
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,32767,0.15379040241241454
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,32767,0.08224160075187684
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,65535,0.2965087890625
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,65535,0.15480639934539794
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,1,0.008376000076532364
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,131071,0.5829887866973877
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,15,0.008355200290679932
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,131071,0.29979839324951174
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,3,0.00838399976491928
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,7,0.00846880003809929
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,15,0.008423999696969987
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,255,0.008419200032949447
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,31,0.008420799672603608
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,1023,0.01037919968366623
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,63,0.008355200290679932
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,1,0.008339200168848038
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,127,0.008353599905967712
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,127,0.008340799808502197
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,255,0.008934400230646133
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,255,0.008371199667453765
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,127,0.008371199667453765
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,511,0.008340799808502197
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,1023,0.008369600027799606
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,511,0.00843840017914772
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,1023,0.008312000334262848
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,2047,0.009358400106430053
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,15,0.00843999981880188
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,2047,0.008441600203514098
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,8191,0.012507200241088867
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,8191,0.012462399899959564
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,16383,0.01664000004529953
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,16383,0.016607999801635742
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,31,0.00841279998421669
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,32767,0.018702399730682374
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,32767,0.016548800468444824
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,65535,0.020520000159740447
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,65535,0.018665599822998046
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,131071,0.022804799675941467
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,131071,0.02064319998025894
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,1,0.008612799644470214
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,1,0.008324799686670303
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,3,0.008555199950933456
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,3,0.008340799808502197
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,7,0.008432000130414962
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,7,0.008339200168848038
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,15,0.008428800106048583
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,15,0.008414400368928909
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,31,0.00841120034456253
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,63,0.008417599648237229
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,63,0.008414400368928909
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,127,0.008504000306129456
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,127,0.00843520015478134
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,31,0.008510400354862214
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,255,0.00843840017914772
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,255,0.008515200018882752
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,511,0.008455999940633774
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,511,0.008455999940633774
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,1023,0.008856000006198883
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,2047,0.010438399761915207
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,2047,0.008607999980449676
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,4095,0.01051200032234192
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,4095,0.010518400371074677
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,63,0.008323200047016144
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,8191,0.01252480000257492
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,8191,0.0124208003282547
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,16383,0.016540800034999848
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,32767,0.018545599281787874
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,3,0.008371199667453765
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,32767,0.01661120057106018
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,65535,0.02067680060863495
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,65535,0.018475200235843658
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,7,0.008427199721336365
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,131071,0.03875040113925934
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,131071,0.020720000565052032
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,1,0.008401600271463394
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,1,0.008355200290679932
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,3,0.00835840031504631
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,4095,0.010382399708032609
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,7,0.008403199911117553
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,3,0.008367999643087386
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,7,0.008372800052165985
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,15,0.008399999886751174
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,4095,0.00843840017914772
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,15,0.008398400247097015
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,31,0.008427199721336365
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,31,0.008363199979066848
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,63,0.008436799794435502
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,127,0.008449599891901017
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,63,0.008376000076532364
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,127,0.008352000266313553
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,255,0.008367999643087386
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,255,0.008382400125265121
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,511,0.008379200100898742
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,1023,0.008964800089597703
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,1023,0.008374399691820144
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,2047,0.010424000024795533
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,4095,0.012441600114107132
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,4095,0.01067039966583252
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,8191,0.027806401252746582
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,8191,0.014494399726390838
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,16383,0.046035200357437134
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,16383,0.028886398673057555
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,32767,0.08195840120315552
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,32767,0.04731679856777191
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,65535,0.08487200140953063
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,65535,0.15370399951934816
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,31,0.008403199911117553
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,1,0.008820799738168716
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,131071,0.16159679889678955
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,131071,0.29751360416412354
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,3,0.008388800173997879
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,3,0.008446399867534638
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,7,0.008376000076532364
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,7,0.00856959968805313
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,15,0.008449599891901017
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,15,0.008414400368928909
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,31,0.00836160033941269
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,31,0.008414400368928909
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,63,0.008416000008583068
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,63,0.008347199857234954
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,127,0.00846560001373291
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,127,0.00844319984316826
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,255,0.008441600203514098
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,511,0.008363199979066848
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,511,0.008366400003433227
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,16383,0.016616000235080718
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,1023,0.008499199897050858
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,1023,0.00843840017914772
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,2047,0.009408000111579894
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,2047,0.010334400087594986
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,4095,0.010409600287675857
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,4095,0.010480000078678131
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,8191,0.012544000148773193
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,8191,0.012537600100040435
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,16383,0.0165583997964859
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,16383,0.014553600549697876
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,32767,0.018009600043296815
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,32767,0.01666239947080612
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,65535,0.018705600500106813
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,131071,0.05231519937515259
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,131071,0.035231998562812804
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,511,0.008406399935483932
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,1,0.00838399976491928
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,2047,0.010417599976062775
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,3,0.008560000360012055
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,7,0.008414400368928909
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,7,0.00838399976491928
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,15,0.008444800227880477
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,15,0.008436799794435502
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,31,0.008422400057315826
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,31,0.008432000130414962
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,63,0.008416000008583068
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,63,0.008436799794435502
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,127,0.008475200086832047
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,127,0.008511999994516373
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,255,0.008393599838018417
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,255,0.008425600081682205
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,1,0.008425600081682205
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,511,0.008422400057315826
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,1023,0.01048479974269867
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,2047,0.0278656005859375
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,2047,0.012491200119256973
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,1023,0.008452799916267396
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,4095,0.04526880085468292
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,4095,0.02865760028362274
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,8191,0.0809664011001587
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,8191,0.04595839977264404
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,16383,0.1536064028739929
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,16383,0.08248639702796937
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,32767,0.15631840229034424
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,32767,0.29661600589752196
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,65535,0.03476960062980652
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,float16,1,0.010380800068378448
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,fp8,1,0.008481600135564805
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,float16,3,0.010463999956846238
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,fp8,3,0.008988799899816513
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,float16,7,0.010470400005578995
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,fp8,7,0.008414400368928909
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,3,0.00838399976491928
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,float16,15,0.010440000146627427
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,fp8,15,0.008884800225496292
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,float16,31,0.010375999659299851
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,65535,0.5832208156585693
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,fp8,31,0.008449599891901017
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,float16,127,0.010449600219726563
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,fp8,63,0.00846719965338707
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,fp8,127,0.008448000252246856
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,float16,255,0.010385599732398988
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,fp8,255,0.008419200032949447
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,float16,511,0.01242239996790886
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,511,0.00974079966545105
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,1023,0.010918399691581726
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,float16,1023,0.026875200867652892
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,fp8,1023,0.012545600533485413
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,float16,2047,0.045326399803161624
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,fp8,2047,0.02799679934978485
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,float16,4095,0.08281919956207276
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,255,0.00843999981880188
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,float16,8191,0.15480159521102904
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,fp8,8191,0.08364160060882568
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,float16,16383,0.29921278953552244
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,float16,1,0.012489599734544754
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,fp8,1,0.012292800098657608
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,float16,3,0.012513600289821625
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,fp8,3,0.01250240057706833
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,1,0.008340799808502197
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,float16,7,0.01250080019235611
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,fp8,7,0.01244639977812767
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,float16,15,0.012476799637079239
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,65535,0.29956960678100586
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,fp8,15,0.012443199753761292
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,float16,31,0.012558400630950928
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,float16,63,0.012535999715328216
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,fp8,63,0.010505600273609162
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,fp8,31,0.012579199671745301
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,float16,127,0.012484800070524216
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,fp8,127,0.010916800051927567
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,float16,255,0.012537600100040435
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,fp8,255,0.011524800211191177
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,float16,511,0.02754560112953186
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,fp8,511,0.014588800072669984
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,float16,1023,0.046609601378440856
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,fp8,1023,0.028859201073646545
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,float16,2047,0.08289920091629029
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,fp8,2047,0.04738720059394837
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,float16,4095,0.15706080198287964
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,fp8,4095,0.08394399881362916
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,fp8,511,0.010462400317192078
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,1,0.008432000130414962
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,3,0.008430399745702744
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,float16,8191,0.30375521183013915
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,3,0.008448000252246856
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,fp8,8191,0.1571552038192749
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,7,0.008483199775218964
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,7,0.008369600027799606
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,15,0.008436799794435502
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,15,0.00883840024471283
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,31,0.008416000008583068
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,fp8,4095,0.04620800018310547
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,31,0.008449599891901017
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,63,0.008635199815034866
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,63,0.008428800106048583
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,127,0.008505599945783615
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,255,0.008708799630403519
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,255,0.008347199857234954
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,511,0.008463999629020691
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,127,0.008849599957466125
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,1023,0.00843999981880188
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,2047,0.010390400141477584
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,2047,0.010471999645233154
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,4095,0.012305600196123123
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,4095,0.01050880029797554
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,8191,0.013935999572277069
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,8191,0.012531200051307678
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,16383,0.016523200273513793
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,32767,0.03156639933586121
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,16383,0.014047999680042268
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,32767,0.016545599699020384
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,fp8,16383,0.15838719606399537
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,65535,0.05039520263671875
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,65535,0.03250400125980377
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,float16,1,0.016652800142765045
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,fp8,1,0.01635199934244156
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,131071,0.08707200288772583
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,131071,0.05114560127258301
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,float16,3,0.01664000004529953
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,fp8,3,0.015880000591278077
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,float16,15,0.016651199758052827
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,float16,7,0.017399999499320983
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,fp8,7,0.016545599699020384
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,fp8,15,0.016335999965667723
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,float16,31,0.016672000288963318
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,fp8,31,0.01650079935789108
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,float16,63,0.018246400356292724
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,float16,127,0.01748960018157959
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,fp8,63,0.0165583997964859
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,fp8,127,0.0165120005607605
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,float16,255,0.02736159861087799
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,fp8,255,0.015291200578212738
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,float16,511,0.04561119973659515
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,float16,1023,0.08216320276260376
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,fp8,1023,0.04783360064029694
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,float16,63,0.01043040007352829
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,float16,2047,0.1552351951599121
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,fp8,2047,0.08470079898834229
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,float16,1,0.028534400463104247
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,float16,4095,0.2986272096633911
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,fp8,4095,0.1591760039329529
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,float16,3,0.028515198826789857
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,fp8,3,0.024873599410057068
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,float16,7,0.028334400057792662
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,fp8,7,0.024907200038433074
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,fp8,15,0.024831999838352204
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,1,0.008436799794435502
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,fp8,31,0.02484800070524216
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,float16,63,0.02871679961681366
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,fp8,63,0.02486560046672821
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,float16,127,0.030907198786735535
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,fp8,127,0.024779200553894043
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,1023,0.009377600252628326
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,float16,255,0.04636479914188385
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,fp8,255,0.030843201279640197
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,float16,511,0.08218399882316589
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,fp8,511,0.048905599117279056
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,float16,1023,0.1542464017868042
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,fp8,1023,0.0854319989681244
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,fp8,2047,0.15777920484542846
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,float16,2047,0.29795839786529543
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,1,0.00846560001373291
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,fp8,511,0.028774398565292358
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,3,0.008428800106048583
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,1,0.008387199789285659
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,7,0.008403199911117553
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,7,0.008393599838018417
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,15,0.008374399691820144
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,15,0.008377599716186523
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,31,0.008463999629020691
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,63,0.00939520001411438
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,63,0.008432000130414962
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,fp8,1,0.024748800694942473
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,127,0.008479999750852585
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,127,0.008382400125265121
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,255,0.008449599891901017
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,511,0.008475200086832047
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,255,0.00843840017914772
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,511,0.008457600325345992
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,1023,0.008671999722719193
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,float16,15,0.02858079969882965
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,2047,0.010420800000429154
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,2047,0.010398399829864503
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,4095,0.01247519999742508
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,float16,31,0.027953600883483885
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,4095,0.010447999835014344
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,8191,0.014523200690746307
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,8191,0.012566399574279786
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,16383,0.0149167999625206
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,32767,0.04874559938907623
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,32767,0.030827200412750243
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,65535,0.08441759943962097
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,65535,0.049476799368858335
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,511,0.008432000130414962
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,131071,0.15582400560379028
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,131071,0.08670880198478699
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,float16,1,0.021427200734615327
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,float16,3,0.02271520048379898
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,fp8,1,0.02067199945449829
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,float16,7,0.020732800662517547
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,fp8,7,0.020732800662517547
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,float16,15,0.020744000375270844
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,float16,31,0.022550399601459502
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,fp8,31,0.020745599269866945
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,fp8,15,0.02080959975719452
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,float16,63,0.02258239984512329
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,fp8,63,0.020667199790477753
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,float16,127,0.02220800071954727
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,fp8,127,0.020751999318599702
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,fp8,255,0.02066880017518997
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,float16,511,0.01857919991016388
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,fp8,511,0.016833600401878358
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,float16,1023,0.02075359970331192
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,fp8,1023,0.01871519982814789
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,float16,2047,0.02483839988708496
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,fp8,2047,0.022750400006771088
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,float16,1,0.010361599922180175
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,fp8,1,0.010390400141477584
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,float16,3,0.010398399829864503
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,fp8,3,0.010371199995279311
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,float16,7,0.010395199805498124
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,fp8,7,0.010478399693965912
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,fp8,15,0.010384000092744827
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,float16,15,0.010468800365924836
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,float16,31,0.010465600341558457
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,fp8,31,0.01043040007352829
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,float16,63,0.010428799688816071
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,fp8,63,0.010369600355625152
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,float16,127,0.010471999645233154
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,fp8,127,0.010407999902963639
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,float16,255,0.010476800054311753
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,fp8,255,0.00944959968328476
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,float16,511,0.012540799379348756
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,fp8,511,0.010390400141477584
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,float16,1023,0.014507199823856353
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,fp8,1023,0.012539200484752655
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,1023,0.008392000198364257
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,fp8,2047,0.018160000443458557
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,float16,2047,0.02948319911956787
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,fp8,1,0.012515200674533844
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,16383,0.0298224002122879
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,float16,3,0.012515200674533844
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,float16,7,0.0124719999730587
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,fp8,3,0.012441600114107132
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,fp8,7,0.012505599856376648
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,float16,15,0.012531200051307678
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,fp8,15,0.012481600046157837
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,float16,31,0.012567999958992004
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,fp8,31,0.012486399710178375
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,fp8,63,0.012531200051307678
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,float16,127,0.012478400021791458
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,fp8,127,0.012492799758911132
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,fp8,3,0.020793600380420683
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,float16,63,0.012534399330615998
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,float16,255,0.012462399899959564
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,fp8,255,0.012439999729394913
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,fp8,511,0.014454400539398194
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,float16,1023,0.02903519868850708
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,fp8,1023,0.018606400489807128
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,float16,255,0.020761600136756896
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,float16,2047,0.0489872008562088
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,fp8,2047,0.032969599962234496
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,float16,1,0.014591999351978302
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,fp8,1,0.012566399574279786
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,float16,3,0.014539200067520141
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,fp8,3,0.012489599734544754
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,float16,7,0.014575999975204468
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,float16,15,0.01451839953660965
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,float16,31,0.014558400213718414
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,fp8,15,0.012534399330615998
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,fp8,31,0.01247360035777092
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,float16,63,0.014521600306034088
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,fp8,63,0.0125231996178627
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,float16,127,0.014556799829006196
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,fp8,127,0.012438400089740754
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,float16,255,0.014606399834156037
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,fp8,255,0.012430399656295776
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,31,0.008441600203514098
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,float16,511,0.028944000601768494
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,fp8,511,0.016542400419712066
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,fp8,1023,0.030731201171875
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,float16,1023,0.04788959920406342
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,float16,2047,0.08568159937858581
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,fp8,2047,0.049534401297569274
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,fp8,1,0.01961279958486557
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,float16,1,0.020660799741744996
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,fp8,3,0.019547200202941893
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,float16,7,0.02062239944934845
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,fp8,7,0.019617600739002226
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,fp8,15,0.019431999325752257
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,float16,31,0.02075680047273636
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,fp8,31,0.02064639925956726
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,float16,1,0.012460800260305405
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,fp8,63,0.02064639925956726
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,fp8,127,0.020641599595546723
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,float16,127,0.018673600256443025
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,float16,255,0.02067359983921051
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,fp8,255,0.01865759938955307
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,float16,511,0.016550399363040924
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,float16,1023,0.014480000734329224
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,fp8,1023,0.013715200126171112
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,float16,2047,0.014521600306034088
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,fp8,2047,0.01449120044708252
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,float16,1,0.014523200690746307
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,float16,511,0.01661760061979294
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,fp8,1,0.013191999495029449
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,float16,3,0.014563199877738953
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,fp8,3,0.01451520025730133
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,float16,7,0.014627200365066529
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,fp8,7,0.014494399726390838
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,float16,15,0.014574399590492249
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,fp8,15,0.014103999733924866
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,float16,31,0.014572800695896148
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,fp8,31,0.014481599628925323
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,float16,63,0.014579200744628906
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,fp8,63,0.013777600228786468
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,float16,127,0.014603200554847717
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,fp8,7,0.012532800436019897
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,fp8,127,0.01255200058221817
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,float16,255,0.01454399973154068
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,float16,511,0.012457600235939026
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,fp8,511,0.012508800625801087
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,float16,1023,0.011646399646997452
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,fp8,1023,0.01162400022149086
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,float16,2047,0.011102399975061416
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,3,0.008348800241947174
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,fp8,2047,0.01079839989542961
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,float16,1,0.01085439994931221
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,float16,3,0.012404800206422806
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,fp8,1,0.010518400371074677
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,fp8,3,0.010499200224876404
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,float16,7,0.011404799669981003
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,fp8,7,0.01048479974269867
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,float16,15,0.010471999645233154
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,fp8,15,0.010470400005578995
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,float16,31,0.011100800335407257
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,fp8,31,0.010518400371074677
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,float16,63,0.010692799836397171
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,fp8,63,0.010683199763298035
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,fp8,127,0.010476800054311753
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,float16,255,0.010463999956846238
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,float16,3,0.020619200170040132
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,fp8,255,0.010463999956846238
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,float16,511,0.010427200049161912
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,fp8,511,0.010503999888896942
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,float16,1023,0.010422399640083313
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,fp8,1023,0.010475199669599533
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,float16,2047,0.01053759977221489
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,fp8,2047,0.010499200224876404
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,float16,63,0.020720000565052032
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,fp8,1,0.009369599819183349
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,float16,3,0.010351999849081039
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,float16,7,0.010382399708032609
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,fp8,7,0.010364799946546554
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,float16,15,0.01037919968366623
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,fp8,511,0.016540800034999848
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,fp8,15,0.010420800000429154
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,float16,31,0.010385599732398988
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,fp8,31,0.010097599774599075
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,float16,63,0.010463999956846238
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,fp8,63,0.010395199805498124
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,float16,127,0.010417599976062775
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,fp8,127,0.008432000130414962
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,float16,255,0.010384000092744827
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,fp8,255,0.009363199770450591
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,fp8,511,0.010380800068378448
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,float16,1023,0.010396800190210342
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,fp8,1023,0.01037440001964569
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,float16,2047,0.010486400127410889
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,fp8,2047,0.01051200032234192
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,float16,1,0.020628799498081208
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,fp8,1,0.02062080055475235
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,float16,3,0.020716799795627593
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,fp8,3,0.020695999264717102
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,float16,7,0.020712000131607056
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,fp8,7,0.020635199546813966
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,float16,15,0.020689600706100465
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,fp8,15,0.020623999834060668
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,float16,31,0.020720000565052032
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,fp8,31,0.020587199926376344
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,float16,127,0.010470400005578995
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,float16,63,0.02061759978532791
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,fp8,63,0.020678399503231047
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,fp8,127,0.020660799741744996
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,float16,127,0.020636799931526183
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,float16,15,0.020683200657367708
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,fp8,255,0.018705600500106813
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,fp8,511,0.016527999937534333
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,float16,1023,0.014560000598430633
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,float16,1,0.01040000021457672
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,fp8,1023,0.014502400159835815
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,float16,2047,0.014505599439144135
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,fp8,2047,0.014499199390411378
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,float16,1,0.014526399970054626
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,fp8,1,0.014481599628925323
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,float16,3,0.014435200393199921
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,fp8,3,0.014511999487876893
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,float16,7,0.014579200744628906
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,fp8,7,0.014455999433994293
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,fp8,15,0.014446400105953217
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,float16,15,0.014556799829006196
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,float16,31,0.014481599628925323
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,fp8,31,0.014537599682807923
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,float16,63,0.014504000544548035
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,fp8,63,0.014511999487876893
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,float16,127,0.014508800208568573
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,float16,511,0.010384000092744827
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,fp8,127,0.014507199823856353
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,float16,255,0.01451359987258911
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,float16,511,0.012515200674533844
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,fp8,255,0.014534400403499603
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,float16,1023,0.012452799826860428
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,fp8,1023,0.012127999961376191
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,float16,2047,0.012438400089740754
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,fp8,2047,0.010543999820947647
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,float16,1,0.010441599786281586
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,fp8,1,0.010454399883747101
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,float16,3,0.010527999699115753
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,fp8,3,0.010475199669599533
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,float16,7,0.010518400371074677
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,float16,15,0.010470400005578995
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,fp8,7,0.010463999956846238
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,fp8,15,0.010412800312042236
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,float16,31,0.010492800176143647
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,fp8,31,0.01053600013256073
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,float16,63,0.010599999874830245
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,fp8,63,0.01051200032234192
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,float16,127,0.010526400059461594
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,fp8,127,0.010513599961996078
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,float16,255,0.010518400371074677
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,float16,511,0.010420800000429154
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,fp8,255,0.014558400213718414
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,float16,1023,0.010476800054311753
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,fp8,1023,0.010476800054311753
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,float16,2047,0.011740799993276596
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,fp8,2047,0.010967999696731567
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,float16,1,0.010424000024795533
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,fp8,1,0.01043040007352829
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,float16,3,0.010419200360774993
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,fp8,3,0.009097599983215332
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,float16,7,0.010353600233793258
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,fp8,7,0.008984000235795975
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,float16,15,0.010363200306892395
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,float16,255,0.020080000162124634
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,fp8,15,0.009387200325727462
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,float16,31,0.010425599664449692
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,fp8,31,0.008966399729251862
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,float16,511,0.01653439998626709
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,float16,63,0.010369600355625152
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,fp8,63,0.008854400366544724
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,float16,127,0.010384000092744827
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,fp8,127,0.0104032002389431
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,float16,255,0.010393600165843963
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,fp8,3,0.009372799843549728
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,float16,511,0.01040000021457672
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,fp8,511,0.010420800000429154
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,float16,1023,0.010393600165843963
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,float16,2047,0.012439999729394913
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,fp8,2047,0.010496000200510025
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,float16,1,0.020707200467586517
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,fp8,1,0.020734399557113647
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,float16,3,0.02072480022907257
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,fp8,3,0.020729599893093108
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,float16,7,0.020761600136756896
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,float16,15,0.020720000565052032
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,fp8,7,0.020665599405765532
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,fp8,15,0.020703999698162077
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,fp8,31,0.02072640061378479
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,fp8,511,0.012520000338554382
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,float16,63,0.02073120027780533
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,float16,127,0.02072480022907257
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,fp8,63,0.020735999941825865
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,fp8,127,0.02070080041885376
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,float16,255,0.020667199790477753
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,fp8,255,0.020633600652217865
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,float16,511,0.018486399948596955
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,fp8,511,0.016620799899101257
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,float16,1023,0.01611520051956177
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,float16,2047,0.018486399948596955
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,fp8,2047,0.016572800278663636
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,fp8,255,0.01051200032234192
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,float16,1,0.014582400023937226
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,fp8,511,0.010497599840164185
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,fp8,1,0.01459999978542328
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,fp8,3,0.014582400023937226
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,float16,7,0.014577600359916686
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,fp8,7,0.014611199498176575
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,float16,15,0.014609600603580474
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,fp8,15,0.014564800262451171
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,float16,31,0.014871999621391296
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,fp8,31,0.01462559998035431
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,float16,63,0.01488959938287735
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,float16,127,0.016604800522327424
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,fp8,127,0.014614400267601014
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,float16,255,0.015345600247383118
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,fp8,255,0.014588800072669984
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,float16,511,0.014524799585342408
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,float16,1023,0.01629280000925064
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,fp8,511,0.012601600587368011
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,fp8,1023,0.014535999298095703
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,float16,2047,0.0186831995844841
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,fp8,2047,0.016726399958133697
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,fp8,1,0.01045759990811348
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,float16,3,0.010412800312042236
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,fp8,1023,0.010480000078678131
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,fp8,3,0.010451199859380722
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,float16,7,0.010385599732398988
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,fp8,7,0.010478399693965912
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,fp8,15,0.010451199859380722
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,float16,31,0.010428799688816071
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,fp8,31,0.01043199971318245
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,float16,31,0.020640000700950623
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,float16,63,0.010449600219726563
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,fp8,63,0.009062399715185165
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,float16,127,0.010465600341558457
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,fp8,127,0.009062399715185165
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,float16,255,0.01043360009789467
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,fp8,255,0.009390400350093841
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,float16,511,0.010473600029945374
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,fp8,511,0.010375999659299851
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,float16,1023,0.014979200065135955
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,fp8,1023,0.012460800260305405
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,float16,2047,0.02890239953994751
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,fp8,2047,0.01656160056591034
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,fp8,1023,0.014572800695896148
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,float16,1,0.010436800122261048
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,fp8,1,0.010396800190210342
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,fp8,3,0.010419200360774993
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,float16,7,0.010467199981212616
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,fp8,7,0.010428799688816071
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,float16,15,0.010441599786281586
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,fp8,15,0.010396800190210342
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,float16,31,0.010499200224876404
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,float16,3,0.014764800667762756
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,fp8,31,0.01045600026845932
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,float16,63,0.010407999902963639
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,fp8,63,0.010395199805498124
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,float16,127,0.010847999900579452
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,fp8,127,0.010369600355625152
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,float16,255,0.010710400342941285
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,fp8,255,0.010387200117111205
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,float16,511,0.014497600495815277
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,fp8,511,0.012436799705028534
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,fp8,63,0.014584000408649444
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,float16,1023,0.028788799047470094
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,fp8,1023,0.014788800477981567
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,float16,2047,0.046876800060272214
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,float16,1,0.020623999834060668
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,fp8,1,0.020659199357032774
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,float16,3,0.020684799551963805
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,fp8,3,0.020636799931526183
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,float16,7,0.02067359983921051
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,fp8,7,0.020670400559902193
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,float16,15,0.020608000457286835
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,fp8,15,0.020659199357032774
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,float16,1,0.010159999877214432
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,float16,31,0.020657600462436677
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,fp8,31,0.020729599893093108
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,fp8,255,0.010427200049161912
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,float16,63,0.020766399800777435
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,float16,127,0.020660799741744996
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,float16,255,0.020684799551963805
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,fp8,255,0.018718400597572328
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,float16,511,0.01669919937849045
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,fp8,511,0.0165583997964859
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,float16,15,0.010369600355625152
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,float16,1023,0.014584000408649444
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,fp8,1023,0.014532800018787383
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,float16,2047,0.01449120044708252
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,fp8,2047,0.01459999978542328
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,float16,1,0.01462559998035431
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,fp8,1,0.014476799964904785
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,float16,3,0.014548799395561219
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,fp8,3,0.01454399973154068
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,fp8,7,0.014655999839305878
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,float16,15,0.014496000111103058
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,float16,31,0.0146479994058609
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,fp8,31,0.014524799585342408
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,float16,63,0.014579200744628906
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,fp8,63,0.014528000354766845
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,float16,127,0.014582400023937226
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,fp8,127,0.014580799639225006
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,float16,255,0.014563199877738953
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,fp8,255,0.01451359987258911
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,float16,511,0.01247360035777092
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,fp8,511,0.01252640038728714
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,float16,1023,0.012441600114107132
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,fp8,1023,0.01085119992494583
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,float16,2047,0.012516799569129943
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,fp8,2047,0.011532799899578094
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,float16,1,0.011019200086593628
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,fp8,1,0.010523200035095215
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,float16,3,0.010548800230026245
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,fp8,3,0.010665600001811982
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,float16,7,0.01072480008006096
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,fp8,7,0.010731200128793717
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,float16,15,0.010667199641466141
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,fp8,15,0.010507199913263321
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,fp8,2047,0.03049120008945465
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,float16,31,0.010582400113344192
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,fp8,31,0.010518400371074677
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,float16,63,0.011612799763679505
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,float16,127,0.011107199639081956
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,float16,255,0.010521599650382995
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,fp8,255,0.010518400371074677
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,float16,511,0.010711999982595444
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,fp8,511,0.010860799998044967
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,fp8,63,0.02067359983921051
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,float16,1023,0.01048159971833229
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,fp8,127,0.020660799741744996
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,fp8,1023,0.010398399829864503
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,float16,1,0.008454400300979614
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,fp8,2047,0.012513600289821625
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,fp8,1,0.008457600325345992
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,float16,3,0.009494400024414063
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,float16,7,0.008390399813652038
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,fp8,3,0.00836160033941269
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,float16,2047,0.012534399330615998
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,fp8,7,0.008433599770069123
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,float16,15,0.00843840017914772
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,float16,7,0.014584000408649444
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,float16,31,0.008745600283145905
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,fp8,31,0.008408000320196151
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,float16,63,0.008691199868917466
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,fp8,63,0.008423999696969987
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,fp8,15,0.014584000408649444
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,fp8,127,0.008366400003433227
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,float16,255,0.00843999981880188
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,fp8,511,0.008457600325345992
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,fp8,255,0.008459199965000153
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,fp8,15,0.00917920023202896
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,float16,1023,0.010391999781131745
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,fp8,1023,0.008528000116348267
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,float16,2047,0.01072160005569458
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,fp8,2047,0.010401599854230881
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,float16,1,0.012559999525547028
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,fp8,1,0.010681600123643876
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,float16,511,0.008591999858617782
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,float16,3,0.012430399656295776
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,float16,7,0.012438400089740754
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,fp8,7,0.011929599940776825
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,float16,15,0.012455999851226807
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,fp8,15,0.011271999776363372
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,float16,31,0.012468799948692322
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,fp8,31,0.011296000331640244
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,float16,63,0.012462399899959564
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,float16,3,0.010681600123643876
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,fp8,127,0.011745599657297134
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,float16,255,0.012441600114107132
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,fp8,255,0.010454399883747101
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,float16,511,0.014192000031471252
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,fp8,511,0.012449599802494049
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,float16,1023,0.01847040057182312
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,fp8,1023,0.016491200029850005
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,float16,1,0.012427199631929398
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,fp8,1,0.011289600282907486
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,fp8,63,0.010878399759531022
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,float16,3,0.012433599680662155
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,fp8,127,0.011020799726247787
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,fp8,3,0.011667200177907944
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,float16,15,0.01250240057706833
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,fp8,15,0.012091200053691863
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,float16,31,0.012489599734544754
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,fp8,31,0.01252799928188324
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,float16,63,0.012697599828243256
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,fp8,63,0.012503999471664428
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,float16,127,0.012809599936008453
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,fp8,127,0.012278400361537933
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,float16,255,0.012508800625801087
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,float16,127,0.008472000062465668
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,fp8,255,0.011596799641847611
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,float16,511,0.016571199893951415
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,fp8,511,0.014529600739479065
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,fp8,1023,0.0187376007437706
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,float16,1023,0.03091520071029663
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,float16,1,0.01865759938955307
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,fp8,3,0.01045759990811348
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,fp8,1,0.016542400419712066
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,float16,3,0.01863519996404648
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,float16,7,0.018606400489807128
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,fp8,7,0.0166143998503685
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,float16,15,0.01863359957933426
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,fp8,15,0.016572800278663636
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,fp8,63,0.010491199791431427
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,fp8,31,0.016627199947834015
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,float16,63,0.018571199476718904
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,fp8,63,0.016620799899101257
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,float16,127,0.018648000061511995
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,fp8,127,0.016607999801635742
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,float16,255,0.01858240067958832
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,fp8,255,0.016590400040149687
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,float16,511,0.03166239857673645
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,float16,1023,0.0513264000415802
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,fp8,511,0.02119999974966049
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,fp8,1023,0.032927998900413515
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,fp8,1,0.016952000558376312
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,float16,3,0.022703999280929567
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,fp8,3,0.016686399281024934
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,float16,7,0.02271360009908676
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,float16,7,0.012470400333404541
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,fp8,7,0.01675360053777695
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,fp8,7,0.01183520033955574
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,float16,15,0.02279680073261261
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,fp8,15,0.01655520051717758
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,fp8,31,0.01674720048904419
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,float16,31,0.022782400250434875
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,float16,63,0.022750400006771088
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,fp8,63,0.0171424001455307
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,fp8,127,0.01655520051717758
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,float16,127,0.022991999983787537
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,fp8,255,0.0184688001871109
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,float16,255,0.028984001278877257
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,float16,511,0.04796479940414429
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,fp8,511,0.030862399935722352
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,float16,1023,0.08533599972724915
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,float16,1,0.016601599752902985
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,fp8,1,0.014588800072669984
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,fp8,1023,0.05085920095443726
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,fp8,3,0.01462559998035431
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,float16,7,0.0165583997964859
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,fp8,7,0.014577600359916686
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,float16,15,0.016633599996566772
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,fp8,15,0.01459839940071106
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,float16,31,0.016527999937534333
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,fp8,31,0.014555199444293976
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,float16,63,0.016631999611854555
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,float16,127,0.016649599373340606
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,fp8,127,0.014571200311183929
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,float16,255,0.01650079935789108
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,fp8,255,0.014567999541759491
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,float16,1,0.01846559941768646
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,fp8,1,0.016572800278663636
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,float16,3,0.018727999925613404
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,fp8,3,0.016524800658226015
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,float16,7,0.018539200723171233
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,float16,15,0.018542400002479552
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,fp8,7,0.016590400040149687
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,float16,31,0.018611200153827667
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,fp8,3,0.01660960018634796
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,float16,31,0.018580800294876097
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,fp8,31,0.01661760061979294
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,float16,63,0.018603199720382692
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,fp8,63,0.01658719927072525
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,float16,127,0.01855359971523285
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,fp8,127,0.016598400473594666
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,float16,255,0.018676799535751343
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,fp8,255,0.016633599996566772
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,float16,1,0.028891199827194215
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,fp8,1,0.026683199405670165
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,float16,3,0.028857600688934327
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,fp8,3,0.026579201221466064
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,fp8,7,0.026764801144599913
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,float16,15,0.029067200422286988
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,fp8,15,0.026947200298309326
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,float16,31,0.029110398888587952
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,fp8,31,0.026888000965118408
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,float16,63,0.028896000981330872
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,fp8,63,0.025905600190162657
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,float16,127,0.029756799340248108
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,fp8,127,0.025940799713134767
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,float16,255,0.03311359882354736
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,fp8,255,0.02712000012397766
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,float16,1,0.039699199795722964
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,fp8,63,0.014508800208568573
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,fp8,1,0.026878398656845093
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,float16,3,0.040299201011657716
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,fp8,3,0.026844799518585205
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,float16,7,0.04019840061664581
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,fp8,7,0.026844799518585205
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,float16,15,0.03967039883136749
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,fp8,15,0.026868799328804018
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,float16,31,0.03964639902114868
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,fp8,31,0.026867198944091796
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,float16,63,0.0408048003911972
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,fp8,63,0.026894399523735048
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,float16,127,0.04130400121212006
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,float16,127,0.012451200187206269
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,fp8,127,0.02690559923648834
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,fp8,255,0.03138239979743958
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,float16,255,0.04928320050239563
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,1,128,1,float16,fp8,1,0.022257600724697114
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,1,128,1,float16,float16,3,0.024715200066566467
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,1,128,1,float16,fp8,3,0.022283199429512023
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,1,128,1,float16,float16,7,0.024750399589538574
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,1,128,1,float16,fp8,7,0.022672000527381896
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,1,128,1,float16,float16,15,0.024823999404907225
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,1,128,1,float16,fp8,15,0.022678400576114654
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,1,128,1,float16,float16,31,0.024827200174331664
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,float16,7,0.02889919877052307
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,1,128,1,float16,fp8,31,0.022700800001621245
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,1,128,1,float16,float16,63,0.024753600358963013
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,1,128,1,float16,fp8,63,0.02186720073223114
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,1,128,1,float16,float16,127,0.024843199551105498
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,1,128,1,float16,fp8,127,0.022686399519443512
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,2,128,1,float16,float16,1,0.02680320143699646
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,2,128,1,float16,fp8,1,0.024014399945735933
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,2,128,1,float16,float16,3,0.027505600452423097
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,2,128,1,float16,float16,7,0.02728799879550934
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,2,128,1,float16,fp8,7,0.02431199997663498
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,2,128,1,float16,float16,15,0.027091199159622194
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,2,128,1,float16,fp8,15,0.02425280064344406
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,2,128,1,float16,float16,31,0.02720159888267517
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,2,128,1,float16,fp8,31,0.024433599412441255
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,2,128,1,float16,float16,63,0.02719680070877075
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,2,128,1,float16,fp8,63,0.024556800723075867
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,2,128,1,float16,float16,127,0.02892639935016632
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,2,128,1,float16,fp8,127,0.023171199858188628
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,4,128,1,float16,float16,1,0.05124800205230713
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,4,128,1,float16,fp8,1,0.04400320053100586
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,4,128,1,float16,float16,3,0.051451200246810914
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,4,128,1,float16,float16,7,0.05092160105705261
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,4,128,1,float16,fp8,3,0.044819200038909913
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,4,128,1,float16,float16,15,0.05114719867706299
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,4,128,1,float16,fp8,15,0.044726398587226865
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,float16,3,0.01652960032224655
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,4,128,1,float16,float16,31,0.05143200159072876
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,4,128,1,float16,fp8,63,0.04483680129051208
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,4,128,1,float16,fp8,31,0.04514400064945221
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,4,128,1,float16,float16,63,0.05243840217590332
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,4,128,1,float16,float16,127,0.053495997190475465
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,4,128,1,float16,fp8,127,0.045623999834060666
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,8,128,1,float16,float16,1,0.07214239835739136
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,8,128,1,float16,fp8,1,0.04556959867477417
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,8,128,1,float16,float16,3,0.0727407991886139
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,8,128,1,float16,float16,7,0.07223359942436218
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,8,128,1,float16,fp8,7,0.04624800086021423
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,8,128,1,float16,float16,15,0.07246879935264587
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,8,128,1,float16,fp8,15,0.04612640142440796
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,8,128,1,float16,fp8,31,0.046054399013519286
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,fp8,15,0.016414399445056915
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,8,128,1,float16,float16,63,0.07406240105628967
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,8,128,1,float16,float16,31,0.07390720248222352
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,8,128,1,float16,float16,127,0.07421600222587585
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,1,128,1,float16,float16,1,0.024846400320529937
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,float16,1,0.020630399882793426
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,fp8,1,0.02072319984436035
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,float16,3,0.020640000700950623
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,fp8,3,0.02075839936733246
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,float16,7,0.020678399503231047
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,8,128,1,float16,fp8,127,0.04756959974765777
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,fp8,7,0.020630399882793426
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,float16,15,0.020737600326538087
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,float16,31,0.020689600706100465
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,fp8,31,0.020720000565052032
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,float16,63,0.0206496000289917
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,fp8,63,0.02072319984436035
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,2,128,1,float16,fp8,3,0.023918400704860687
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,float16,127,0.020715199410915375
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,fp8,127,0.020665599405765532
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,float16,255,0.02066880017518997
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,float16,511,0.016638399660587312
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,fp8,255,0.020644800364971162
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,fp8,511,0.016622400283813475
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,float16,1023,0.014511999487876893
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,fp8,1023,0.01451680064201355
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,float16,2047,0.014504000544548035
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,fp8,2047,0.014494399726390838
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,float16,1,0.014508800208568573
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,fp8,1,0.0144896000623703
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,float16,3,0.014484800398349762
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,fp8,3,0.014481599628925323
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,4,128,1,float16,fp8,7,0.045265600085258484
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,float16,7,0.014531199634075165
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,fp8,7,0.014510400593280792
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,float16,15,0.014571200311183929
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,fp8,15,0.014494399726390838
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,float16,31,0.014529600739479065
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,fp8,31,0.014529600739479065
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,float16,63,0.014500799775123595
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,float16,127,0.014593599736690522
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,fp8,127,0.014481599628925323
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,float16,255,0.01449120044708252
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,fp8,255,0.014452800154685974
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,float16,511,0.012534399330615998
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,fp8,511,0.012430399656295776
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,fp8,63,0.014580799639225006
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,float16,1023,0.01247519999742508
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,fp8,1023,0.012452799826860428
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,float16,2047,0.012555199861526489
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,fp8,2047,0.012433599680662155
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,float16,1,0.010473600029945374
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,float16,3,0.010375999659299851
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,fp8,3,0.010492800176143647
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,fp8,7,0.01051200032234192
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,float16,7,0.010395199805498124
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,fp8,15,0.010428799688816071
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,float16,31,0.01048320010304451
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,fp8,31,0.010416000336408614
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,float16,63,0.010507199913263321
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,fp8,63,0.01048320010304451
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,8,128,1,float16,fp8,63,0.0472896009683609
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,float16,127,0.010411199927330018
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,fp8,127,0.010470400005578995
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,float16,255,0.01043040007352829
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,fp8,255,0.01048320010304451
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,float16,511,0.010470400005578995
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,fp8,511,0.010401599854230881
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,float16,1023,0.010497599840164185
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,fp8,1023,0.010486400127410889
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,float16,1,0.02274399995803833
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,fp8,2047,0.01051200032234192
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,float16,1,0.010492800176143647
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,fp8,1,0.008849599957466125
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,float16,3,0.010427200049161912
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,fp8,3,0.009078399837017059
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,float16,7,0.0103472001850605
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,fp8,7,0.008878400176763534
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,float16,15,0.010425599664449692
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,fp8,15,0.010366400331258773
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,float16,31,0.010412800312042236
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,fp8,31,0.01032319962978363
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,float16,63,0.010420800000429154
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,fp8,63,0.010382399708032609
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,float16,127,0.01043360009789467
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,fp8,127,0.010393600165843963
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,float16,255,0.010467199981212616
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,fp8,255,0.010342399775981902
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,float16,511,0.010424000024795533
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,fp8,511,0.00860000029206276
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,float16,1023,0.010449600219726563
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,float16,2047,0.012492799758911132
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,fp8,2047,0.01234399974346161
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,fp8,1023,0.010380800068378448
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,1,128,1,float16,float16,1,0.03907679915428162
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,1,128,1,float16,fp8,1,0.034004798531532286
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,1,128,1,float16,float16,3,0.03909760117530823
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,1,128,1,float16,fp8,3,0.033215999603271484
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,8,128,1,float16,fp8,3,0.04541600048542023
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,1,128,1,float16,fp8,7,0.03345440030097961
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,1,128,1,float16,float16,7,0.03914879858493805
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,1,128,1,float16,float16,15,0.03914879858493805
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,fp8,1,0.010496000200510025
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,1,128,1,float16,float16,31,0.03912639915943146
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,1,128,1,float16,fp8,31,0.03477280139923096
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,1,128,1,float16,float16,63,0.040905600786209105
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,1,128,1,float16,fp8,63,0.03342239856719971
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,2,128,1,float16,fp8,1,0.04043039977550507
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,2,128,1,float16,fp8,3,0.04022879898548126
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,2,128,1,float16,float16,3,0.04665760099887848
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,2,128,1,float16,float16,7,0.04710240066051483
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,2,128,1,float16,fp8,7,0.04047839939594269
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,2,128,1,float16,float16,15,0.0473904013633728
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,2,128,1,float16,fp8,15,0.04020639955997467
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,float16,15,0.010444799810647965
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,2,128,1,float16,float16,31,0.04800800085067749
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,2,128,1,float16,fp8,31,0.04036479890346527
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,2,128,1,float16,fp8,63,0.040140798687934874
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,4,128,1,float16,float16,1,0.09337279796600342
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,4,128,1,float16,fp8,1,0.0822431981563568
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,4,128,1,float16,float16,3,0.0932416021823883
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,fp8,15,0.020628799498081208
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,4,128,1,float16,fp8,3,0.08218880295753479
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,2,128,1,float16,float16,63,0.0494735985994339
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,4,128,1,float16,float16,7,0.0936959981918335
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,4,128,1,float16,fp8,7,0.08230559825897217
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,4,128,1,float16,float16,15,0.09543840289115905
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,4,128,1,float16,float16,31,0.09467999935150147
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,4,128,1,float16,fp8,31,0.08346239924430847
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,4,128,1,float16,float16,63,0.09551680088043213
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,8,128,1,float16,float16,1,0.13962240219116212
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,8,128,1,float16,float16,3,0.14023200273513795
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,8,128,1,float16,fp8,3,0.08421760201454162
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,8,128,1,float16,float16,7,0.14036320447921752
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,8,128,1,float16,fp8,7,0.08430079817771911
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,8,128,1,float16,float16,15,0.14035840034484864
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,8,128,1,float16,fp8,15,0.08438559770584106
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,1,128,1,float16,fp8,15,0.03333599865436554
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,8,128,1,float16,float16,31,0.13971999883651734
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,2,128,1,float16,float16,1,0.0462224006652832
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,8,128,1,float16,fp8,31,0.08461599946022033
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,8,128,1,float16,float16,63,0.14031840562820436
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,8,128,1,float16,fp8,63,0.08469120264053345
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,1,128,1,float16,float16,1,0.07177760004997254
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,1,128,1,float16,fp8,1,0.059620797634124756
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,1,128,1,float16,float16,3,0.07117760181427002
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,1,128,1,float16,float16,7,0.07153760194778443
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,1,128,1,float16,fp8,7,0.059699201583862306
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,1,128,1,float16,float16,15,0.07165759801864624
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,1,128,1,float16,fp8,15,0.059729599952697755
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,1,128,1,float16,float16,31,0.07208319902420043
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,1,128,1,float16,fp8,31,0.06017919778823853
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,2,128,1,float16,float16,1,0.0856383979320526
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,2,128,1,float16,fp8,1,0.07263839840888978
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,2,128,1,float16,float16,3,0.08622080087661743
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,float16,2047,0.01241919994354248
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,2,128,1,float16,fp8,3,0.07277439832687378
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,2,128,1,float16,float16,7,0.0863856017589569
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,2,128,1,float16,fp8,7,0.07274240255355835
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,4,128,1,float16,fp8,15,0.08102719783782959
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,2,128,1,float16,float16,15,0.0864080011844635
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,2,128,1,float16,fp8,15,0.07317919731140136
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,4,128,1,float16,fp8,63,0.08329280018806458
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,2,128,1,float16,fp8,31,0.07298079729080201
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,4,128,1,float16,float16,1,0.17969440221786498
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,4,128,1,float16,fp8,1,0.15811840295791627
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,4,128,1,float16,float16,3,0.17980480194091797
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,4,128,1,float16,fp8,3,0.15471839904785156
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,4,128,1,float16,fp8,7,0.1577615976333618
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,4,128,1,float16,float16,15,0.18019200563430787
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,4,128,1,float16,fp8,15,0.15481920242309571
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,4,128,1,float16,float16,31,0.1808143973350525
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,4,128,1,float16,float16,7,0.17915359735488892
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,4,128,1,float16,fp8,31,0.1562399983406067
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,8,128,1,float16,fp8,1,0.1600607991218567
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,8,128,1,float16,float16,1,0.2723439931869507
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,8,128,1,float16,float16,3,0.27251999378204345
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,8,128,1,float16,float16,7,0.27212479114532473
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,8,128,1,float16,fp8,7,0.1602015972137451
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,8,128,1,float16,fp8,15,0.16025760173797607
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,float16,1,0.02070239931344986
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,8,128,1,float16,fp8,31,0.1603600025177002
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,8,128,1,float16,float16,31,0.2721472024917603
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,fp8,1,0.020644800364971162
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,float16,3,0.020761600136756896
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,float16,7,0.02072799950838089
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,fp8,3,0.020630399882793426
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,fp8,7,0.02072640061378479
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,float16,15,0.020652799308300017
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,fp8,15,0.020662400126457214
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,float16,31,0.020798400044441223
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,float16,63,0.02067520022392273
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,fp8,63,0.020720000565052032
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,float16,127,0.02064639925956726
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,fp8,127,0.02072480022907257
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,float16,255,0.020688000321388244
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,fp8,255,0.019387200474739075
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,1,128,1,float16,fp8,3,0.05976319909095764
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,fp8,511,0.016657599806785585
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,float16,1023,0.014569599926471711
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,fp8,1023,0.014617599546909332
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,float16,2047,0.016420799493789672
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,fp8,2047,0.014563199877738953
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,float16,1,0.01451680064201355
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,fp8,1,0.01459839940071106
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,float16,3,0.014476799964904785
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,float16,7,0.014548799395561219
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,fp8,3,0.014627200365066529
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,fp8,7,0.014558400213718414
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,float16,15,0.014555199444293976
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,fp8,15,0.014537599682807923
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,float16,31,0.014614400267601014
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,fp8,31,0.014553600549697876
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,2,128,1,float16,float16,31,0.0870959997177124
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,float16,63,0.014596800506114959
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,fp8,63,0.014584000408649444
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,8,128,1,float16,fp8,1,0.08427680134773255
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,fp8,127,0.014591999351978302
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,float16,255,0.01462559998035431
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,fp8,255,0.014587199687957764
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,float16,511,0.014564800262451171
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,fp8,511,0.01255040019750595
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,float16,1023,0.012588800489902496
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,fp8,1023,0.012534399330615998
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,float16,2047,0.014550399780273438
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,fp8,2047,0.012508800625801087
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,float16,1,0.010860799998044967
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,fp8,1,0.010478399693965912
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,float16,3,0.01106399968266487
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,fp8,3,0.010478399693965912
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,float16,7,0.01064639985561371
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,fp8,7,0.010513599961996078
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,float16,15,0.010532800108194351
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,fp8,15,0.010543999820947647
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,float16,31,0.010451199859380722
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,8,128,1,float16,float16,15,0.27245440483093264
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,float16,63,0.010478399693965912
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,fp8,31,0.010620799660682679
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,8,128,1,float16,fp8,3,0.1603600025177002
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,fp8,63,0.010531199723482132
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,float16,127,0.010550399869680404
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,fp8,127,0.010790400207042694
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,float16,255,0.011137600243091583
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,float16,511,0.010476800054311753
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,fp8,511,0.010507199913263321
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,float16,1023,0.012468799948692322
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,fp8,1023,0.010523200035095215
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,fp8,2047,0.013150399923324585
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,float16,1,0.008422400057315826
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,fp8,1,0.00835999995470047
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,float16,3,0.008387199789285659
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,fp8,3,0.008353599905967712
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,float16,7,0.008481600135564805
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,fp8,7,0.008347199857234954
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,float16,511,0.01664319932460785
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,float16,31,0.008446399867534638
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,fp8,31,0.008366400003433227
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,float16,63,0.008376000076532364
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,fp8,63,0.008452799916267396
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,float16,127,0.008343999832868576
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,fp8,127,0.008352000266313553
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,float16,255,0.008444800227880477
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,fp8,255,0.008329600095748901
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,float16,511,0.010388799756765366
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,fp8,511,0.008441600203514098
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,float16,1023,0.012462399899959564
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,float16,127,0.01459999978542328
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,fp8,1023,0.010424000024795533
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,float16,2047,0.027060800790786745
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,fp8,2047,0.01451520025730133
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,float16,1,0.019331200420856474
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,fp8,1,0.018611200153827667
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,float16,3,0.019108800590038298
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,float16,7,0.019312000274658202
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,fp8,7,0.018606400489807128
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,float16,15,0.01913599967956543
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,fp8,15,0.018713599443435668
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,float16,31,0.019409599900245666
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,fp8,31,0.018619200587272643
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,float16,63,0.019275200366973878
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,float16,127,0.018673600256443025
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,fp8,127,0.018614399433135986
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,float16,255,0.018670399487018586
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,fp8,255,0.010492800176143647
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,fp8,255,0.01858240067958832
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,float16,511,0.016574400663375854
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,fp8,511,0.016551999747753142
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,float16,2047,0.014552000164985656
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,fp8,31,0.02064319998025894
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,float16,1023,0.01860480010509491
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,fp8,1023,0.017159999907016756
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,float16,2047,0.02299039959907532
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,float16,1,0.010395199805498124
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,fp8,1,0.008366400003433227
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,fp8,2047,0.020732800662517547
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,float16,3,0.010407999902963639
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,fp8,3,0.009198399633169175
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,float16,7,0.010353600233793258
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,fp8,7,0.009388799965381622
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,float16,15,0.010470400005578995
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,fp8,15,0.008430399745702744
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,fp8,15,0.009567999839782714
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,fp8,31,0.00989760011434555
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,fp8,63,0.008430399745702744
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,float16,127,0.010471999645233154
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,fp8,127,0.008473599702119828
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,float16,255,0.01035040020942688
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,fp8,255,0.008856000006198883
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,float16,511,0.012329600006341934
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,float16,1023,0.01451839953660965
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,fp8,1023,0.012503999471664428
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,float16,2047,0.02913280129432678
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,fp8,2047,0.018198400735855103
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,float16,1,0.012467200309038163
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,fp8,1,0.012355200201272964
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,float16,3,0.012468799948692322
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,fp8,3,0.012483199685811996
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,float16,7,0.012516799569129943
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,fp8,7,0.012342400103807449
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,float16,15,0.012479999661445617
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,fp8,15,0.012268800288438797
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,fp8,3,0.018628799915313722
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,float16,31,0.012572799623012543
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,fp8,31,0.012438400089740754
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,float16,63,0.01252640038728714
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,float16,127,0.01252640038728714
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,fp8,127,0.012438400089740754
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,float16,255,0.012537600100040435
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,fp8,63,0.012452799826860428
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,fp8,255,0.012467200309038163
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,fp8,511,0.014511999487876893
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,float16,1023,0.029558399319648744
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,fp8,1023,0.018675200641155243
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,float16,2047,0.048747199773788455
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,fp8,2047,0.03292959928512573
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,float16,1,0.014588800072669984
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,fp8,1,0.012540799379348756
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,float16,3,0.014591999351978302
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,float16,15,0.00838399976491928
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,float16,31,0.010391999781131745
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,fp8,3,0.012603199481964112
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,float16,7,0.01456640064716339
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,float16,15,0.014529600739479065
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,fp8,15,0.012511999905109405
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,float16,31,0.01462559998035431
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,float16,63,0.014535999298095703
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,fp8,31,0.012529599666595458
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,fp8,63,0.012489599734544754
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,float16,127,0.014528000354766845
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,fp8,127,0.01239679977297783
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,float16,255,0.014547200500965118
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,fp8,255,0.012465599924325943
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,float16,511,0.028880000114440918
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,fp8,511,0.016627199947834015
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,float16,1023,0.04817599952220917
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,fp8,1023,0.030568000674247742
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,float16,2047,0.08602560162544251
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,fp8,2047,0.04935519993305206
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,float16,1,0.01658719927072525
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,fp8,1,0.01664000004529953
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,float16,3,0.016603200137615202
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,fp8,3,0.016684800386428833
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,float16,7,0.016659200191497803
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,fp8,7,0.016638399660587312
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,float16,15,0.0166143998503685
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,fp8,15,0.016672000288963318
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,float16,31,0.01659200042486191
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,fp8,31,0.016649599373340606
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,float16,63,0.016572800278663636
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,fp8,63,0.01656319946050644
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,float16,127,0.01660960018634796
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,fp8,127,0.016652800142765045
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,fp8,255,0.016516800224781036
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,float16,511,0.014556799829006196
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,fp8,511,0.014534400403499603
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,fp8,63,0.018670399487018586
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,fp8,1023,0.012465599924325943
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,float16,1023,0.014481599628925323
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,float16,2047,0.013566400110721587
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,fp8,2047,0.012542399764060973
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,float16,1,0.012558400630950928
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,fp8,1,0.01252640038728714
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,float16,3,0.012511999905109405
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,fp8,3,0.012580800056457519
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,float16,7,0.012510399520397186
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,float16,15,0.012963199615478515
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,fp8,7,0.0125231996178627
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,fp8,15,0.012462399899959564
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,float16,31,0.012558400630950928
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,float16,63,0.012452799826860428
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,fp8,31,0.012508800625801087
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,fp8,63,0.012529599666595458
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,float16,127,0.012547199428081513
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,fp8,127,0.012535999715328216
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,float16,255,0.012476799637079239
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,fp8,255,0.012460800260305405
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,float16,511,0.01244800016283989
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,float16,63,0.010465600341558457
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,fp8,511,0.01170559972524643
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,float16,1023,0.012462399899959564
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,fp8,1023,0.01045600026845932
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,float16,2047,0.010496000200510025
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,fp8,2047,0.010496000200510025
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,float16,1,0.010503999888896942
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,fp8,1,0.010406400263309478
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,float16,3,0.010515200346708298
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,fp8,3,0.010494399815797806
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,fp8,7,0.010425599664449692
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,float16,15,0.010470400005578995
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,fp8,15,0.010558400303125381
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,float16,31,0.010468800365924836
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,fp8,31,0.010480000078678131
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,float16,63,0.010480000078678131
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,float16,127,0.010505600273609162
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,fp8,127,0.010384000092744827
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,float16,255,0.010409600287675857
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,fp8,255,0.010478399693965912
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,float16,511,0.010388799756765366
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,fp8,511,0.010475199669599533
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,float16,1023,0.010494399815797806
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,fp8,1023,0.010460799932479859
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,float16,2047,0.011635199934244157
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,fp8,2047,0.010502400249242783
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,float16,1,0.010424000024795533
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,fp8,1,0.009379199892282485
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,float16,3,0.01037919968366623
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,fp8,3,0.009332799911499023
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,float16,255,0.01661120057106018
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,float16,7,0.010355199873447418
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,float16,15,0.010419200360774993
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,fp8,7,0.00894080027937889
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,fp8,15,0.01034879982471466
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,float16,31,0.010412800312042236
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,float16,511,0.01653279960155487
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,float16,63,0.010492800176143647
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,float16,127,0.010489600151777268
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,fp8,63,0.010422399640083313
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,fp8,127,0.009359999746084213
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,float16,255,0.010438399761915207
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,fp8,255,0.00942080020904541
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,float16,511,0.010409600287675857
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,fp8,511,0.01045600026845932
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,float16,1023,0.010371199995279311
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,fp8,1023,0.010463999956846238
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,fp8,2047,0.010489600151777268
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,float16,1,0.01706880033016205
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,fp8,1,0.016622400283813475
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,float16,3,0.01711200028657913
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,fp8,7,0.012534399330615998
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,fp8,3,0.016628800332546233
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,float16,7,0.01857759952545166
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,fp8,7,0.016612799465656282
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,float16,15,0.01857440024614334
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,fp8,15,0.016596800088882445
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,fp8,63,0.010494399815797806
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,fp8,31,0.016612799465656282
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,float16,63,0.01663520038127899
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,fp8,63,0.016571199893951415
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,float16,127,0.01661120057106018
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,fp8,127,0.0166143998503685
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,fp8,255,0.01659359931945801
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,float16,255,0.01664479970932007
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,float16,511,0.015150399506092071
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,fp8,511,0.01459999978542328
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,float16,1023,0.014422400295734406
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,fp8,1023,0.012534399330615998
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,float16,2047,0.014502400159835815
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,fp8,2047,0.012572799623012543
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,float16,1,0.012492799758911132
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,fp8,1,0.01260959953069687
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,float16,3,0.012596799433231354
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,fp8,3,0.012516799569129943
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,float16,7,0.012492799758911132
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,fp8,7,0.013118399679660797
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,float16,15,0.012617599964141846
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,fp8,15,0.014585599303245544
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,fp8,31,0.012641599774360657
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,float16,63,0.01449279934167862
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,fp8,63,0.012558400630950928
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,fp8,31,0.01040000021457672
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,fp8,127,0.012534399330615998
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,float16,255,0.012723200023174286
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,fp8,255,0.012547199428081513
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,fp8,511,0.012460800260305405
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,float16,1023,0.011209599673748016
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,fp8,1023,0.010487999767065048
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,float16,2047,0.01048479974269867
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,float16,2047,0.01165440008044243
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,fp8,2047,0.01043360009789467
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,float16,1,0.010507199913263321
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,fp8,1,0.010441599786281586
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,fp8,3,0.010407999902963639
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,float16,7,0.010422399640083313
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,fp8,511,0.010384000092744827
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,float16,7,0.010470400005578995
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,fp8,7,0.010427200049161912
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,float16,15,0.010465600341558457
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,fp8,15,0.010438399761915207
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,fp8,31,0.010465600341558457
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,float16,31,0.01051200032234192
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,float16,31,0.018673600256443025
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,float16,63,0.010515200346708298
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,float16,127,0.010492800176143647
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,fp8,127,0.010489600151777268
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,float16,255,0.010468800365924836
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,fp8,255,0.010473600029945374
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,float16,511,0.010473600029945374
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,fp8,511,0.010444799810647965
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,float16,1023,0.010443200170993806
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,fp8,1023,0.01043360009789467
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,float16,2047,0.010999999940395355
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,fp8,2047,0.010497599840164185
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,float16,1,0.01040000021457672
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,fp8,1,0.008638399839401244
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,fp8,3,0.008753599971532822
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,float16,7,0.010390400141477584
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,fp8,7,0.010372799634933472
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,float16,15,0.010409600287675857
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,fp8,15,0.010372799634933472
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,float16,31,0.010424000024795533
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,fp8,31,0.008729600161314011
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,float16,31,0.012603199481964112
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,float16,63,0.01043360009789467
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,fp8,63,0.00974079966545105
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,float16,127,0.010380800068378448
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,fp8,127,0.009518399834632874
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,float16,255,0.010390400141477584
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,float16,511,0.010396800190210342
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,float16,127,0.01345600038766861
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,fp8,511,0.010398399829864503
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,float16,1023,0.010502400249242783
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,float16,511,0.012425599992275238
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,fp8,2047,0.010500799864530563
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,float16,1,0.01868640035390854
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,fp8,1,0.018603199720382692
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,float16,3,0.01857759952545166
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,float16,3,0.010507199913263321
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,fp8,3,0.01857759952545166
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,float16,7,0.018667200207710268
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,fp8,7,0.018569600582122803
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,float16,15,0.0186256006360054
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,fp8,15,0.018564799427986146
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,float16,31,0.018667200207710268
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,fp8,31,0.01857600063085556
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,fp8,63,0.018580800294876097
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,float16,63,0.018692800402641298
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,float16,127,0.018588800728321076
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,fp8,63,0.010382399708032609
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,fp8,127,0.018607999384403228
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,float16,255,0.018566399812698364
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,fp8,255,0.01807679980993271
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,float16,511,0.01658719927072525
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,fp8,511,0.01560640037059784
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,float16,1023,0.014563199877738953
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,fp8,1023,0.01449279934167862
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,float16,2047,0.016655999422073364
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,fp8,2047,0.015062400698661804
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,float16,1,0.014548799395561219
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,fp8,1,0.014190399646759033
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,float16,3,0.014454400539398194
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,fp8,3,0.013995200395584106
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,float16,3,0.010203199833631516
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,float16,7,0.014563199877738953
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,float16,15,0.014497600495815277
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,fp8,15,0.012486399710178375
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,float16,31,0.014531199634075165
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,fp8,31,0.014236800372600555
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,float16,63,0.014521600306034088
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,fp8,63,0.014494399726390838
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,float16,127,0.014480000734329224
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,fp8,127,0.014499199390411378
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,float16,255,0.014467200636863709
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,fp8,255,0.012544000148773193
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,float16,511,0.012756800651550293
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,float16,1023,0.01462240070104599
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,fp8,511,0.012465599924325943
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,float16,2047,0.01849759966135025
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,fp8,2047,0.016495999693870545
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,float16,1,0.010422399640083313
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,fp8,1,0.008441600203514098
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,float16,3,0.010422399640083313
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,fp8,3,0.008422400057315826
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,float16,7,0.010395199805498124
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,fp8,7,0.010366400331258773
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,float16,15,0.010409600287675857
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,fp8,15,0.010360000282526016
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,fp8,255,0.010087999701499938
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,fp8,31,0.01040479987859726
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,float16,31,0.0104592002928257
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,fp8,63,0.01037440001964569
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,float16,127,0.0104032002389431
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,fp8,1023,0.0104032002389431
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,float16,63,0.010470400005578995
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,fp8,127,0.009046400338411332
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,float16,2047,0.01096000000834465
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,fp8,255,0.009377600252628326
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,float16,511,0.011401599645614624
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,fp8,511,0.010385599732398988
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,float16,1023,0.014575999975204468
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,fp8,1023,0.012556800246238708
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,float16,2047,0.028811201453208923
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,float16,1,0.010710400342941285
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,fp8,2047,0.01661919951438904
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,float16,3,0.010660800337791442
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,fp8,7,0.010351999849081039
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,float16,15,0.010476800054311753
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,fp8,3,0.010460799932479859
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,float16,31,0.01075040027499199
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,fp8,31,0.010494399815797806
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,float16,63,0.010729599744081497
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,fp8,63,0.010444799810647965
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,float16,127,0.010531199723482132
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,fp8,127,0.010441599786281586
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,float16,255,0.010471999645233154
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,fp8,255,0.01045600026845932
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,float16,511,0.013444800674915314
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,fp8,511,0.012441600114107132
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,float16,1023,0.028259199857711793
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,fp8,7,0.014176000654697419
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,float16,2047,0.0470335990190506
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,fp8,2047,0.02908639907836914
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,float16,1,0.018531200289726258
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,fp8,1,0.016654400527477263
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,fp8,1023,0.014529600739479065
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,fp8,3,0.016651199758052827
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,float16,7,0.018638400733470915
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,fp8,7,0.016646400094032288
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,float16,15,0.01857919991016388
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,fp8,15,0.016648000478744505
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,float16,31,0.01855839937925339
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,fp8,31,0.016628800332546233
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,float16,255,0.010390400141477584
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,float16,63,0.01858399957418442
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,fp8,63,0.01860480010509491
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,float16,127,0.01857440024614334
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,fp8,127,0.01664479970932007
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,float16,255,0.016838400065898894
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,fp8,1,0.010420800000429154
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,fp8,255,0.016631999611854555
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,float16,511,0.015094399452209473
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,float16,7,0.010516799986362457
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,float16,1023,0.013179199397563934
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,fp8,1023,0.012470400333404541
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,fp8,15,0.010420800000429154
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,fp8,2047,0.012539200484752655
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,float16,1,0.012535999715328216
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,fp8,1,0.012467200309038163
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,fp8,3,0.012452799826860428
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,float16,7,0.012535999715328216
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,fp8,7,0.012464000284671784
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,float16,15,0.012460800260305405
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,fp8,15,0.012468799948692322
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,fp8,31,0.01255040019750595
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,float16,31,0.012532800436019897
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,float16,63,0.012539200484752655
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,fp8,63,0.012516799569129943
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,float16,127,0.012542399764060973
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,fp8,127,0.01255040019750595
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,float16,255,0.01255040019750595
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,fp8,255,0.012518399953842163
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,fp8,511,0.01244639977812767
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,float16,511,0.012462399899959564
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,float16,1023,0.012427199631929398
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,fp8,1023,0.010920000076293946
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,float16,2047,0.012428800016641617
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,fp8,2047,0.012337599694728852
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,float16,1,0.010447999835014344
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,fp8,1,0.010393600165843963
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,float16,3,0.010496000200510025
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,fp8,3,0.01040479987859726
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,float16,7,0.010494399815797806
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,fp8,7,0.010409600287675857
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,float16,15,0.010440000146627427
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,fp8,1023,0.014734399318695069
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,fp8,15,0.010390400141477584
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,fp8,31,0.010428799688816071
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,float16,63,0.010398399829864503
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,float16,3,0.01860480010509491
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,fp8,63,0.010460799932479859
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,float16,127,0.01048479974269867
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,fp8,127,0.010414399951696397
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,float16,255,0.010523200035095215
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,fp8,255,0.0104032002389431
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,float16,511,0.010480000078678131
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,fp8,511,0.010449600219726563
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,float16,1023,0.010496000200510025
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,fp8,1023,0.01053439974784851
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,float16,2047,0.012484800070524216
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,float16,1,0.008427199721336365
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,float16,3,0.008403199911117553
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,fp8,3,0.008449599891901017
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,float16,7,0.008425600081682205
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,fp8,7,0.00843840017914772
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,float16,15,0.008454400300979614
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,fp8,15,0.008416000008583068
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,float16,31,0.008726400136947633
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,fp8,31,0.009142400324344635
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,fp8,63,0.008591999858617782
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,float16,127,0.010395199805498124
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,fp8,511,0.014619199931621552
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,fp8,127,0.008457600325345992
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,float16,255,0.009590400010347366
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,fp8,255,0.008433599770069123
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,float16,511,0.008823999762535095
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,float16,3,0.012652799487113953
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,fp8,511,0.008452799916267396
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,float16,1023,0.010385599732398988
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,fp8,1023,0.009808000177145004
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,float16,2047,0.010516799986362457
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,fp8,2047,0.010417599976062775
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,float16,1,0.012443199753761292
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,fp8,1,0.011128000169992446
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,float16,3,0.012451200187206269
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,float16,7,0.012457600235939026
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,fp8,3,0.010441599786281586
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,fp8,7,0.010806400328874588
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,fp8,15,0.011003199964761734
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,float16,31,0.01247360035777092
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,fp8,31,0.010467199981212616
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,float16,63,0.0123648002743721
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,fp8,63,0.010470400005578995
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,float16,127,0.012467200309038163
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,fp8,127,0.010760000348091126
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,float16,31,0.010406400263309478
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,float16,255,0.010529600083827972
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,fp8,255,0.010516799986362457
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,float16,511,0.013977600634098053
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,fp8,511,0.012518399953842163
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,float16,1023,0.018571199476718904
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,float16,1,0.01250240057706833
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,fp8,2047,0.012249600142240524
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,fp8,1,0.00843520015478134
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,float16,3,0.012432000041007996
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,float16,7,0.012432000041007996
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,fp8,7,0.010710400342941285
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,float16,15,0.012462399899959564
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,fp8,15,0.010943999886512757
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,float16,63,0.010260800272226334
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,float16,31,0.012454400211572647
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,fp8,31,0.012464000284671784
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,float16,63,0.012531200051307678
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,fp8,63,0.012144000083208085
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,float16,127,0.012494400143623352
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,float16,2047,0.014478400349617004
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,float16,255,0.01249919980764389
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,fp8,255,0.010515200346708298
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,fp8,511,0.014475199580192565
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,fp8,1023,0.018606400489807128
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,float16,1,0.018595199286937713
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,fp8,1,0.016543999314308167
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,float16,15,0.01242400035262108
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,float16,3,0.018595199286937713
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,fp8,3,0.01656000018119812
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,float16,7,0.018585599958896637
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,fp8,7,0.01653600037097931
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,float16,15,0.01860159933567047
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,fp8,15,0.016550399363040924
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,fp8,31,0.01652960032224655
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,float16,63,0.018590399622917177
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,fp8,63,0.016551999747753142
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,float16,127,0.018649600446224213
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,fp8,127,0.016542400419712066
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,float16,255,0.018668800592422485
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,fp8,255,0.016625599563121797
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,float16,511,0.031137600541114807
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,fp8,511,0.02106879949569702
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,float16,1023,0.050521600246429446
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,fp8,1,0.011537600308656693
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,float16,1,0.02266400009393692
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,fp8,1023,0.032913601398468016
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,fp8,1,0.01659359931945801
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,float16,3,0.022788800299167633
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,fp8,3,0.01661120057106018
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,float16,7,0.022759999334812164
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,fp8,7,0.0166143998503685
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,float16,15,0.02280000001192093
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,fp8,15,0.01690720021724701
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,float16,31,0.022752000391483305
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,fp8,31,0.016908800601959227
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,float16,63,0.022881600260734557
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,fp8,63,0.016918399930000307
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,float16,127,0.02277279943227768
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,fp8,127,0.01693760007619858
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,float16,255,0.029139199852943422
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,fp8,255,0.016873599588871004
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,float16,511,0.04764800071716309
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,float16,511,0.01656319946050644
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,float16,1023,0.08541280031204224
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,fp8,1023,0.050254398584365846
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,float16,1,0.01652960032224655
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,float16,1023,0.03095200061798096
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,fp8,1,0.014531199634075165
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,float16,3,0.016521599888801575
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,float16,7,0.016550399363040924
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,fp8,3,0.014606399834156037
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,fp8,7,0.014473600685596466
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,float16,15,0.01658399999141693
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,float16,31,0.016518400609493257
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,fp8,31,0.014486399292945863
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,float16,63,0.016550399363040924
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,fp8,63,0.01446399986743927
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,float16,127,0.016550399363040924
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,fp8,127,0.01448799967765808
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,float16,255,0.01648319959640503
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,float16,31,0.018670399487018586
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,fp8,255,0.014476799964904785
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,float16,1,0.018059200048446654
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,float16,3,0.017790399491786957
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,fp8,3,0.014983999729156493
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,float16,7,0.01789119988679886
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,fp8,7,0.015119999647140503
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,float16,15,0.017969599366188048
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,fp8,15,0.014820800721645355
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,float16,31,0.018009600043296815
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,fp8,31,0.016315199434757233
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,float16,63,0.018592000007629395
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,fp8,63,0.016489599645137788
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,float16,127,0.018595199286937713
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,fp8,127,0.01509920060634613
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,fp8,3,0.010740800201892853
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,float16,255,0.018593600392341612
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,fp8,255,0.014907200634479523
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,fp8,1,0.026137599349021913
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,float16,3,0.028881600499153136
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,fp8,3,0.026131200790405273
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,float16,7,0.028944000601768494
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,fp8,127,0.01250240057706833
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,fp8,7,0.02664479911327362
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,float16,15,0.028889599442481994
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,fp8,15,0.02622399926185608
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,fp8,511,0.02911359965801239
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,fp8,31,0.02624320089817047
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,float16,63,0.028923198580741882
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,float16,127,0.028958401083946227
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,fp8,63,0.02632319927215576
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,fp8,127,0.024835200607776643
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,float16,255,0.031656000018119815
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,fp8,255,0.026824000477790832
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,fp8,15,0.014572800695896148
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,float16,1,0.039427199959754945
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,fp8,1,0.026795199513435362
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,float16,3,0.039470401406288144
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,fp8,7,0.02686559855937958
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,float16,15,0.039182400703430174
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,fp8,15,0.026902401447296144
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,fp8,1,0.016550399363040924
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,float16,31,0.03916159868240356
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,fp8,31,0.0268640011548996
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,float16,63,0.04010559916496277
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,fp8,63,0.026804798841476442
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,float16,127,0.04121600091457367
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,fp8,127,0.026888000965118408
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,float16,255,0.04907839894294739
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,fp8,255,0.03087199926376343
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,1,128,1,float16,fp8,1,0.02077440023422241
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,1,128,1,float16,float16,1,0.024740800261497498
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,fp8,1023,0.01653439998626709
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,1,128,1,float16,float16,3,0.02443680018186569
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,1,128,1,float16,fp8,3,0.02136320024728775
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,1,128,1,float16,float16,7,0.02476319968700409
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,1,128,1,float16,fp8,7,0.020662400126457214
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,1,128,1,float16,fp8,15,0.021070399880409242
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,1,128,1,float16,float16,31,0.024609600007534028
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,1,128,1,float16,fp8,31,0.021184000372886657
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,1,128,1,float16,float16,63,0.02476799935102463
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,1,128,1,float16,fp8,63,0.021161599457263945
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,float16,1,0.02881920039653778
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,1,128,1,float16,float16,127,0.024753600358963013
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,1,128,1,float16,fp8,127,0.021379199624061585
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,2,128,1,float16,float16,1,0.026900801062583923
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,2,128,1,float16,float16,3,0.02709920108318329
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,2,128,1,float16,fp8,3,0.022776000201702118
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,2,128,1,float16,float16,7,0.026974400877952574
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,2,128,1,float16,fp8,7,0.022782400250434875
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,2,128,1,float16,float16,15,0.02690559923648834
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,2,128,1,float16,float16,31,0.027510398626327516
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,2,128,1,float16,fp8,15,0.02279199957847595
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,2,128,1,float16,float16,63,0.02768160104751587
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,2,128,1,float16,fp8,63,0.0227743998169899
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,float16,31,0.02881760001182556
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,2,128,1,float16,fp8,127,0.022761599719524385
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,2,128,1,float16,float16,127,0.028961598873138428
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,4,128,1,float16,float16,1,0.049795201420783995
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,4,128,1,float16,fp8,1,0.04328480064868927
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,4,128,1,float16,float16,3,0.05127360224723816
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,4,128,1,float16,fp8,3,0.04325760006904602
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,4,128,1,float16,float16,7,0.05093439817428589
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,4,128,1,float16,fp8,7,0.04490559995174408
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,4,128,1,float16,float16,15,0.05050240159034729
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,4,128,1,float16,fp8,15,0.04387199878692627
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,4,128,1,float16,fp8,31,0.04394879937171936
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,4,128,1,float16,float16,31,0.05138239860534668
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,4,128,1,float16,float16,63,0.05177760124206543
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,4,128,1,float16,fp8,63,0.04417439997196197
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,4,128,1,float16,float16,127,0.05336800217628479
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,4,128,1,float16,fp8,127,0.04520159959793091
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,float16,7,0.03923520147800445
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,8,128,1,float16,float16,1,0.07253119945526124
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,8,128,1,float16,float16,3,0.0721552014350891
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,8,128,1,float16,fp8,3,0.045521599054336545
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,8,128,1,float16,float16,7,0.07286080121994018
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,8,128,1,float16,fp8,7,0.04551840126514435
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,8,128,1,float16,float16,15,0.07309600114822387
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,8,128,1,float16,fp8,15,0.04646559953689575
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,8,128,1,float16,float16,31,0.07396479845046997
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,8,128,1,float16,fp8,31,0.045819199085235594
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,8,128,1,float16,float16,63,0.07407680153846741
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,8,128,1,float16,fp8,63,0.047049599885940555
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,8,128,1,float16,float16,127,0.0741919994354248
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,8,128,1,float16,fp8,127,0.047332799434661864
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,float16,1,0.018617600202560425
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,fp8,1,0.018580800294876097
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,float16,3,0.018670399487018586
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,fp8,3,0.018588800728321076
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,float16,7,0.018585599958896637
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,fp8,7,0.018467199802398682
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,float16,15,0.018571199476718904
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,fp8,15,0.018436799943447112
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,fp8,31,0.01857600063085556
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,float16,31,0.01871040016412735
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,float16,63,0.018561600148677825
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,1,128,1,float16,float16,15,0.024758400022983552
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,float16,127,0.018593600392341612
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,fp8,63,0.018681600689888
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,fp8,127,0.01866080015897751
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,float16,255,0.016595199704170227
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,fp8,255,0.01677599996328354
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,float16,511,0.014878399670124054
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,fp8,511,0.014550399780273438
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,float16,1023,0.014511999487876893
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,fp8,1023,0.013041600584983826
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,float16,2047,0.014585599303245544
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,fp8,2047,0.014510400593280792
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,float16,1,0.012529599666595458
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,2,128,1,float16,fp8,1,0.0227743998169899
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,float16,3,0.012507200241088867
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,fp8,3,0.012601600587368011
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,float16,7,0.013372799754142762
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,fp8,7,0.01252799928188324
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,float16,15,0.012656000256538392
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,fp8,15,0.012547199428081513
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,fp8,31,0.012567999958992004
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,float16,63,0.013084800541400909
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,2,128,1,float16,fp8,31,0.022697600722312927
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,float16,127,0.01329759955406189
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,fp8,127,0.01255200058221817
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,float16,255,0.012520000338554382
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,float16,511,0.012492799758911132
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,fp8,511,0.012443199753761292
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,float16,1023,0.012414400279521943
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,fp8,1023,0.010692799836397171
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,float16,2047,0.012479999661445617
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,fp8,2047,0.012409599870443344
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,float16,1,0.010416000336408614
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,fp8,3,0.026830399036407472
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,fp8,1,0.010385599732398988
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,float16,3,0.01043199971318245
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,8,128,1,float16,fp8,1,0.04573439955711365
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,fp8,3,0.010692799836397171
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,float16,7,0.010462400317192078
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,fp8,7,0.010380800068378448
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,float16,15,0.010401599854230881
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,fp8,15,0.010844799876213073
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,float16,31,0.010385599732398988
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,fp8,31,0.010409600287675857
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,float16,63,0.010444799810647965
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,fp8,63,0.010384000092744827
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,float16,127,0.01037440001964569
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,float16,255,0.010396800190210342
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,fp8,127,0.010523200035095215
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,float16,511,0.010428799688816071
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,fp8,511,0.010601600259542465
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,float16,1023,0.01040479987859726
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,fp8,1023,0.010463999956846238
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,float16,2047,0.012412799894809723
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,fp8,2047,0.010580799728631973
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,float16,1,0.01040000021457672
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,fp8,1,0.009204799681901932
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,float16,3,0.010364799946546554
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,fp8,3,0.009011200070381165
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,float16,7,0.0104032002389431
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,float16,15,0.010364799946546554
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,fp8,15,0.010382399708032609
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,float16,31,0.01048320010304451
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,fp8,7,0.009356799721717834
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,fp8,31,0.010391999781131745
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,fp8,63,0.009176000207662582
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,float16,63,0.01040000021457672
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,fp8,1,0.012542399764060973
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,float16,127,0.010393600165843963
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,fp8,127,0.009116800129413604
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,float16,255,0.010367999970912933
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,fp8,255,0.008953599631786347
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,float16,511,0.010438399761915207
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,fp8,511,0.00843999981880188
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,float16,31,0.01406400054693222
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,float16,1023,0.010391999781131745
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,fp8,1023,0.010540799796581268
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,float16,2047,0.012569600343704223
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,fp8,63,0.012505599856376648
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,1,128,1,float16,fp8,1,0.0330159991979599
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,1,128,1,float16,fp8,3,0.03297759890556336
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,fp8,255,0.012542399764060973
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,1,128,1,float16,fp8,7,0.03297759890556336
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,1,128,1,float16,float16,15,0.03888159990310669
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,1,128,1,float16,float16,31,0.038966399431228635
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,1,128,1,float16,fp8,31,0.03291040062904358
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,1,128,1,float16,float16,63,0.03903839886188507
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,1,128,1,float16,fp8,63,0.03300159871578216
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,2,128,1,float16,float16,1,0.04589279890060425
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,2,128,1,float16,fp8,1,0.03848319947719574
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,2,128,1,float16,float16,3,0.04575519859790802
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,2,128,1,float16,fp8,3,0.03822560012340546
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,2,128,1,float16,float16,7,0.04552479982376099
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,2,128,1,float16,fp8,7,0.03836320042610168
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,fp8,255,0.010398399829864503
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,2,128,1,float16,float16,15,0.04560000002384186
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,2,128,1,float16,fp8,15,0.038473600149154664
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,2,128,1,float16,float16,31,0.0463344007730484
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,2,128,1,float16,float16,63,0.047875198721885684
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,2,128,1,float16,fp8,63,0.038920000195503235
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,4,128,1,float16,float16,1,0.09259679913520813
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,4,128,1,float16,fp8,1,0.08039839863777161
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,4,128,1,float16,float16,3,0.09262239933013916
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,4,128,1,float16,fp8,3,0.08022400140762329
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,4,128,1,float16,float16,7,0.09312000274658203
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,4,128,1,float16,fp8,7,0.08021759986877441
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,1,128,1,float16,float16,1,0.03916800022125244
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,1,128,1,float16,float16,3,0.03894400000572205
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,4,128,1,float16,float16,15,0.09434559941291809
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,4,128,1,float16,fp8,15,0.08220319747924805
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,1,128,1,float16,float16,7,0.03912160098552704
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,1,128,1,float16,fp8,15,0.033022400736808774
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,4,128,1,float16,float16,63,0.0949679970741272
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,4,128,1,float16,fp8,63,0.0826640009880066
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,8,128,1,float16,fp8,1,0.08424320220947265
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,8,128,1,float16,float16,1,0.13878719806671141
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,8,128,1,float16,float16,3,0.13913120031356813
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,8,128,1,float16,fp8,3,0.08333920240402222
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,8,128,1,float16,fp8,7,0.0839680016040802
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,8,128,1,float16,float16,15,0.139956796169281
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,8,128,1,float16,fp8,15,0.08457599878311158
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,8,128,1,float16,float16,31,0.14005919694900512
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,8,128,1,float16,fp8,31,0.08502560257911682
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,8,128,1,float16,fp8,63,0.08565599918365478
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,1,128,1,float16,float16,1,0.06872959733009339
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,1,128,1,float16,fp8,1,0.056848001480102536
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,2,128,1,float16,fp8,31,0.0385343998670578
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,1,128,1,float16,fp8,3,0.055878400802612305
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,1,128,1,float16,float16,3,0.06856480240821838
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,1,128,1,float16,float16,7,0.06824480295181275
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,1,128,1,float16,float16,15,0.06834080219268798
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,1,128,1,float16,fp8,15,0.05607680082321167
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,1,128,1,float16,float16,31,0.06874240040779114
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,1,128,1,float16,fp8,31,0.05613279938697815
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,2,128,1,float16,float16,1,0.08369759917259216
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,1,128,1,float16,fp8,7,0.05650879740715027
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,2,128,1,float16,float16,3,0.08432160019874572
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,2,128,1,float16,fp8,3,0.06898720264434814
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,2,128,1,float16,float16,7,0.08411359786987305
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,fp8,2047,0.012329600006341934
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,2,128,1,float16,fp8,7,0.06931040287017823
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,2,128,1,float16,float16,15,0.08473439812660218
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,4,128,1,float16,float16,31,0.09412320256233216
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,2,128,1,float16,fp8,15,0.0691536009311676
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,2,128,1,float16,float16,31,0.08465279936790467
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,2,128,1,float16,fp8,31,0.07021759748458863
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,4,128,1,float16,float16,1,0.17851040363311768
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,4,128,1,float16,fp8,3,0.1546223998069763
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,4,128,1,float16,float16,7,0.17848800420761107
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,4,128,1,float16,fp8,7,0.15494240522384645
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,8,128,1,float16,float16,7,0.14037920236587526
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,4,128,1,float16,float16,15,0.17861920595169067
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,4,128,1,float16,fp8,15,0.15503840446472167
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,4,128,1,float16,float16,31,0.18042399883270263
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,4,128,1,float16,fp8,31,0.15451359748840332
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,8,128,1,float16,fp8,1,0.1601696014404297
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,8,128,1,float16,float16,63,0.1404736042022705
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,8,128,1,float16,float16,3,0.2725327968597412
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,8,128,1,float16,fp8,3,0.1606719970703125
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,8,128,1,float16,fp8,7,0.1605471968650818
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,8,128,1,float16,float16,7,0.27258241176605225
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,8,128,1,float16,float16,15,0.27275838851928713
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,8,128,1,float16,fp8,15,0.16059199571609498
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,float16,1,0.01865600049495697
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,8,128,1,float16,float16,31,0.2725359916687012
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,fp8,1,0.01865759938955307
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,8,128,1,float16,fp8,31,0.1607792019844055
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,float16,3,0.018595199286937713
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,fp8,3,0.018564799427986146
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,float16,7,0.01858399957418442
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,fp8,7,0.018483200669288637
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,fp8,15,0.018596799671649934
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,float16,31,0.01863359957933426
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,fp8,31,0.01855359971523285
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,float16,63,0.01857919991016388
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,fp8,63,0.018598400056362152
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,float16,127,0.018612800538539885
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,fp8,127,0.018641600012779237
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,float16,255,0.018590399622917177
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,fp8,255,0.017467199265956877
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,float16,511,0.014897599816322327
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,2,128,1,float16,fp8,1,0.06952000260353089
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,float16,1023,0.014510400593280792
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,fp8,1023,0.012836800515651703
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,float16,2047,0.014697599411010741
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,fp8,2047,0.01456640064716339
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,float16,1,0.013393600285053254
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,fp8,1,0.012868799269199371
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,float16,3,0.012548799812793731
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,fp8,3,0.012508800625801087
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,4,128,1,float16,fp8,31,0.08123520016670227
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,float16,7,0.012547199428081513
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,fp8,7,0.01271200031042099
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,float16,15,0.012547199428081513
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,fp8,15,0.013075199723243714
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,4,128,1,float16,fp8,1,0.15448479652404784
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,4,128,1,float16,float16,3,0.18041280508041382
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,fp8,31,0.012868799269199371
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,float16,63,0.01327199935913086
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,fp8,63,0.012520000338554382
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,fp8,127,0.012563200294971466
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,float16,255,0.012697599828243256
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,fp8,255,0.012535999715328216
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,float16,511,0.01252799928188324
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,float16,31,0.012590399384498597
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,fp8,511,0.012470400333404541
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,float16,1023,0.012486399710178375
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,fp8,1023,0.012412799894809723
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,float16,2047,0.014524799585342408
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,fp8,2047,0.01247519999742508
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,float16,1,0.010406400263309478
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,float16,3,0.01053439974784851
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,fp8,3,0.010475199669599533
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,8,128,1,float16,float16,1,0.27162721157073977
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,float16,7,0.01058880016207695
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,fp8,7,0.010398399829864503
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,float16,15,0.01043199971318245
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,fp8,15,0.01048159971833229
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,float16,31,0.01048159971833229
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,fp8,31,0.010500799864530563
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,float16,63,0.01056160032749176
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,fp8,63,0.010555200278759003
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,fp8,127,0.01053759977221489
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,float16,255,0.010505600273609162
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,float16,511,0.010395199805498124
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,fp8,511,0.010401599854230881
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,float16,1023,0.012415999919176102
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,fp8,1023,0.010496000200510025
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,float16,2047,0.01459999978542328
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,float16,15,0.018727999925613404
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,float16,1,0.008379200100898742
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,fp8,1,0.00833280012011528
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,float16,3,0.008326400071382523
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,fp8,3,0.00833439975976944
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,float16,7,0.008388800173997879
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,fp8,7,0.008327999711036682
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,fp8,511,0.014659200608730317
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,float16,15,0.008414400368928909
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,fp8,15,0.00843999981880188
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,float16,31,0.008430399745702744
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,float16,63,0.008420799672603608
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,fp8,63,0.008371199667453765
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,float16,127,0.008441600203514098
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,fp8,127,0.008342400193214417
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,float16,255,0.008427199721336365
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,fp8,255,0.008382400125265121
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,float16,511,0.010398399829864503
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,fp8,511,0.008481600135564805
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,float16,1023,0.012444800138473511
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,fp8,1023,0.010476800054311753
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,float16,2047,0.026974400877952574
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,float16,127,0.014480000734329224
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,fp8,1,0.01459839940071106
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,float16,3,0.016201600432395935
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,fp8,3,0.014564800262451171
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,float16,7,0.014550399780273438
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,fp8,7,0.014558400213718414
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,fp8,1,0.010555200278759003
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,float16,15,0.014575999975204468
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,fp8,15,0.014603200554847717
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,float16,31,0.014580799639225006
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,fp8,31,0.01459839940071106
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,float16,63,0.014670400321483612
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,fp8,63,0.014572800695896148
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,float16,127,0.01043040007352829
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,fp8,127,0.014585599303245544
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,fp8,255,0.010553599894046783
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,fp8,255,0.014591999351978302
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,float16,511,0.014528000354766845
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,fp8,511,0.012539200484752655
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,float16,1023,0.01658560037612915
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,fp8,1023,0.014575999975204468
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,float16,2047,0.018697600066661834
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,fp8,2047,0.016654400527477263
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,fp8,2047,0.014472000300884247
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,float16,4095,0.03294720053672791
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,float16,1,0.010401599854230881
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,fp8,1,0.00941760018467903
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,float16,3,0.010364799946546554
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,fp8,3,0.00942239984869957
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,fp8,7,0.00942559987306595
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,float16,15,0.010480000078678131
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,fp8,15,0.010463999956846238
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,float16,7,0.010372799634933472
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,fp8,31,0.008422400057315826
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,float16,63,0.009758400171995163
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,fp8,63,0.008628799766302108
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,float16,127,0.010215999931097031
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,fp8,2047,0.01456640064716339
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,float16,1,0.014572800695896148
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,fp8,127,0.010384000092744827
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,fp8,255,0.01040479987859726
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,float16,255,0.010356800258159637
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,float16,511,0.01077599972486496
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,fp8,511,0.010427200049161912
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,float16,1023,0.014455999433994293
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,fp8,1023,0.012534399330615998
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,float16,2047,0.028884801268577575
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,fp8,2047,0.016873599588871004
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,float16,4095,0.047228801250457766
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,fp8,4095,0.03033120036125183
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,float16,1,0.01048320010304451
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,fp8,1,0.01037440001964569
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,float16,3,0.0104592002928257
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,fp8,3,0.010393600165843963
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,float16,7,0.010494399815797806
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,fp8,7,0.010356800258159637
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,float16,15,0.010473600029945374
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,fp8,15,0.010409600287675857
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,float16,31,0.010463999956846238
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,fp8,31,0.010395199805498124
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,float16,63,0.010576000064611435
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,fp8,63,0.010398399829864503
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,float16,127,0.010436800122261048
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,fp8,127,0.010420800000429154
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,fp8,255,0.010411199927330018
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,float16,511,0.01387999951839447
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,fp8,511,0.01239679977297783
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,float16,1023,0.028908801078796387
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,fp8,1023,0.014684799313545226
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,float16,31,0.010441599786281586
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,float16,2047,0.04742240011692047
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,fp8,31,0.010364799946546554
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,fp8,2047,0.029057601094245912
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,float16,1,0.012489599734544754
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,float16,4095,0.08400160074234009
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,fp8,1,0.010979200154542923
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,float16,3,0.01250080019235611
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,fp8,3,0.010518400371074677
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,fp8,7,0.010596799850463866
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,float16,15,0.0125231996178627
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,float16,31,0.012507200241088867
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,fp8,31,0.010473600029945374
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,float16,63,0.012457600235939026
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,fp8,63,0.010500799864530563
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,float16,127,0.012406399846076966
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,fp8,127,0.01051200032234192
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,float16,255,0.012488000094890594
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,fp8,255,0.010679999738931656
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,float16,511,0.028835201263427736
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,fp8,511,0.014577600359916686
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,float16,1023,0.047225600481033324
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,float16,127,0.015500800311565399
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,fp8,1023,0.029523199796676634
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,float16,2047,0.0841152012348175
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,fp8,2047,0.04916960000991821
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,float16,255,0.010518400371074677
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,float16,4095,0.1579983949661255
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,float16,1,0.012761600315570831
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,fp8,4095,0.08874239921569824
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,float16,3,0.01255200058221817
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,fp8,1,0.014457599818706512
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,fp8,3,0.012510399520397186
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,float16,7,0.012535999715328216
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,fp8,4095,0.022073599696159362
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,fp8,7,0.014567999541759491
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,float16,15,0.013803200423717498
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,fp8,15,0.012511999905109405
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,float16,31,0.014572800695896148
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,float16,63,0.014494399726390838
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,fp8,31,0.014537599682807923
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,fp8,63,0.012520000338554382
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,float16,127,0.01446239948272705
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,float16,255,0.014575999975204468
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,fp8,255,0.012544000148773193
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,float16,511,0.012462399899959564
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,fp8,511,0.012521600723266602
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,float16,1023,0.010438399761915207
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,fp8,1023,0.010476800054311753
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,fp8,2047,0.010516799986362457
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,float16,4095,0.012537600100040435
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,fp8,4095,0.012443199753761292
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,float16,1,0.010896000266075134
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,fp8,1,0.010454399883747101
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,float16,3,0.010513599961996078
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,float16,7,0.010492800176143647
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,fp8,4095,0.04929920136928558
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,fp8,7,0.010478399693965912
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,float16,15,0.010438399761915207
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,float16,7,0.012491200119256973
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,float16,31,0.010473600029945374
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,fp8,31,0.010540799796581268
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,float16,63,0.010411199927330018
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,fp8,15,0.010590399801731109
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,fp8,63,0.010463999956846238
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,float16,255,0.010713600367307664
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,fp8,127,0.010425599664449692
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,fp8,255,0.010475199669599533
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,float16,511,0.010438399761915207
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,fp8,511,0.010447999835014344
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,float16,1023,0.010443200170993806
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,fp8,1023,0.010499200224876404
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,float16,2047,0.010446400195360184
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,fp8,2047,0.010451199859380722
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,float16,4095,0.01250080019235611
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,fp8,4095,0.012518399953842163
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,float16,1,0.010444799810647965
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,fp8,1,0.010505600273609162
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,float16,255,0.014575999975204468
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,float16,3,0.010406400263309478
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,float16,7,0.01043199971318245
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,fp8,3,0.009692800045013428
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,fp8,7,0.008923199772834779
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,float16,15,0.010416000336408614
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,fp8,15,0.009459199756383896
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,float16,31,0.010396800190210342
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,float16,63,0.010380800068378448
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,fp8,31,0.009108799695968627
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,fp8,63,0.008470399677753449
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,float16,127,0.010382399708032609
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,fp8,127,0.009384000301361084
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,float16,255,0.010463999956846238
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,fp8,255,0.008993600308895112
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,float16,511,0.010395199805498124
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,fp8,127,0.012515200674533844
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,fp8,511,0.008776000142097473
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,float16,1023,0.010444799810647965
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,fp8,1023,0.009483200311660767
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,float16,2047,0.010417599976062775
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,float16,4095,0.012438400089740754
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,fp8,2047,0.0104592002928257
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,float16,2047,0.012433599680662155
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,float16,1,0.008425600081682205
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,fp8,1,0.008446399867534638
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,fp8,4095,0.011121600121259689
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,fp8,3,0.008353599905967712
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,float16,7,0.008529599756002426
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,float16,15,0.008420799672603608
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,fp8,15,0.0084927998483181
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,float16,31,0.008428800106048583
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,fp8,3,0.010524799674749374
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,fp8,31,0.008403199911117553
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,float16,63,0.008379200100898742
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,fp8,63,0.008455999940633774
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,float16,127,0.008417599648237229
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,fp8,127,0.008436799794435502
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,fp8,15,0.010478399693965912
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,fp8,511,0.008708799630403519
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,float16,1023,0.009006399661302567
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,float16,127,0.010480000078678131
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,fp8,1023,0.00846719965338707
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,fp8,2047,0.010438399761915207
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,float16,4095,0.012076800316572189
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,fp8,4095,0.010507199913263321
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,float16,1,0.014588800072669984
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,fp8,1,0.014483200013637542
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,float16,3,0.014555199444293976
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,fp8,3,0.014523200690746307
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,float16,7,0.014606399834156037
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,fp8,7,0.014478400349617004
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,float16,15,0.014590400457382201
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,fp8,15,0.014507199823856353
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,float16,31,0.014564800262451171
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,fp8,31,0.014499199390411378
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,float16,63,0.014569599926471711
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,fp8,63,0.014510400593280792
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,float16,127,0.014815999567508698
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,float16,255,0.014574399590492249
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,fp8,127,0.014483200013637542
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,fp8,255,0.014574399590492249
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,float16,511,0.0125231996178627
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,fp8,511,0.012540799379348756
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,float16,1023,0.012430399656295776
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,float16,2047,0.012449599802494049
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,fp8,1023,0.01141280010342598
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,fp8,2047,0.010622400045394897
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,float16,4095,0.012577599287033081
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,fp8,4095,0.012444800138473511
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,float16,1,0.011105599999427795
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,fp8,1,0.010523200035095215
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,float16,3,0.011028800159692764
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,fp8,3,0.010487999767065048
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,fp8,7,0.010521599650382995
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,float16,15,0.012121599912643433
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,fp8,15,0.010521599650382995
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,float16,31,0.01223360002040863
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,fp8,31,0.01053439974784851
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,float16,3,0.008414400368928909
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,float16,63,0.01144160032272339
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,fp8,7,0.008452799916267396
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,fp8,63,0.010494399815797806
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,float16,127,0.011006399989128113
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,fp8,127,0.010515200346708298
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,float16,255,0.010480000078678131
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,fp8,255,0.010478399693965912
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,fp8,511,0.010438399761915207
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,float16,255,0.008428800106048583
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,fp8,1023,0.010478399693965912
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,fp8,255,0.008419200032949447
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,float16,2047,0.010436800122261048
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,fp8,2047,0.010478399693965912
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,float16,2047,0.010382399708032609
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,float16,4095,0.012492799758911132
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,fp8,4095,0.012476799637079239
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,float16,1,0.010446400195360184
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,float16,3,0.010425599664449692
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,fp8,3,0.010375999659299851
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,float16,7,0.010436800122261048
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,fp8,7,0.010416000336408614
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,float16,15,0.010422399640083313
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,fp8,15,0.009268800169229508
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,float16,31,0.010416000336408614
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,fp8,31,0.009335999935865402
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,float16,63,0.010436800122261048
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,fp8,63,0.009857600182294845
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,float16,127,0.010440000146627427
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,fp8,127,0.010233599692583084
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,float16,255,0.010422399640083313
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,fp8,255,0.010468800365924836
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,float16,511,0.010468800365924836
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,float16,7,0.01170239970088005
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,float16,1023,0.010492800176143647
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,fp8,511,0.010579200088977813
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,fp8,1023,0.010492800176143647
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,fp8,2047,0.010520000010728836
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,float16,4095,0.01249919980764389
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,fp8,4095,0.012203200161457062
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,float16,1,0.008444800227880477
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,fp8,1,0.008416000008583068
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,float16,511,0.010921599715948105
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,float16,3,0.008743999898433686
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,fp8,3,0.008460800349712371
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,float16,7,0.008844800293445587
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,float16,1023,0.010516799986362457
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,fp8,7,0.008379200100898742
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,float16,15,0.008745600283145905
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,fp8,15,0.008476799726486206
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,float16,31,0.008433599770069123
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,float16,63,0.008436799794435502
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,fp8,63,0.00835679993033409
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,fp8,31,0.00841120034456253
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,float16,127,0.008473599702119828
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,float16,255,0.008449599891901017
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,fp8,255,0.008451200276613235
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,float16,511,0.008422400057315826
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,fp8,511,0.008451200276613235
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,float16,1023,0.00873119980096817
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,fp8,1,0.009752000123262406
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,fp8,1023,0.008478400111198426
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,float16,2047,0.010499200224876404
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,fp8,2047,0.010417599976062775
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,float16,4095,0.01252799928188324
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,fp8,4095,0.010527999699115753
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,float16,1,0.014499199390411378
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,fp8,1,0.014571200311183929
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,float16,3,0.014572800695896148
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,fp8,3,0.014496000111103058
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,float16,7,0.014588800072669984
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,fp8,7,0.014550399780273438
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,float16,15,0.014596800506114959
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,fp8,15,0.014523200690746307
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,float16,31,0.014569599926471711
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,fp8,31,0.014505599439144135
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,float16,63,0.014678399264812469
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,fp8,63,0.014574399590492249
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,float16,127,0.014588800072669984
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,fp8,127,0.014612799882888794
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,float16,255,0.01459999978542328
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,float16,511,0.014209599792957306
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,fp8,511,0.01244800016283989
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,float16,2047,0.014608000218868256
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,fp8,2047,0.013315199315547943
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,fp8,255,0.014523200690746307
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,float16,4095,0.018305599689483643
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,fp8,4095,0.016448000073432924
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,float16,1,0.011030399799346923
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,fp8,1,0.01040000021457672
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,float16,3,0.010467199981212616
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,fp8,3,0.01048159971833229
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,float16,7,0.010518400371074677
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,fp8,7,0.010486400127410889
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,float16,15,0.010497599840164185
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,fp8,15,0.010497599840164185
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,float16,31,0.010425599664449692
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,fp8,31,0.010745599865913391
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,float16,63,0.010387200117111205
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,fp8,63,0.01043040007352829
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,float16,127,0.010476800054311753
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,fp8,127,0.01050880029797554
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,float16,255,0.01048479974269867
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,fp8,255,0.01048479974269867
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,float16,511,0.010460799932479859
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,fp8,511,0.010440000146627427
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,float16,1023,0.012491200119256973
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,fp8,1023,0.01064160019159317
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,float16,2047,0.014611199498176575
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,fp8,2047,0.014499199390411378
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,float16,4095,0.029423999786376952
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,float16,1,0.008457600325345992
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,fp8,4095,0.018745599687099455
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,fp8,1,0.008446399867534638
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,float16,3,0.008430399745702744
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,float16,7,0.008366400003433227
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,fp8,7,0.00841279998421669
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,float16,15,0.008449599891901017
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,fp8,15,0.008318399637937545
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,float16,31,0.00843520015478134
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,fp8,31,0.008430399745702744
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,float16,63,0.008430399745702744
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,fp8,63,0.0083856001496315
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,float16,127,0.008369600027799606
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,fp8,127,0.008326400071382523
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,float16,255,0.00835999995470047
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,fp8,255,0.008406399935483932
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,float16,511,0.0104592002928257
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,fp8,1023,0.01249919980764389
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,fp8,511,0.00939679965376854
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,fp8,1023,0.010500799864530563
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,float16,2047,0.028164801001548768
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,fp8,2047,0.014523200690746307
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,float16,4095,0.04569759964942932
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,fp8,4095,0.028828799724578857
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,float16,2047,0.010655999928712846
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,float16,1,0.010364799946546554
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,fp8,1,0.008388800173997879
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,float16,3,0.010372799634933472
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,fp8,3,0.008396799862384795
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,fp8,7,0.008366400003433227
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,float16,15,0.010416000336408614
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,fp8,15,0.008371199667453765
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,float16,31,0.010401599854230881
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,fp8,31,0.008396799862384795
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,float16,511,0.008463999629020691
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,float16,63,0.010487999767065048
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,float16,127,0.010395199805498124
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,fp8,63,0.008455999940633774
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,fp8,127,0.008372800052165985
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,float16,511,0.012443199753761292
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,fp8,511,0.010391999781131745
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,float16,1023,0.028361600637435914
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,fp8,1023,0.014230400323867798
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,float16,2047,0.046419200301170346
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,fp8,2047,0.028806400299072266
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,float16,4095,0.08275520205497741
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,float16,1023,0.012521600723266602
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,fp8,4095,0.04768959879875183
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,float16,1,0.014584000408649444
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,fp8,1,0.01454080045223236
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,float16,3,0.014580799639225006
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,fp8,3,0.014542399346828461
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,float16,7,0.01465280055999756
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,fp8,7,0.01451680064201355
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,float16,15,0.014563199877738953
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,fp8,15,0.014547200500965118
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,fp8,31,0.01451359987258911
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,float16,31,0.014596800506114959
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,float16,63,0.014478400349617004
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,fp8,63,0.014591999351978302
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,float16,127,0.014587199687957764
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,fp8,127,0.014571200311183929
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,float16,255,0.014510400593280792
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,fp8,255,0.014555199444293976
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,float16,511,0.012544000148773193
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,fp8,511,0.012488000094890594
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,float16,1023,0.012432000041007996
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,float16,7,0.010375999659299851
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,float16,2047,0.011417599767446518
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,fp8,2047,0.010814400017261505
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,fp8,4095,0.012529599666595458
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,float16,1,0.010683199763298035
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,fp8,1,0.010462400317192078
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,float16,3,0.010468800365924836
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,fp8,3,0.010475199669599533
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,fp8,127,0.009143999963998794
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,float16,7,0.010475199669599533
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,float16,255,0.010678400099277497
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,float16,15,0.010441599786281586
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,fp8,15,0.010390400141477584
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,float16,31,0.010470400005578995
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,fp8,31,0.01050880029797554
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,fp8,63,0.010579200088977813
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,float16,127,0.010460799932479859
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,fp8,127,0.010505600273609162
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,float16,255,0.010467199981212616
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,fp8,255,0.010468800365924836
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,float16,511,0.010465600341558457
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,fp8,511,0.010499200224876404
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,float16,1023,0.010468800365924836
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,fp8,1023,0.010473600029945374
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,float16,2047,0.012492799758911132
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,fp8,2047,0.01159520000219345
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,float16,1023,0.012457600235939026
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,fp8,4095,0.012449599802494049
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,float16,1,0.01043360009789467
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,fp8,1,0.009393599629402161
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,float16,3,0.01037440001964569
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,fp8,3,0.009180799871683121
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,float16,7,0.010391999781131745
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,fp8,1023,0.010592000186443329
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,float16,4095,0.012443199753761292
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,float16,15,0.01040479987859726
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,fp8,15,0.008908800035715102
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,float16,31,0.010395199805498124
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,fp8,31,0.010401599854230881
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,float16,63,0.01040000021457672
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,fp8,63,0.010396800190210342
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,fp8,255,0.00846880003809929
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,fp8,7,0.010699199885129929
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,fp8,127,0.008470399677753449
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,float16,255,0.010443200170993806
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,float16,63,0.010478399693965912
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,float16,511,0.010440000146627427
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,fp8,511,0.009623999893665313
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,float16,1023,0.010449600219726563
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,fp8,1023,0.010371199995279311
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,float16,2047,0.010804799944162368
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,fp8,2047,0.010468800365924836
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,float16,4095,0.012622399628162384
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,fp8,4095,0.012433599680662155
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,float16,1,0.008376000076532364
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,float16,3,0.008374399691820144
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,fp8,3,0.008473599702119828
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,float16,7,0.00838399976491928
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,fp8,7,0.008460800349712371
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,float16,15,0.008508799970149994
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,fp8,15,0.008432000130414962
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,float16,4095,0.0131632000207901
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,float16,31,0.008417599648237229
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,fp8,31,0.008345600217580795
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,float16,63,0.00849440023303032
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,fp8,63,0.00833439975976944
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,float16,127,0.0083856001496315
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,fp8,127,0.008340799808502197
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,float16,255,0.00835999995470047
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,fp8,255,0.008343999832868576
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,float16,511,0.008364800363779068
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,fp8,511,0.008376000076532364
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,float16,1023,0.008875200152397155
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,fp8,1023,0.00846719965338707
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,float16,2047,0.010576000064611435
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,fp8,2047,0.01040000021457672
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,float16,4095,0.012462399899959564
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,fp8,4095,0.010518400371074677
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,fp8,7,0.009455999732017517
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,fp8,1,0.010115200281143188
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,float16,3,0.010471999645233154
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,fp8,3,0.010315199941396713
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,float16,7,0.010497599840164185
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,float16,15,0.010502400249242783
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,fp8,15,0.010260800272226334
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,float16,31,0.010502400249242783
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,fp8,31,0.008436799794435502
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,float16,63,0.01045600026845932
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,fp8,63,0.008436799794435502
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,fp8,255,0.010377600044012069
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,fp8,127,0.0104032002389431
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,float16,255,0.010424000024795533
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,fp8,255,0.00894080027937889
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,float16,511,0.012700800597667695
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,fp8,511,0.01043040007352829
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,fp8,1,0.008340799808502197
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,fp8,1023,0.01252640038728714
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,float16,2047,0.029707199335098265
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,fp8,2047,0.01847680062055588
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,float16,1,0.012529599666595458
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,float16,3,0.012529599666595458
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,fp8,3,0.012483199685811996
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,float16,7,0.01244639977812767
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,fp8,7,0.012398400157690049
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,float16,15,0.012548799812793731
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,fp8,15,0.012428800016641617
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,float16,31,0.012574400007724761
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,fp8,31,0.012449599802494049
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,float16,63,0.012544000148773193
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,fp8,63,0.012441600114107132
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,float16,127,0.012438400089740754
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,float16,1,0.010452800244092942
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,fp8,127,0.012436799705028534
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,float16,255,0.012622399628162384
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,fp8,255,0.012388800084590913
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,float16,511,0.016540800034999848
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,fp8,7,0.010396800190210342
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,float16,1023,0.029091200232505797
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,fp8,1023,0.018667200207710268
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,float16,2047,0.04924960136413574
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,float16,1,0.01451680064201355
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,fp8,3,0.008380799740552902
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,fp8,1,0.012491200119256973
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,float16,3,0.014579200744628906
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,fp8,3,0.012460800260305405
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,float16,127,0.010427200049161912
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,float16,7,0.014572800695896148
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,fp8,7,0.012462399899959564
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,fp8,15,0.01255200058221817
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,float16,31,0.014550399780273438
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,float16,15,0.014587199687957764
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,fp8,31,0.012539200484752655
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,float16,63,0.014519999921321868
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,fp8,63,0.012555199861526489
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,float16,127,0.014604799449443817
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,fp8,127,0.012566399574279786
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,float16,1023,0.014638400077819825
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,fp8,255,0.012520000338554382
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,float16,511,0.02894560098648071
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,float16,1023,0.0482448011636734
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,fp8,1023,0.030649599432945252
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,fp8,1,0.012454400211572647
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,float16,2047,0.08536319732666016
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,fp8,2047,0.04983200132846832
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,float16,1,0.016641600430011748
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,fp8,1,0.014579200744628906
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,fp8,3,0.01459999978542328
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,float16,7,0.016673600673675536
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,fp8,7,0.014563199877738953
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,float16,15,0.01661120057106018
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,fp8,15,0.014524799585342408
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,float16,31,0.016652800142765045
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,fp8,31,0.014612799882888794
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,float16,63,0.01775680035352707
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,fp8,63,0.014535999298095703
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,float16,127,0.018300800025463103
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,fp8,511,0.014480000734329224
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,fp8,127,0.014606399834156037
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,float16,255,0.028673601150512696
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,fp8,255,0.01510400027036667
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,fp8,2047,0.03277119994163513
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,float16,511,0.047200000286102294
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,fp8,511,0.029097598791122437
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,float16,127,0.01037440001964569
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,float16,1023,0.08414400219917298
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,fp8,1023,0.049316799640655516
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,float16,1,0.012428800016641617
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,float16,2047,0.15823520421981813
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,float16,3,0.012494400143623352
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,float16,7,0.012444800138473511
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,fp8,3,0.012427199631929398
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,fp8,7,0.01244800016283989
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,fp8,15,0.012508800625801087
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,float16,255,0.01451680064201355
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,float16,31,0.01250080019235611
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,fp8,31,0.012350399792194367
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,float16,63,0.012516799569129943
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,fp8,63,0.012027200311422348
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,float16,127,0.012544000148773193
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,fp8,511,0.0165583997964859
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,fp8,127,0.012470400333404541
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,float16,255,0.012478400021791458
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,fp8,255,0.012515200674533844
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,float16,511,0.016612799465656282
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,fp8,511,0.01451359987258911
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,fp8,1,0.016521599888801575
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,float16,3,0.018580800294876097
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,float16,7,0.018636800348758698
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,fp8,3,0.016627199947834015
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,fp8,7,0.016513599455356597
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,float16,15,0.0186256006360054
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,fp8,15,0.016569599509239197
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,fp8,31,0.016571199893951415
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,float16,3,0.016631999611854555
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,float16,63,0.018688000738620758
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,fp8,63,0.016548800468444824
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,float16,127,0.018568000197410582
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,fp8,127,0.01658560037612915
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,fp8,255,0.01661120057106018
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,float16,255,0.01881760060787201
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,float16,511,0.03160800039768219
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,fp8,511,0.02102400064468384
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,float16,1,0.022793599963188173
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,fp8,1,0.017374399304389953
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,float16,3,0.02279040068387985
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,fp8,3,0.016872000694274903
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,float16,7,0.022745600342750548
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,fp8,7,0.016961599886417388
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,float16,15,0.022779199481010436
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,fp8,15,0.017102399468421937
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,float16,31,0.022771200537681578
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,fp8,31,0.016833600401878358
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,fp8,63,0.016945600509643555
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,float16,63,0.022731199860572815
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,float16,127,0.022705599665641785
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,fp8,127,0.01716320067644119
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,float16,255,0.029075199365615846
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,fp8,255,0.016672000288963318
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,float16,511,0.0485615998506546
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,fp8,511,0.03094240128993988
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,float16,1,0.0269679993391037
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,fp8,1,0.023183999955654143
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,fp8,3,0.022675199806690215
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,float16,7,0.028302401304244995
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,float16,3,0.027238398790359497
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,fp8,7,0.023179200291633607
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,float16,15,0.027063998579978942
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,fp8,15,0.02274399995803833
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,float16,31,0.027820798754692077
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,fp8,31,0.022763200104236603
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,float16,63,0.02804960012435913
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,fp8,63,0.02338559925556183
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,float16,127,0.030875200033187868
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,fp8,127,0.023175999522209167
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,float16,255,0.04753600060939789
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,fp8,2047,0.08821600079536437
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,fp8,255,0.03006559908390045
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,fp8,1,0.011929599940776825
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,float16,1,0.018675200641155243
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,fp8,511,0.049292799830436704
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,float16,511,0.08463360071182251
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,float16,15,0.012532800436019897
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,float16,3,0.018688000738620758
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,fp8,3,0.01653439998626709
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,float16,7,0.018598400056362152
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,float16,15,0.018592000007629395
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,fp8,15,0.016545599699020384
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,float16,31,0.018617600202560425
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,fp8,31,0.016620799899101257
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,fp8,63,0.01653279960155487
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,float16,127,0.0186256006360054
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,fp8,127,0.016543999314308167
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,float16,255,0.01863359957933426
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,fp8,255,0.01656160056591034
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,float16,1,0.018676799535751343
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,float16,1,0.028910401463508605
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,fp8,1,0.026668798923492432
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,float16,3,0.0289247989654541
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,float16,7,0.0288239985704422
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,fp8,3,0.02690880000591278
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,fp8,7,0.026814401149749756
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,float16,15,0.028833600878715514
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,float16,31,0.02897599935531616
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,fp8,31,0.026790401339530943
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,float16,31,0.01855199933052063
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,float16,63,0.029204800724983215
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,fp8,63,0.0267984002828598
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,float16,127,0.030347201228141784
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,float16,255,0.033195200562477115
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,fp8,255,0.02727839946746826
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,fp8,127,0.026969599723815917
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,float16,1,0.0402864009141922
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,fp8,1,0.02688480019569397
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,float16,3,0.03951039910316467
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,float16,7,0.03937920033931732
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,fp8,7,0.02683840095996857
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,float16,15,0.039238399267196654
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,fp8,15,0.026899200677871705
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,float16,31,0.03933759927749634
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,fp8,31,0.027140799164772033
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,float16,63,0.03932160139083862
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,fp8,63,0.02701919972896576
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,float16,127,0.0410975992679596
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,fp8,127,0.02717120051383972
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,float16,255,0.049384000897407535
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,fp8,255,0.03140319883823395
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,float16,1,0.047417598962783816
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,fp8,1,0.03934719860553741
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,float16,3,0.047619199752807616
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,fp8,3,0.03922399878501892
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,float16,7,0.04775519967079163
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,fp8,7,0.03930560052394867
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,float16,15,0.04894559979438782
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,fp8,15,0.03931359946727753
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,float16,31,0.049188798666000365
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,fp8,1,0.01650400012731552
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,float16,63,0.04948480129241943
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,fp8,63,0.04110879898071289
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,fp8,7,0.01655679941177368
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,float16,127,0.05287039875984192
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,float16,63,0.01847199946641922
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,fp8,255,0.05154399871826172
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,float16,1,0.014480000734329224
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,fp8,1,0.014604799449443817
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,fp8,3,0.014494399726390838
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,float16,7,0.014572800695896148
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,fp8,7,0.014535999298095703
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,float16,15,0.01446560025215149
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,fp8,15,0.026785600185394286
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,fp8,15,0.014460800588130951
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,float16,31,0.014550399780273438
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,fp8,31,0.014526399970054626
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,float16,63,0.01456640064716339
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,fp8,63,0.01446560025215149
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,float16,127,0.014582400023937226
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,fp8,127,0.014481599628925323
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,float16,255,0.014496000111103058
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,fp8,255,0.01456640064716339
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,float16,511,0.012505599856376648
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,fp8,511,0.012464000284671784
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,float16,1023,0.012503999471664428
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,fp8,1023,0.011825600266456604
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,fp8,3,0.026881599426269533
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,float16,2047,0.012491200119256973
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,fp8,2047,0.012510399520397186
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,float16,1,0.01101119965314865
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,fp8,1,0.01072160005569458
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,float16,3,0.01132960021495819
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,fp8,3,0.010491199791431427
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,fp8,7,0.010444799810647965
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,float16,15,0.01045759990811348
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,fp8,15,0.0104592002928257
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,float16,31,0.010497599840164185
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,fp8,31,0.03926079869270325
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,fp8,31,0.01045600026845932
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,float16,63,0.010438399761915207
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,fp8,127,0.04246560037136078
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,fp8,63,0.010579200088977813
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,float16,255,0.0862559974193573
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,float16,127,0.011308799684047698
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,fp8,127,0.010622400045394897
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,float16,3,0.014539200067520141
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,fp8,255,0.010473600029945374
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,float16,511,0.010518400371074677
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,fp8,511,0.010521599650382995
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,float16,1023,0.010475199669599533
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,fp8,1023,0.010424000024795533
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,float16,2047,0.012432000041007996
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,fp8,2047,0.012559999525547028
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,float16,4095,0.01451359987258911
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,fp8,4095,0.01250080019235611
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,float16,3,0.00846560001373291
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,fp8,1,0.009419199824333192
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,float16,7,0.009435199946165086
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,fp8,7,0.009200000017881394
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,float16,15,0.009699200093746186
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,fp8,15,0.00840959995985031
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,float16,31,0.008419200032949447
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,fp8,31,0.008470399677753449
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,float16,4095,0.01448799967765808
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,float16,63,0.008451200276613235
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,fp8,4095,0.012491200119256973
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,float16,127,0.009323199838399887
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,fp8,63,0.0086496002972126
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,fp8,127,0.009464000165462495
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,float16,255,0.008587200194597244
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,float16,7,0.010500799864530563
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,fp8,255,0.008753599971532822
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,float16,511,0.008454400300979614
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,fp8,511,0.008363199979066848
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,float16,1023,0.01037440001964569
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,fp8,1023,0.009867200255393982
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,float16,2047,0.010491199791431427
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,float16,1,0.008399999886751174
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,fp8,2047,0.010451199859380722
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,float16,4095,0.012486399710178375
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,fp8,1,0.008446399867534638
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,fp8,3,0.008348800241947174
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,float16,7,0.008432000130414962
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,fp8,7,0.008379200100898742
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,float16,15,0.008441600203514098
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,fp8,15,0.008454400300979614
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,fp8,31,0.008420799672603608
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,float16,63,0.00836160033941269
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,fp8,63,0.008654399961233138
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,float16,127,0.00835679993033409
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,fp8,127,0.008367999643087386
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,float16,255,0.008427199721336365
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,fp8,255,0.008417599648237229
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,float16,511,0.008376000076532364
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,fp8,511,0.008427199721336365
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,float16,1023,0.010391999781131745
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,fp8,1023,0.01029599979519844
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,float16,255,0.0105103999376297
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,float16,2047,0.012444800138473511
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,fp8,2047,0.010436800122261048
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,fp8,4095,0.013366399705410004
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,float16,4095,0.028628799319267272
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,float16,1,0.009804800152778625
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,1,128,1,float16,float16,1,0.026910400390625
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,1,128,1,float16,fp8,1,0.02475520074367523
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,1,128,1,float16,float16,3,0.02697120010852814
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,fp8,3,0.008475200086832047
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,1,128,1,float16,float16,7,0.027620801329612733
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,1,128,1,float16,fp8,7,0.024726399779319765
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,1,128,1,float16,fp8,15,0.024721600115299225
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,1,128,1,float16,float16,15,0.027513599395751952
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,1,128,1,float16,float16,31,0.026873600482940675
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,1,128,1,float16,fp8,31,0.0247296005487442
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,1,128,1,float16,float16,63,0.027492800354957582
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,1,128,1,float16,float16,127,0.028863999247550964
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,1,128,1,float16,fp8,63,0.024743999540805816
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,1,128,1,float16,fp8,127,0.024751999974250795
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,2,128,1,float16,float16,1,0.051046401262283325
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,2,128,1,float16,fp8,1,0.04512960016727448
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,2,128,1,float16,fp8,3,0.04529759883880615
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,2,128,1,float16,float16,7,0.05105599761009216
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,2,128,1,float16,fp8,7,0.04516479969024658
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,2,128,1,float16,float16,15,0.05148959755897522
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,2,128,1,float16,fp8,15,0.04522239863872528
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,2,128,1,float16,fp8,31,0.04524320065975189
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,2,128,1,float16,float16,31,0.051256000995635986
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,2,128,1,float16,fp8,63,0.0452320009469986
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,2,128,1,float16,float16,127,0.05343359708786011
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,2,128,1,float16,fp8,127,0.045649600028991696
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,fp8,4095,0.012465599924325943
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,float16,3,0.008422400057315826
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,4,128,1,float16,fp8,1,0.04675999879837036
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,4,128,1,float16,float16,3,0.07247520089149476
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,4,128,1,float16,fp8,3,0.047095999121665955
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,4,128,1,float16,fp8,7,0.04660640060901642
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,4,128,1,float16,float16,7,0.07267360091209411
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,float16,31,0.008387199789285659
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,4,128,1,float16,fp8,15,0.04644320011138916
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,4,128,1,float16,float16,31,0.07411519885063171
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,4,128,1,float16,fp8,31,0.04676159918308258
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,4,128,1,float16,float16,63,0.07461599707603454
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,4,128,1,float16,fp8,63,0.04735519886016846
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,4,128,1,float16,float16,127,0.07438880205154419
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,4,128,1,float16,fp8,127,0.04813919961452484
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,8,128,1,float16,float16,1,0.08796799778938294
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,8,128,1,float16,fp8,1,0.07237120270729065
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,8,128,1,float16,float16,3,0.08804000020027161
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,8,128,1,float16,fp8,3,0.07234879732131957
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,8,128,1,float16,float16,7,0.08842080235481262
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,8,128,1,float16,fp8,7,0.07243520021438599
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,8,128,1,float16,float16,15,0.08919039964675904
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,8,128,1,float16,fp8,15,0.07252479791641235
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,8,128,1,float16,float16,31,0.08921279907226562
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,8,128,1,float16,fp8,31,0.07364959716796875
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,8,128,1,float16,float16,63,0.08910880088806153
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,8,128,1,float16,fp8,63,0.07408639788627625
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,1,128,1,float16,fp8,3,0.024750399589538574
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,8,128,1,float16,float16,127,0.09574080109596253
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,1,128,1,float16,float16,1,0.047040000557899475
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,8,128,1,float16,fp8,127,0.07512320280075073
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,1,128,1,float16,fp8,1,0.04122720062732697
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,1,128,1,float16,float16,3,0.04686880111694336
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,1,128,1,float16,fp8,3,0.04121600091457367
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,1,128,1,float16,fp8,7,0.04121919870376587
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,1,128,1,float16,float16,15,0.04738079905509949
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,1,128,1,float16,fp8,15,0.04120000004768372
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,1,128,1,float16,float16,31,0.04844320118427277
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,1,128,1,float16,fp8,31,0.04115520119667053
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,1,128,1,float16,float16,63,0.049267199635505673
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,1,128,1,float16,fp8,63,0.04114719927310943
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,2,128,1,float16,float16,63,0.053011202812194826
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,2,128,1,float16,float16,1,0.09465759992599487
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,2,128,1,float16,fp8,1,0.08217920064926147
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,2,128,1,float16,float16,3,0.0940496027469635
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,2,128,1,float16,float16,7,0.09432799816131592
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,4,128,1,float16,float16,1,0.07307360172271729
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,2,128,1,float16,fp8,7,0.08229119777679443
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,2,128,1,float16,float16,15,0.09456480145454407
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,2,128,1,float16,fp8,15,0.08228800296783448
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,2,128,1,float16,float16,31,0.09465919733047486
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,2,128,1,float16,fp8,31,0.08309280276298522
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,2,128,1,float16,float16,63,0.09494720101356506
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,4,128,1,float16,float16,15,0.07321439981460572
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,4,128,1,float16,float16,1,0.1394592046737671
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,4,128,1,float16,fp8,1,0.08484479784965515
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,4,128,1,float16,float16,3,0.13961440324783325
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,4,128,1,float16,fp8,3,0.0849888026714325
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,4,128,1,float16,float16,7,0.1398368000984192
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,4,128,1,float16,fp8,7,0.08501759767532349
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,4,128,1,float16,float16,15,0.13954880237579345
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,4,128,1,float16,fp8,15,0.08593279719352723
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,4,128,1,float16,float16,31,0.13984320163726807
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,4,128,1,float16,fp8,31,0.08640000224113464
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,4,128,1,float16,float16,63,0.14016640186309814
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,4,128,1,float16,fp8,63,0.08654879927635192
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,8,128,1,float16,float16,1,0.1684272050857544
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,8,128,1,float16,fp8,1,0.13671679496765138
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,8,128,1,float16,float16,3,0.16849759817123414
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,8,128,1,float16,fp8,3,0.1365023970603943
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,8,128,1,float16,float16,7,0.1687888026237488
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,8,128,1,float16,fp8,7,0.13755199909210206
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,8,128,1,float16,fp8,15,0.13782559633255004
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,8,128,1,float16,float16,15,0.1683776021003723
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,8,128,1,float16,float16,31,0.16855520009994507
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,8,128,1,float16,fp8,31,0.13803520202636718
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,8,128,1,float16,float16,63,0.1684720039367676
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,fp8,1,0.01451359987258911
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,8,128,1,float16,fp8,63,0.13840960264205932
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,float16,3,0.014505599439144135
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,1,128,1,float16,float16,7,0.046928000450134275
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,float16,7,0.01451359987258911
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,fp8,7,0.01451839953660965
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,float16,15,0.014635199308395385
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,2,128,1,float16,float16,3,0.05075200200080872
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,float16,31,0.014617599546909332
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,fp8,31,0.014545600116252898
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,float16,63,0.014556799829006196
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,float16,127,0.014588800072669984
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,fp8,127,0.014550399780273438
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,float16,255,0.014577600359916686
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,fp8,255,0.014582400023937226
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,float16,511,0.012555199861526489
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,float16,1023,0.012484800070524216
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,2,128,1,float16,fp8,3,0.08330399990081787
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,fp8,1023,0.012478400021791458
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,float16,2047,0.012532800436019897
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,fp8,2047,0.012444800138473511
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,float16,4095,0.01496479958295822
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,fp8,4095,0.014497600495815277
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,float16,1,0.010419200360774993
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,fp8,1,0.010467199981212616
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,float16,3,0.010417599976062775
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,fp8,3,0.010411199927330018
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,float16,7,0.010489600151777268
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,fp8,7,0.010414399951696397
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,float16,15,0.010419200360774993
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,fp8,15,0.010416000336408614
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,float16,31,0.010492800176143647
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,fp8,31,0.010547199845314026
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,float16,63,0.010419200360774993
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,fp8,63,0.010412800312042236
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,float16,127,0.01048159971833229
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,fp8,127,0.010475199669599533
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,float16,255,0.010496000200510025
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,fp8,255,0.01040479987859726
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,float16,511,0.010416000336408614
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,fp8,511,0.01045759990811348
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,float16,1023,0.010505600273609162
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,fp8,1023,0.01040479987859726
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,float16,2047,0.012441600114107132
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,fp8,2047,0.010523200035095215
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,float16,4095,0.014603200554847717
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,float16,1,0.01448799967765808
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,fp8,4095,0.013347199559211731
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,float16,1,0.010391999781131745
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,fp8,1,0.0087567999958992
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,float16,3,0.010387200117111205
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,fp8,3,0.01449279934167862
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,fp8,3,0.0087567999958992
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,float16,7,0.010396800190210342
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,fp8,15,0.014534400403499603
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,fp8,7,0.01037440001964569
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,float16,15,0.010447999835014344
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,fp8,15,0.010398399829864503
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,float16,31,0.010351999849081039
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,fp8,63,0.014507199823856353
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,fp8,63,0.010393600165843963
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,fp8,127,0.009471999853849411
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,float16,127,0.01048159971833229
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,float16,255,0.010344000160694122
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,fp8,255,0.009012799710035324
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,float16,511,0.010449600219726563
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,fp8,511,0.008436799794435502
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,fp8,511,0.012521600723266602
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,float16,1023,0.010526400059461594
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,fp8,1023,0.010425599664449692
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,float16,2047,0.012449599802494049
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,float16,4095,0.028825598955154418
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,fp8,4095,0.01462559998035431
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,float16,3,0.008379200100898742
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,fp8,3,0.008399999886751174
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,float16,7,0.008448000252246856
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,fp8,7,0.008398400247097015
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,2,128,1,float16,fp8,63,0.08422399759292602
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,float16,15,0.008337599784135818
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,fp8,15,0.00838399976491928
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,float16,31,0.008422400057315826
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,fp8,31,0.008353599905967712
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,float16,63,0.008515200018882752
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,fp8,63,0.008388800173997879
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,float16,127,0.008377599716186523
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,fp8,127,0.008347199857234954
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,float16,255,0.008403199911117553
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,fp8,255,0.008348800241947174
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,fp8,511,0.008382400125265121
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,float16,1023,0.010835199803113937
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,fp8,1023,0.010387200117111205
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,float16,2047,0.02683520019054413
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,fp8,2047,0.0124719999730587
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,float16,4095,0.04540160000324249
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,fp8,4095,0.02787039875984192
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,float16,1,0.010527999699115753
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,fp8,1,0.010524799674749374
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,float16,3,0.011337599903345107
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,fp8,3,0.01051200032234192
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,float16,7,0.010604800283908844
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,fp8,7,0.010475199669599533
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,float16,15,0.010891199856996537
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,fp8,31,0.010369600355625152
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,float16,63,0.010414399951696397
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,float16,31,0.011115200072526931
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,fp8,31,0.010520000010728836
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,float16,63,0.010475199669599533
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,fp8,63,0.010604800283908844
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,float16,127,0.010491199791431427
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,fp8,127,0.010998400300741196
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,float16,255,0.010480000078678131
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,fp8,255,0.01053600013256073
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,float16,511,0.010371199995279311
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,float16,1,0.008337599784135818
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,fp8,511,0.010473600029945374
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,fp8,1,0.0083856001496315
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,float16,1023,0.012577599287033081
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,fp8,1023,0.01048479974269867
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,float16,2047,0.014603200554847717
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,fp8,2047,0.01451359987258911
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,fp8,4095,0.018668800592422485
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,float16,4095,0.029942399263381957
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,fp8,8191,0.031174400448799135
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,float16,1,0.008416000008583068
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,float16,3,0.008367999643087386
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,float16,8191,0.04722720086574554
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,float16,7,0.008423999696969987
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,fp8,3,0.008441600203514098
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,fp8,7,0.008364800363779068
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,float16,15,0.008419200032949447
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,float16,31,0.008446399867534638
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,fp8,31,0.008399999886751174
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,fp8,63,0.00843520015478134
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,float16,127,0.00846880003809929
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,fp8,127,0.008406399935483932
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,float16,255,0.008472000062465668
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,fp8,255,0.00846719965338707
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,float16,511,0.01040000021457672
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,fp8,511,0.009991999715566635
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,float16,1023,0.012532800436019897
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,fp8,1023,0.0105103999376297
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,float16,2047,0.027401599287986755
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,fp8,2047,0.014476799964904785
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,float16,4095,0.045286399126052854
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,fp8,4095,0.028910401463508605
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,float16,8191,0.08187999725341796
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,fp8,8191,0.04748159945011139
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,float16,1,0.009694399684667588
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,fp8,15,0.010579200088977813
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,float16,3,0.010440000146627427
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,fp8,3,0.008364800363779068
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,float16,7,0.009886399656534196
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,fp8,7,0.008454400300979614
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,float16,15,0.010409600287675857
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,fp8,15,0.008401600271463394
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,float16,31,0.010358399897813796
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,fp8,31,0.008481600135564805
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,float16,63,0.01072160005569458
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,fp8,1,0.008420799672603608
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,float16,127,0.010382399708032609
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,fp8,15,0.00835999995470047
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,float16,63,0.008355200290679932
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,float16,255,0.010390400141477584
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,fp8,255,0.008406399935483932
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,float16,511,0.012462399899959564
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,fp8,511,0.010497599840164185
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,float16,1023,0.027667200565338133
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,fp8,1023,0.013977600634098053
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,float16,511,0.01048320010304451
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,fp8,2047,0.028806400299072266
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,float16,4095,0.08225600123405456
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,fp8,4095,0.0473471999168396
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,float16,8191,0.15643999576568604
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,float16,1,0.012492799758911132
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,fp8,8191,0.08543840050697327
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,fp8,1,0.011430399864912033
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,fp8,3,0.010468800365924836
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,float16,3,0.01244800016283989
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,float16,7,0.012438400089740754
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,fp8,7,0.010480000078678131
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,float16,15,0.01247360035777092
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,fp8,15,0.010497599840164185
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,float16,31,0.0124208003282547
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,fp8,31,0.010507199913263321
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,float16,63,0.01250080019235611
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,fp8,63,0.010505600273609162
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,float16,127,0.012518399953842163
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,fp8,127,0.010497599840164185
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,float16,255,0.012451200187206269
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,fp8,255,0.010443200170993806
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,float16,511,0.028395199775695802
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,fp8,511,0.014467200636863709
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,float16,1023,0.046982398629188536
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,float16,2047,0.0830560028553009
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,fp8,2047,0.04831359982490539
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,float16,4095,0.15776000022888184
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,fp8,4095,0.08807680010795593
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,fp8,1,0.00843520015478134
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,float16,1,0.010503999888896942
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,fp8,1,0.010513599961996078
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,float16,3,0.010502400249242783
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,fp8,8191,0.16638720035552979
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,fp8,3,0.010480000078678131
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,float16,8191,0.30581440925598147
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,fp8,2047,0.012535999715328216
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,fp8,7,0.010516799986362457
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,float16,7,0.010462400317192078
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,float16,31,0.010441599786281586
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,fp8,15,0.010505600273609162
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,fp8,31,0.01043199971318245
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,float16,63,0.010487999767065048
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,float16,127,0.010414399951696397
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,fp8,127,0.010428799688816071
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,float16,255,0.010417599976062775
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,fp8,255,0.010424000024795533
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,float16,511,0.010436800122261048
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,fp8,63,0.008595199882984161
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,fp8,511,0.010385599732398988
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,float16,1023,0.008979199826717377
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,fp8,1023,0.010424000024795533
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,float16,2047,0.01050880029797554
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,fp8,2047,0.010470400005578995
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,fp8,127,0.008430399745702744
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,float16,8191,0.014539200067520141
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,fp8,8191,0.012703999876976013
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,float16,1,0.010422399640083313
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,fp8,1,0.008428800106048583
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,float16,3,0.010460799932479859
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,float16,2047,0.04723039865493774
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,float16,7,0.010435199737548828
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,fp8,7,0.009308800101280212
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,float16,15,0.010470400005578995
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,fp8,15,0.009497600048780442
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,float16,31,0.009833600372076035
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,fp8,31,0.008993600308895112
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,float16,63,0.010454399883747101
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,fp8,63,0.009080000221729279
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,float16,127,0.010401599854230881
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,fp8,127,0.009532800316810608
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,float16,255,0.010454399883747101
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,fp8,255,0.01043199971318245
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,float16,511,0.010425599664449692
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,fp8,1023,0.02903839945793152
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,fp8,511,0.0088128000497818
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,fp8,1023,0.010424000024795533
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,float16,2047,0.010419200360774993
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,fp8,2047,0.01045600026845932
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,float16,4095,0.01106560006737709
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,fp8,4095,0.010451199859380722
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,float16,8191,0.012988799810409546
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,fp8,8191,0.012702399492263794
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,float16,1,0.008414400368928909
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,fp8,1,0.008398400247097015
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,float16,3,0.008441600203514098
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,fp8,3,0.008406399935483932
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,float16,7,0.008380799740552902
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,float16,15,0.010492800176143647
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,fp8,7,0.008459199965000153
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,fp8,63,0.010489600151777268
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,float16,15,0.008423999696969987
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,fp8,15,0.00840959995985031
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,fp8,31,0.008454400300979614
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,float16,63,0.008459199965000153
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,fp8,63,0.008452799916267396
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,float16,127,0.00851999968290329
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,fp8,127,0.008423999696969987
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,float16,255,0.008433599770069123
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,fp8,255,0.008444800227880477
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,float16,4095,0.012432000041007996
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,float16,511,0.00835999995470047
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,fp8,511,0.008428800106048583
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,float16,1023,0.008584000170230865
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,float16,2047,0.010390400141477584
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,fp8,1023,0.008401600271463394
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,fp8,2047,0.008372800052165985
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,float16,4095,0.010463999956846238
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,fp8,4095,0.010446400195360184
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,float16,8191,0.012639999389648438
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,fp8,8191,0.012510399520397186
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,fp8,3,0.008446399867534638
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,float16,1,0.008769600093364716
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,fp8,1,0.00843520015478134
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,fp8,7,0.008422400057315826
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,float16,15,0.008372800052165985
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,float16,7,0.008390399813652038
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,fp8,15,0.008489599823951722
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,float16,31,0.008348800241947174
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,fp8,31,0.00843520015478134
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,float16,63,0.008457600325345992
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,float16,127,0.00835999995470047
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,fp8,127,0.008420799672603608
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,float16,255,0.008433599770069123
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,fp8,3,0.00838399976491928
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,fp8,255,0.008395200222730636
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,float16,511,0.00843520015478134
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,float16,1023,0.008422400057315826
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,fp8,511,0.00844319984316826
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,fp8,1023,0.008398400247097015
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,float16,2047,0.010419200360774993
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,fp8,2047,0.010417599976062775
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,float16,4095,0.011398400366306304
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,fp8,4095,0.010443200170993806
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,float16,8191,0.01369439959526062
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,fp8,8191,0.012511999905109405
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,float16,1,0.01045759990811348
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,fp8,1,0.010503999888896942
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,float16,3,0.010406400263309478
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,fp8,3,0.010523200035095215
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,float16,7,0.0105103999376297
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,fp8,7,0.010487999767065048
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,float16,15,0.010520000010728836
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,fp8,15,0.010526400059461594
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,float16,31,0.010452800244092942
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,fp8,31,0.010500799864530563
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,float16,63,0.01058880016207695
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,float16,1023,0.010356800258159637
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,fp8,63,0.010390400141477584
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,float16,127,0.010452800244092942
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,fp8,127,0.010520000010728836
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,float16,255,0.010460799932479859
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,fp8,255,0.010427200049161912
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,float16,511,0.010454399883747101
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,float16,1023,0.010407999902963639
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,fp8,511,0.01043040007352829
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,fp8,1023,0.010406400263309478
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,float16,2047,0.010569600015878677
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,fp8,2047,0.010449600219726563
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,float16,8191,0.014575999975204468
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,fp8,8191,0.014505599439144135
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,float16,1,0.010416000336408614
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,fp8,1,0.010406400263309478
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,float16,3,0.010391999781131745
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,fp8,3,0.009078399837017059
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,float16,7,0.01035040020942688
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,fp8,7,0.008798400312662125
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,float16,15,0.010444799810647965
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,fp8,15,0.010420800000429154
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,float16,31,0.010427200049161912
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,fp8,31,0.010471999645233154
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,float16,31,0.008486399799585343
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,float16,63,0.010419200360774993
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,fp8,63,0.010465600341558457
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,float16,127,0.01043199971318245
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,float16,255,0.010446400195360184
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,fp8,127,0.010391999781131745
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,fp8,4095,0.010809600353240967
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,fp8,255,0.010355199873447418
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,float16,511,0.010494399815797806
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,float16,1023,0.010417599976062775
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,fp8,511,0.010278400033712387
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,fp8,1023,0.01048320010304451
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,float16,2047,0.01045759990811348
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,fp8,2047,0.010411199927330018
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,float16,4095,0.0125231996178627
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,float16,8191,0.014507199823856353
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,float16,3,0.008425600081682205
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,fp8,8191,0.014584000408649444
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,fp8,1,0.008367999643087386
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,float16,3,0.008507200330495835
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,fp8,3,0.008417599648237229
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,float16,7,0.00862239971756935
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,fp8,7,0.008403199911117553
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,float16,15,0.008417599648237229
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,fp8,15,0.00835999995470047
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,float16,31,0.00840959995985031
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,fp8,31,0.008451200276613235
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,float16,63,0.008420799672603608
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,fp8,63,0.008395200222730636
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,float16,127,0.008423999696969987
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,fp8,127,0.008388800173997879
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,float16,255,0.008427199721336365
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,fp8,255,0.008353599905967712
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,float16,511,0.008452799916267396
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,fp8,511,0.008345600217580795
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,float16,1023,0.008476799726486206
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,fp8,1023,0.00843999981880188
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,float16,2047,0.010499200224876404
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,fp8,2047,0.010183999687433243
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,float16,4095,0.010451199859380722
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,fp8,4095,0.010463999956846238
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,fp8,8191,0.012401600182056428
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,float16,8191,0.013064000010490417
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,fp8,1,0.008403199911117553
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,float16,4095,0.012432000041007996
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,fp8,4095,0.012479999661445617
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,fp8,3,0.008436799794435502
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,float16,7,0.008459199965000153
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,float16,15,0.008497600257396699
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,fp8,15,0.00843999981880188
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,float16,31,0.008369600027799606
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,fp8,31,0.008366400003433227
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,float16,63,0.008479999750852585
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,fp8,63,0.00836160033941269
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,float16,127,0.008457600325345992
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,fp8,127,0.008364800363779068
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,float16,255,0.008420799672603608
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,fp8,255,0.008433599770069123
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,float16,511,0.008460800349712371
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,fp8,511,0.008432000130414962
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,float16,1023,0.009876800328493118
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,fp8,4095,0.010518400371074677
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,float16,1,0.008436799794435502
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,fp8,2047,0.010436800122261048
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,float16,4095,0.012547199428081513
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,fp8,4095,0.010908800363540649
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,float16,8191,0.014633600413799287
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,fp8,8191,0.01255040019750595
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,fp8,1,0.010463999956846238
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,float16,3,0.010406400263309478
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,fp8,3,0.01040479987859726
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,float16,7,0.010441599786281586
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,fp8,7,0.010387200117111205
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,float16,15,0.010475199669599533
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,fp8,15,0.010491199791431427
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,float16,31,0.010390400141477584
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,float16,1,0.008433599770069123
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,fp8,31,0.010395199805498124
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,float16,63,0.010391999781131745
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,float16,3,0.008422400057315826
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,fp8,127,0.010497599840164185
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,fp8,7,0.008363199979066848
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,fp8,255,0.010391999781131745
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,float16,511,0.010419200360774993
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,fp8,511,0.010411199927330018
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,float16,1023,0.010478399693965912
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,fp8,1023,0.010491199791431427
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,float16,2047,0.012455999851226807
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,fp8,2047,0.011324799805879592
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,float16,4095,0.01451839953660965
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,fp8,4095,0.01427839994430542
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,float16,8191,0.029207998514175416
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,fp8,1023,0.008636800199747085
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,float16,2047,0.010391999781131745
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,fp8,8191,0.018590399622917177
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,float16,1,0.0103472001850605
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,fp8,1,0.010396800190210342
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,float16,3,0.010409600287675857
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,fp8,63,0.008340799808502197
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,float16,7,0.010252799838781357
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,float16,15,0.010422399640083313
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,fp8,15,0.01043199971318245
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,float16,31,0.01043360009789467
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,fp8,31,0.010480000078678131
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,float16,63,0.010411199927330018
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,fp8,63,0.010526400059461594
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,float16,127,0.010451199859380722
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,fp8,127,0.010425599664449692
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,float16,255,0.01040000021457672
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,fp8,255,0.009028799831867218
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,float16,511,0.010419200360774993
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,fp8,63,0.010424000024795533
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,fp8,511,0.008441600203514098
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,float16,1023,0.010478399693965912
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,float16,255,0.010516799986362457
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,float16,2047,0.012515200674533844
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,fp8,2047,0.0125231996178627
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,float16,4095,0.02889440059661865
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,fp8,4095,0.014982399344444276
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,float16,8191,0.046744000911712644
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,fp8,8191,0.029236799478530882
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,float16,1,0.008353599905967712
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,fp8,1,0.008398400247097015
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,fp8,3,0.008395200222730636
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,float16,7,0.008388800173997879
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,fp8,7,0.008457600325345992
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,float16,15,0.008408000320196151
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,fp8,15,0.0084927998483181
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,float16,31,0.008473599702119828
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,fp8,31,0.008423999696969987
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,float16,63,0.008454400300979614
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,fp8,63,0.00840959995985031
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,float16,127,0.00843520015478134
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,fp8,127,0.008228799700737
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,fp8,3,0.010367999970912933
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,float16,255,0.008446399867534638
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,fp8,255,0.008367999643087386
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,float16,511,0.009672000259160995
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,fp8,511,0.00843840017914772
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,fp8,1023,0.01037440001964569
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,float16,2047,0.02725439965724945
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,fp8,2047,0.012435200065374375
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,float16,4095,0.04539200067520142
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,fp8,4095,0.027559998631477355
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,float16,8191,0.08218079805374146
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,float16,127,0.010372799634933472
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,fp8,8191,0.045824000239372255
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,float16,1,0.010377600044012069
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,fp8,1,0.008555199950933456
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,float16,3,0.010416000336408614
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,fp8,3,0.008619199693202972
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,float16,7,0.010411199927330018
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,fp8,7,0.008380799740552902
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,fp8,15,0.008428800106048583
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,float16,31,0.010179200023412705
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,fp8,31,0.008457600325345992
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,float16,63,0.010372799634933472
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,fp8,63,0.008459199965000153
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,float16,127,0.010371199995279311
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,fp8,127,0.008657599985599517
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,float16,255,0.010391999781131745
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,float16,3,0.00835679993033409
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,fp8,255,0.008460800349712371
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,float16,511,0.012439999729394913
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,float16,1023,0.026761600375175477
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,fp8,511,0.010414399951696397
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,fp8,1023,0.013025599718093871
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,float16,2047,0.045665600895881654
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,fp8,2047,0.02890239953994751
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,float16,4095,0.08295199871063233
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,fp8,4095,0.04804159998893738
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,float16,1,0.010446400195360184
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,fp8,8191,0.08858240246772767
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,float16,3,0.010473600029945374
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,fp8,3,0.010471999645233154
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,float16,7,0.01051200032234192
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,fp8,7,0.010497599840164185
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,float16,15,0.010478399693965912
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,fp8,15,0.010476800054311753
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,float16,31,0.010465600341558457
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,fp8,7,0.008852799981832504
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,fp8,31,0.010476800054311753
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,float16,63,0.010524799674749374
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,fp8,63,0.010467199981212616
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,float16,127,0.01048479974269867
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,fp8,127,0.010444799810647965
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,float16,255,0.010420800000429154
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,fp8,255,0.010411199927330018
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,float16,511,0.01053439974784851
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,fp8,511,0.010382399708032609
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,float16,1023,0.010406400263309478
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,fp8,1023,0.010382399708032609
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,float16,2047,0.011137600243091583
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,fp8,2047,0.01045600026845932
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,float16,4095,0.012432000041007996
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,fp8,4095,0.01249760016798973
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,float16,8191,0.014579200744628906
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,fp8,8191,0.014511999487876893
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,fp8,1023,0.0104592002928257
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,float16,1,0.010391999781131745
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,float16,15,0.010428799688816071
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,fp8,3,0.01040000021457672
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,float16,7,0.010417599976062775
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,fp8,7,0.008807999640703201
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,float16,15,0.010419200360774993
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,fp8,15,0.008660800009965896
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,float16,31,0.010396800190210342
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,fp8,31,0.010375999659299851
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,float16,63,0.010395199805498124
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,fp8,63,0.010407999902963639
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,float16,127,0.0104032002389431
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,fp8,1,0.010486400127410889
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,float16,255,0.010384000092744827
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,fp8,127,0.008894400298595428
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,fp8,255,0.010419200360774993
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,float16,511,0.010385599732398988
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,fp8,511,0.010393600165843963
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,float16,1023,0.010409600287675857
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,fp8,1023,0.0104032002389431
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,float16,2047,0.010500799864530563
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,fp8,2047,0.010487999767065048
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,float16,4095,0.012460800260305405
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,fp8,4095,0.012441600114107132
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,float16,8191,0.01449120044708252
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,fp8,8191,0.01252640038728714
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,float16,1,0.010491199791431427
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,float16,1,0.00844319984316826
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,fp8,1,0.008353599905967712
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,float16,3,0.008455999940633774
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,float16,1023,0.010955200344324113
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,fp8,7,0.008347199857234954
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,float16,15,0.008401600271463394
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,fp8,15,0.00846719965338707
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,float16,31,0.008452799916267396
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,fp8,31,0.008455999940633774
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,float16,63,0.008436799794435502
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,fp8,63,0.00843999981880188
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,float16,127,0.008743999898433686
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,fp8,127,0.008486399799585343
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,float16,255,0.008427199721336365
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,fp8,255,0.008475200086832047
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,float16,511,0.008529599756002426
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,float16,1023,0.010473600029945374
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,fp8,1023,0.00846719965338707
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,float16,2047,0.010491199791431427
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,fp8,2047,0.010499200224876404
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,float16,4095,0.012488000094890594
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,fp8,4095,0.010491199791431427
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,fp8,1,0.010436800122261048
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,float16,3,0.010406400263309478
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,float16,8191,0.014560000598430633
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,float16,1,0.008448000252246856
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,fp8,8191,0.012558400630950928
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,fp8,1,0.008419200032949447
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,float16,3,0.008364800363779068
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,fp8,3,0.00838399976491928
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,fp8,7,0.00836160033941269
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,float16,15,0.00838399976491928
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,fp8,15,0.008353599905967712
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,float16,31,0.008339200168848038
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,fp8,31,0.008372800052165985
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,float16,63,0.008414400368928909
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,fp8,63,0.008432000130414962
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,float16,127,0.008393599838018417
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,float16,255,0.008364800363779068
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,fp8,255,0.008380799740552902
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,float16,8191,0.15730559825897217
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,float16,511,0.008392000198364257
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,float16,1023,0.008446399867534638
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,fp8,511,0.008388800173997879
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,fp8,1023,0.008478400111198426
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,float16,2047,0.01037440001964569
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,fp8,2047,0.010425599664449692
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,float16,4095,0.012457600235939026
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,fp8,4095,0.010494399815797806
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,float16,8191,0.028254398703575136
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,fp8,8191,0.013676799833774567
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,float16,1,0.010414399951696397
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,fp8,1,0.010491199791431427
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,float16,3,0.010369600355625152
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,fp8,3,0.010444799810647965
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,float16,7,0.010417599976062775
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,fp8,7,0.010087999701499938
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,float16,15,0.010407999902963639
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,fp8,15,0.01053600013256073
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,float16,31,0.010372799634933472
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,fp8,3,0.008339200168848038
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,float16,63,0.010441599786281586
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,fp8,63,0.010279999673366546
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,fp8,31,0.010375999659299851
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,fp8,127,0.01045759990811348
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,float16,255,0.01026879996061325
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,float16,511,0.011126399785280228
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,fp8,511,0.010496000200510025
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,float16,1023,0.014548799395561219
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,fp8,1023,0.012495999783277511
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,float16,2047,0.028841599822044373
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,fp8,2047,0.016663999855518342
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,float16,4095,0.047284799814224246
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,fp8,4095,0.03072479963302612
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,float16,1,0.010513599961996078
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,fp8,1,0.010441599786281586
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,float16,3,0.010809600353240967
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,float16,7,0.008420799672603608
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,float16,7,0.010478399693965912
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,fp8,7,0.010471999645233154
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,float16,15,0.010487999767065048
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,fp8,15,0.010503999888896942
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,fp8,127,0.008379200100898742
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,float16,31,0.010486400127410889
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,fp8,31,0.010387200117111205
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,float16,63,0.010460799932479859
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,float16,127,0.010779199749231338
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,fp8,63,0.01043199971318245
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,fp8,127,0.01048479974269867
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,float16,255,0.010401599854230881
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,fp8,255,0.010454399883747101
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,float16,511,0.013803200423717498
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,fp8,511,0.012452799826860428
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,float16,1023,0.02853280007839203
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,fp8,1023,0.014771200716495514
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,float16,2047,0.04716640114784241
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,fp8,2047,0.028984001278877257
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,float16,4095,0.08258399963378907
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,float16,7,0.008353599905967712
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,float16,1,0.012467200309038163
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,fp8,1,0.010478399693965912
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,float16,127,0.010384000092744827
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,float16,3,0.012467200309038163
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,fp8,3,0.01043199971318245
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,float16,7,0.01252799928188324
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,fp8,255,0.010396800190210342
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,fp8,7,0.010702399909496308
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,fp8,15,0.01045600026845932
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,float16,15,0.012460800260305405
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,float16,31,0.012486399710178375
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,float16,63,0.012479999661445617
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,fp8,63,0.010471999645233154
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,float16,127,0.012548799812793731
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,fp8,127,0.011099199950695037
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,float16,255,0.012462399899959564
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,float16,511,0.028380799293518066
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,fp8,511,0.014526399970054626
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,fp8,511,0.008399999886751174
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,fp8,1023,0.02882719933986664
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,float16,2047,0.08412320017814637
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,fp8,2047,0.04851360023021698
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,float16,1,0.016926400363445282
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,float16,4095,0.15953600406646729
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,fp8,4095,0.08473280072212219
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,fp8,1,0.01451359987258911
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,float16,3,0.016862399876117706
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,float16,7,0.0168272003531456
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,fp8,7,0.0147599995136261
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,float16,15,0.016638399660587312
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,fp8,15,0.014727999269962311
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,float16,31,0.016633599996566772
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,fp8,31,0.014611199498176575
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,fp8,3,0.010505600273609162
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,fp8,63,0.014627200365066529
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,float16,127,0.017817600071430205
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,float16,255,0.027668800950050355
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,fp8,255,0.014555199444293976
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,float16,511,0.04620479941368103
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,fp8,511,0.028859201073646545
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,float16,1023,0.08328319787979126
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,fp8,1023,0.048974400758743285
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,float16,2047,0.15621440410614013
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,fp8,2047,0.0871295988559723
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,fp8,31,0.010550399869680404
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,float16,4095,0.30207359790802
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,fp8,255,0.011105599999427795
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,fp8,1,0.012430399656295776
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,float16,1023,0.0477616012096405
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,float16,1,0.012508800625801087
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,float16,3,0.0125231996178627
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,fp8,4095,0.16570719480514526
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,fp8,7,0.01239679977297783
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,fp8,15,0.01242400035262108
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,float16,15,0.012535999715328216
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,float16,31,0.01252799928188324
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,fp8,31,0.01242400035262108
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,float16,63,0.012452799826860428
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,fp8,63,0.01242239996790886
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,float16,127,0.012488000094890594
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,fp8,127,0.012444800138473511
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,float16,255,0.01252640038728714
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,fp8,255,0.01244639977812767
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,float16,511,0.01648319959640503
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,fp8,511,0.014563199877738953
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,float16,1023,0.029416000843048094
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,float16,63,0.016683200001716615
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,fp8,1023,0.01868959963321686
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,float16,1,0.01448799967765808
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,fp8,1,0.012460800260305405
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,float16,3,0.014496000111103058
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,fp8,127,0.014553600549697876
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,fp8,3,0.012511999905109405
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,float16,7,0.01451839953660965
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,float16,15,0.01451839953660965
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,fp8,15,0.012427199631929398
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,fp8,7,0.012478400021791458
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,float16,31,0.01456640064716339
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,fp8,31,0.012548799812793731
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,float16,63,0.014564800262451171
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,fp8,63,0.012457600235939026
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,fp8,127,0.012441600114107132
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,float16,255,0.015302400290966033
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,fp8,255,0.012457600235939026
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,float16,511,0.028915199637413024
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,fp8,511,0.016519999504089354
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,float16,1023,0.0474016010761261
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,fp8,1023,0.03012000024318695
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,float16,1,0.016630400717258454
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,fp8,1,0.014497600495815277
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,float16,3,0.01661120057106018
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,fp8,3,0.014595200121402741
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,float16,7,0.016625599563121797
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,fp8,7,0.015006400644779205
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,float16,15,0.017665599286556245
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,fp8,15,0.014913600683212281
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,float16,31,0.01812800019979477
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,fp8,31,0.015105600655078887
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,float16,63,0.01807519942522049
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,fp8,63,0.01499200016260147
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,float16,127,0.018544000387191773
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,fp8,127,0.015299199521541596
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,float16,255,0.02839359939098358
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,fp8,255,0.014980800449848175
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,float16,511,0.047249600291252136
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,fp8,511,0.029104000329971312
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,float16,1023,0.08402400016784668
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,fp8,1023,0.0490880012512207
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,float16,1,0.027321600914001466
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,float16,3,0.02699199914932251
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,fp8,3,0.02279199957847595
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,float16,7,0.02709760069847107
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,fp8,3,0.0124208003282547
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,fp8,7,0.022708800435066224
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,float16,15,0.02752479910850525
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,fp8,15,0.022811199724674224
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,float16,31,0.02690880000591278
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,float16,63,0.027241599559783936
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,fp8,3,0.014510400593280792
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,float16,127,0.030641600489616394
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,fp8,127,0.022705599665641785
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,float16,255,0.04734880030155182
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,fp8,255,0.029155200719833373
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,float16,127,0.014507199823856353
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,fp8,511,0.04858559966087341
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,float16,1023,0.1563680052757263
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,fp8,1023,0.0866703987121582
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,float16,1,0.018540799617767334
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,fp8,1,0.01648319959640503
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,float16,3,0.01857759952545166
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,fp8,4095,0.049300798773765565
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,float16,7,0.018695999681949616
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,fp8,7,0.01664000004529953
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,float16,15,0.01870879977941513
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,fp8,1,0.022716799378395082
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,float16,31,0.018676799535751343
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,fp8,15,0.016568000614643096
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,fp8,31,0.016604800522327424
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,float16,63,0.018668800592422485
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,fp8,63,0.01658560037612915
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,fp8,127,0.016657599806785585
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,float16,255,0.018595199286937713
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,float16,7,0.012520000338554382
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,fp8,255,0.016620799899101257
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,float16,511,0.031150400638580322
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,fp8,511,0.021612800657749176
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,float16,1,0.022755199670791627
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,fp8,1,0.016641600430011748
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,fp8,31,0.022884799540042876
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,fp8,3,0.016705599427223206
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,float16,7,0.022815999388694764
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,fp8,7,0.018559999763965607
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,float16,15,0.02276639938354492
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,float16,31,0.022776000201702118
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,fp8,63,0.02276960015296936
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,float16,63,0.022710399329662324
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,fp8,63,0.016847999393939973
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,float16,127,0.023212799429893495
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,fp8,127,0.01831520050764084
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,float16,255,0.028788799047470094
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,fp8,255,0.017617599666118623
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,float16,511,0.04755359888076782
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,fp8,511,0.030819201469421388
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,float16,1,0.028308799862861632
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,fp8,1,0.023470400273799895
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,float16,511,0.08379999995231628
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,fp8,3,0.02276480048894882
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,float16,7,0.02686559855937958
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,fp8,7,0.02271520048379898
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,float16,15,0.026913601160049438
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,fp8,15,0.023396800458431243
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,float16,31,0.027902400493621825
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,fp8,31,0.023612800240516662
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,float16,63,0.028313601016998292
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,fp8,63,0.022843199968338012
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,float16,127,0.030849599838256837
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,fp8,3,0.0166143998503685
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,fp8,127,0.022892799973487855
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,float16,255,0.04757600128650665
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,fp8,255,0.029096001386642457
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,float16,511,0.08582720160484314
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,fp8,511,0.049163201451301576
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,float16,1,0.047679999470710756
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,fp8,1,0.039340800046920775
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,float16,3,0.04737760126590729
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,fp8,3,0.0391728013753891
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,float16,127,0.018566399812698364
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,float16,7,0.04759039878845215
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,fp8,7,0.039289599657058714
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,float16,15,0.04742720127105713
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,fp8,15,0.03919999897480011
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,float16,31,0.048449599742889406
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,fp8,31,0.03935999870300293
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,float16,63,0.04940159916877747
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,fp8,63,0.039852800965309146
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,float16,127,0.051868802309036253
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,fp8,127,0.04115999937057495
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,float16,255,0.08523359894752502
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,fp8,255,0.05108640193939209
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,fp8,15,0.017683200538158417
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,float16,1,0.010791999846696853
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,float16,511,0.15795520544052125
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,fp8,1,0.010505600273609162
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,float16,3,0.010396800190210342
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,fp8,511,0.08656960129737853
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,fp8,3,0.010387200117111205
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,fp8,7,0.01050880029797554
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,float16,15,0.0104592002928257
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,fp8,15,0.010489600151777268
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,float16,31,0.010497599840164185
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,fp8,31,0.010494399815797806
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,float16,63,0.010492800176143647
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,fp8,63,0.010487999767065048
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,float16,127,0.010470400005578995
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,fp8,127,0.010521599650382995
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,float16,255,0.01045600026845932
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,fp8,255,0.010526400059461594
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,float16,511,0.010409600287675857
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,fp8,511,0.010471999645233154
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,float16,1023,0.010375999659299851
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,float16,2047,0.011912000179290772
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,float16,3,0.027804800868034364
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,fp8,2047,0.010454399883747101
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,float16,4095,0.012464000284671784
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,fp8,4095,0.012531200051307678
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,float16,8191,0.014617599546909332
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,fp8,8191,0.014569599926471711
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,float16,1,0.010412800312042236
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,fp8,1,0.009278400242328644
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,float16,3,0.010358399897813796
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,fp8,3,0.010391999781131745
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,float16,7,0.01040479987859726
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,fp8,7,0.010422399640083313
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,float16,15,0.010388799756765366
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,fp8,15,0.010406400263309478
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,float16,31,0.010366400331258773
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,fp8,31,0.01040000021457672
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,float16,63,0.010416000336408614
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,fp8,63,0.010366400331258773
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,float16,127,0.010401599854230881
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,fp8,127,0.010382399708032609
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,float16,255,0.010416000336408614
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,fp8,255,0.009459199756383896
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,float16,511,0.010492800176143647
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,fp8,511,0.01037440001964569
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,float16,1023,0.01040000021457672
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,fp8,1023,0.01040000021457672
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,float16,2047,0.012430399656295776
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,fp8,2047,0.010427200049161912
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,float16,4095,0.013115200400352477
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,fp8,4095,0.012436799705028534
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,float16,8191,0.014569599926471711
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,fp8,8191,0.014484800398349762
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,float16,1,0.00838399976491928
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,fp8,1,0.008363199979066848
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,float16,3,0.008352000266313553
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,fp8,3,0.008337599784135818
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,float16,7,0.008372800052165985
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,fp8,7,0.008342400193214417
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,float16,15,0.008348800241947174
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,fp8,15,0.008350399881601333
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,float16,31,0.008352000266313553
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,fp8,31,0.008326400071382523
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,float16,63,0.00846719965338707
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,fp8,63,0.008342400193214417
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,fp8,127,0.008377599716186523
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,float16,255,0.008352000266313553
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,fp8,255,0.008367999643087386
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,float16,511,0.008369600027799606
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,fp8,511,0.008376000076532364
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,float16,1023,0.008462399989366532
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,fp8,1023,0.008432000130414962
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,float16,2047,0.010380800068378448
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,fp8,2047,0.010020799934864044
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,float16,4095,0.012539200484752655
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,fp8,4095,0.01048479974269867
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,float16,3,0.022720000147819518
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,fp8,8191,0.01451680064201355
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,float16,8191,0.027371200919151305
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,fp8,1,0.008441600203514098
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,float16,3,0.008395200222730636
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,fp8,3,0.008455999940633774
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,float16,7,0.008352000266313553
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,fp8,7,0.008476799726486206
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,float16,15,0.008432000130414962
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,float16,7,0.010699199885129929
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,fp8,15,0.00843999981880188
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,float16,31,0.00843840017914772
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,float16,63,0.0083856001496315
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,fp8,31,0.008454400300979614
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,fp8,63,0.00846719965338707
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,float16,127,0.008345600217580795
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,fp8,127,0.00846719965338707
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,float16,255,0.008404800295829773
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,fp8,255,0.00844319984316826
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,fp8,511,0.008401600271463394
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,float16,1023,0.010385599732398988
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,fp8,1023,0.010491199791431427
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,fp8,1023,0.010020799934864044
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,float16,2047,0.012467200309038163
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,fp8,2047,0.010507199913263321
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,float16,4095,0.0274399995803833
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,fp8,4095,0.013022400438785553
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,float16,8191,0.045988801121711734
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,fp8,8191,0.02884959876537323
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,float16,1,0.02892799973487854
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,fp8,1,0.02675360143184662
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,fp8,3,0.02677760124206543
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,float16,3,0.029249599575996398
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,float16,7,0.02901119887828827
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,fp8,7,0.026782399415969847
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,float16,15,0.029147198796272276
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,float16,31,0.029043200612068176
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,fp8,31,0.026875200867652892
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,float16,63,0.029228800535202028
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,fp8,63,0.026817598938941957
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,float16,127,0.030368000268936157
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,fp8,127,0.02677760124206543
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,float16,255,0.03294079899787903
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,fp8,255,0.02725279927253723
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,float16,1,0.03964479863643646
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,float16,3,0.03967039883136749
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,fp8,3,0.027771198749542238
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,float16,127,0.008324799686670303
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,float16,7,0.03980959951877594
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,float16,15,0.04058080017566681
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,fp8,7,0.0271263986825943
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,fp8,15,0.028734400868415833
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,float16,31,0.039139199256896975
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,fp8,31,0.028830400109291075
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,fp8,63,0.028622400760650635
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,float16,127,0.0417248010635376
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,fp8,127,0.028518399596214293
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,float16,255,0.049270400404930116
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,float16,1,0.008374399691820144
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,fp8,255,0.0326992005109787
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,float16,1,0.04761120080947876
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,fp8,1,0.040387201309204104
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,fp8,31,0.01669439971446991
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,float16,7,0.047574400901794434
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,fp8,7,0.040094399452209474
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,float16,15,0.04786880016326904
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,float16,511,0.008985599875450135
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,fp8,15,0.04100320041179657
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,float16,31,0.04918079972267151
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,fp8,31,0.04082399904727936
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,float16,63,0.04947839975357056
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,fp8,63,0.04115360081195831
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,float16,127,0.05254719853401184
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,fp8,127,0.04324800074100495
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,float16,255,0.08658400177955627
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,fp8,255,0.05188480019569397
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,float16,1,0.08760640025138855
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,fp8,15,0.026800000667572023
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,fp8,1,0.07192639708518982
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,float16,3,0.08701440095901489
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,fp8,3,0.07213760018348694
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,float16,7,0.08782560229301453
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,float16,15,0.08870559930801392
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,float16,31,0.088372802734375
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,fp8,31,0.07232159972190857
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,float16,63,0.08841120004653931
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,fp8,1,0.02706719934940338
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,fp8,63,0.07327839732170105
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,float16,127,0.09433599710464477
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,fp8,127,0.07410719990730286
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,fp8,255,0.0925055980682373
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,float16,255,0.16087839603424073
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,1,128,1,float16,float16,1,0.05146080255508423
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,1,128,1,float16,float16,3,0.05143839716911316
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,1,128,1,float16,fp8,1,0.04554080069065094
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,float16,63,0.040510401129722595
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,1,128,1,float16,fp8,3,0.045311999320983884
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,1,128,1,float16,float16,7,0.05143839716911316
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,1,128,1,float16,fp8,7,0.045505601167678836
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,1,128,1,float16,float16,15,0.05140960216522217
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,1,128,1,float16,float16,31,0.05156959891319275
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,1,128,1,float16,fp8,31,0.045337599515914914
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,1,128,1,float16,float16,63,0.05228000283241272
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,1,128,1,float16,fp8,63,0.04531840085983276
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,1,128,1,float16,fp8,127,0.04715040028095245
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,float16,3,0.0474128007888794
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,2,128,1,float16,float16,1,0.07212640047073364
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,2,128,1,float16,fp8,1,0.04739840030670166
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,2,128,1,float16,float16,3,0.07227039933204651
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,2,128,1,float16,fp8,3,0.04740320146083832
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,2,128,1,float16,float16,7,0.07264000177383423
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,2,128,1,float16,fp8,7,0.04737280011177063
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,2,128,1,float16,float16,15,0.07359039783477783
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,2,128,1,float16,fp8,15,0.04742079973220825
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,2,128,1,float16,float16,31,0.07420480251312256
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,2,128,1,float16,fp8,31,0.04833439886569977
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,2,128,1,float16,float16,63,0.0741216003894806
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,2,128,1,float16,fp8,63,0.04783680140972137
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,2,128,1,float16,float16,127,0.07419520020484924
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,2,128,1,float16,fp8,127,0.049553599953651425
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,4,128,1,float16,float16,1,0.08812000155448914
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,fp8,7,0.0719327986240387
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,4,128,1,float16,fp8,1,0.07347360253334045
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,4,128,1,float16,float16,3,0.08821280002593994
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,fp8,15,0.07209280133247375
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,4,128,1,float16,fp8,3,0.07245759963989258
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,4,128,1,float16,float16,7,0.08842399716377258
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,4,128,1,float16,fp8,7,0.07386720180511475
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,4,128,1,float16,float16,15,0.0891759991645813
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,4,128,1,float16,float16,31,0.08916320204734803
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,4,128,1,float16,fp8,31,0.07396479845046997
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,4,128,1,float16,float16,63,0.08891519904136658
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,4,128,1,float16,fp8,63,0.07463999986648559
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,4,128,1,float16,float16,127,0.09587200284004212
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,4,128,1,float16,fp8,127,0.07614240050315857
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,8,128,1,float16,float16,1,0.1664031982421875
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,8,128,1,float16,fp8,1,0.13548959493637086
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,8,128,1,float16,float16,3,0.16701279878616332
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,8,128,1,float16,fp8,3,0.13556480407714844
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,8,128,1,float16,float16,7,0.1680415987968445
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,8,128,1,float16,fp8,7,0.1355520009994507
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,8,128,1,float16,float16,15,0.1676784038543701
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,1,128,1,float16,fp8,15,0.04554400146007538
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,8,128,1,float16,fp8,15,0.13595839738845825
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,8,128,1,float16,float16,31,0.16791679859161376
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,8,128,1,float16,fp8,31,0.136516797542572
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,8,128,1,float16,float16,63,0.1680559992790222
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,8,128,1,float16,fp8,63,0.13742079734802246
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,float16,1,0.010513599961996078
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,fp8,1,0.01048479974269867
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,float16,3,0.011151999980211259
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,8,128,1,float16,fp8,127,0.13950400352478026
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,8,128,1,float16,float16,127,0.17830560207366944
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,fp8,3,0.04039680063724518
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,float16,7,0.011214400082826615
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,fp8,15,0.010480000078678131
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,float16,31,0.010486400127410889
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,fp8,7,0.010824000090360641
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,float16,15,0.010611200332641601
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,fp8,31,0.010460799932479859
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,fp8,63,0.010460799932479859
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,fp8,127,0.010444799810647965
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,float16,255,0.010784000158309937
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,fp8,255,0.010407999902963639
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,float16,511,0.010412800312042236
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,fp8,511,0.010497599840164185
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,float16,1023,0.01043360009789467
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,fp8,1023,0.010363200306892395
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,float16,2047,0.012481600046157837
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,fp8,2047,0.012545600533485413
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,float16,4095,0.014511999487876893
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,fp8,4095,0.013128000497817992
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,float16,8191,0.016659200191497803
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,fp8,8191,0.015465599298477174
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,float16,1,0.009465599805116654
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,fp8,1,0.008460800349712371
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,fp8,3,0.008505599945783615
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,float16,7,0.008399999886751174
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,fp8,7,0.00856959968805313
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,float16,15,0.008401600271463394
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,fp8,15,0.008644799888134002
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,float16,31,0.008484800159931184
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,fp8,31,0.009403199702501298
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,float16,63,0.010156799852848054
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,fp8,63,0.008416000008583068
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,float16,127,0.009478399902582169
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,4,128,1,float16,fp8,15,0.07283200025558471
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,float16,255,0.010105600208044052
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,fp8,255,0.009275200217962265
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,fp8,511,0.008392000198364257
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,float16,1023,0.010452800244092942
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,1,128,1,float16,float16,127,0.05328480005264282
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,fp8,1023,0.008473599702119828
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,float16,2047,0.010492800176143647
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,fp8,3,0.01048479974269867
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,fp8,2047,0.010422399640083313
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,float16,4095,0.013284799456596375
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,float16,8191,0.028841599822044373
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,fp8,8191,0.014910399913787842
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,float16,1,0.008448000252246856
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,fp8,1,0.008444800227880477
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,float16,3,0.008428800106048583
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,fp8,3,0.008388800173997879
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,float16,7,0.008448000252246856
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,fp8,7,0.008473599702119828
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,float16,15,0.008374399691820144
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,fp8,15,0.008420799672603608
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,float16,3,0.010419200360774993
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,float16,31,0.008444800227880477
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,fp8,31,0.008428800106048583
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,float16,63,0.008489599823951722
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,float16,127,0.008523199707269669
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,fp8,127,0.008432000130414962
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,fp8,127,0.008448000252246856
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,float16,255,0.008444800227880477
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,fp8,255,0.008423999696969987
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,float16,511,0.008420799672603608
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,fp8,511,0.008433599770069123
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,float16,1023,0.010391999781131745
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,fp8,1023,0.008782400190830231
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,float16,2047,0.012515200674533844
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,fp8,2047,0.01048479974269867
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,float16,4095,0.028569599986076354
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,fp8,4095,0.013995200395584106
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,float16,8191,0.04567520022392273
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,fp8,8191,0.02884800136089325
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,float16,1,0.008399999886751174
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,float16,3,0.00841120034456253
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,fp8,3,0.008392000198364257
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,fp8,7,0.008289600163698197
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,float16,15,0.008417599648237229
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,fp8,15,0.00841279998421669
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,float16,31,0.00833280012011528
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,fp8,31,0.008387199789285659
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,float16,63,0.00841120034456253
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,fp8,63,0.008388800173997879
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,float16,127,0.00835999995470047
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,fp8,4095,0.012436799705028534
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,fp8,127,0.008484800159931184
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,float16,255,0.008428800106048583
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,float16,127,0.011287999898195266
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,float16,511,0.01040000021457672
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,fp8,255,0.008369600027799606
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,fp8,511,0.008422400057315826
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,fp8,1023,0.010406400263309478
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,float16,2047,0.026873600482940675
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,fp8,2047,0.012545600533485413
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,float16,4095,0.045326399803161624
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,fp8,4095,0.027544000744819643
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,float16,8191,0.08170880079269409
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,fp8,8191,0.045793598890304564
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,float16,1,0.010396800190210342
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,fp8,1,0.00920960009098053
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,float16,3,0.0104592002928257
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,fp8,3,0.009329599887132644
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,float16,7,0.01040479987859726
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,fp8,63,0.00835999995470047
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,fp8,15,0.010382399708032609
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,float16,31,0.010361599922180175
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,fp8,31,0.009755200147628785
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,float16,511,0.008423999696969987
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,float16,63,0.010382399708032609
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,float16,127,0.010380800068378448
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,fp8,127,0.009976000338792802
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,float16,255,0.010372799634933472
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,fp8,255,0.009151999652385712
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,fp8,1,0.008348800241947174
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,float16,511,0.010398399829864503
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,float16,7,0.008427199721336365
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,fp8,511,0.008460800349712371
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,float16,2047,0.01252640038728714
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,fp8,2047,0.012428800016641617
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,float16,4095,0.028814399242401124
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,fp8,1023,0.010411199927330018
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,float16,8191,0.046172800660133365
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,fp8,8191,0.029019200801849367
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,float16,16383,0.0821295976638794
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,fp8,16383,0.04735519886016846
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,float16,1,0.008342400193214417
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,float16,1023,0.01244800016283989
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,fp8,1,0.008376000076532364
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,float16,3,0.008345600217580795
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,fp8,3,0.008355200290679932
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,float16,7,0.008363199979066848
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,fp8,7,0.00841120034456253
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,fp8,15,0.00833439975976944
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,float16,31,0.008455999940633774
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,fp8,31,0.008374399691820144
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,float16,63,0.008432000130414962
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,fp8,63,0.008310399949550629
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,float16,127,0.008462399989366532
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,fp8,7,0.010384000092744827
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,fp8,127,0.008403199911117553
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,float16,255,0.008472000062465668
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,fp8,255,0.008364800363779068
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,float16,511,0.010428799688816071
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,fp8,63,0.008947200328111648
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,fp8,511,0.008537600189447403
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,float16,1023,0.011881600320339202
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,fp8,1023,0.010496000200510025
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,float16,2047,0.026979199051856993
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,fp8,2047,0.012547199428081513
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,float16,4095,0.045342400670051575
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,fp8,4095,0.027961599826812743
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,float16,8191,0.08156319856643676
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,fp8,8191,0.04628799855709076
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,float16,16383,0.1535248041152954
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,fp8,16383,0.08275520205497741
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,fp8,4095,0.01488959938287735
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,fp8,1,0.008408000320196151
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,fp8,3,0.00836160033941269
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,float16,7,0.010451199859380722
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,fp8,7,0.008403199911117553
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,float16,15,0.010436800122261048
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,float16,63,0.010520000010728836
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,fp8,15,0.008582399785518646
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,float16,31,0.010422399640083313
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,fp8,31,0.008747199922800064
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,float16,63,0.010412800312042236
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,fp8,63,0.00859680026769638
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,float16,127,0.010417599976062775
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,fp8,127,0.008991999924182892
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,float16,255,0.010414399951696397
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,fp8,255,0.008401600271463394
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,float16,511,0.0124719999730587
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,fp8,511,0.010411199927330018
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,fp8,1023,0.014251199364662171
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,float16,2047,0.04601759910583496
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,float16,15,0.010390400141477584
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,fp8,2047,0.028566399216651918
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,fp8,4095,0.047167998552322385
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,float16,4095,0.08214560151100159
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,fp8,8191,0.08429920077323913
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,float16,8191,0.15648319721221923
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,float16,16383,0.30131518840789795
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,float16,1023,0.010478399693965912
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,float16,1,0.012547199428081513
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,fp8,1,0.010494399815797806
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,float16,3,0.012567999958992004
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,fp8,3,0.010487999767065048
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,fp8,7,0.010480000078678131
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,float16,15,0.012505599856376648
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,float16,31,0.012508800625801087
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,float16,3,0.010401599854230881
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,float16,7,0.012887999415397644
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,fp8,31,0.010520000010728836
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,fp8,63,0.010553599894046783
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,float16,127,0.012427199631929398
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,fp8,127,0.010763200372457505
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,float16,255,0.012382400035858155
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,fp8,255,0.010467199981212616
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,float16,511,0.027544000744819643
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,fp8,511,0.014550399780273438
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,float16,1023,0.04652799963951111
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,fp8,1023,0.029022398591041564
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,float16,2047,0.08287519812583924
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,fp8,2047,0.04784800112247467
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,float16,15,0.008393599838018417
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,fp8,4095,0.08644800186157227
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,float16,4095,0.15717920064926147
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,float16,8191,0.3078336000442505
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,fp8,8191,0.16536959409713745
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,float16,1,0.008548799902200699
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,fp8,1,0.00843840017914772
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,float16,3,0.010468800365924836
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,fp8,3,0.008977600187063218
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,float16,16383,0.5994480133056641
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,fp8,16383,0.32410240173339844
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,float16,15,0.00843520015478134
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,fp8,16383,0.15951039791107177
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,fp8,15,0.008390399813652038
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,fp8,31,0.008430399745702744
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,float16,63,0.008428800106048583
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,fp8,63,0.00835840031504631
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,float16,127,0.008455999940633774
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,fp8,127,0.008347199857234954
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,float16,255,0.009371200203895569
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,fp8,255,0.008449599891901017
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,float16,1,0.010419200360774993
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,float16,511,0.00873439982533455
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,fp8,511,0.008779200166463852
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,float16,63,0.012465599924325943
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,fp8,1023,0.008670400083065032
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,float16,2047,0.010409600287675857
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,fp8,2047,0.010380800068378448
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,float16,4095,0.010491199791431427
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,fp8,4095,0.010424000024795533
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,float16,8191,0.012940800189971924
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,fp8,8191,0.012516799569129943
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,float16,16383,0.016630400717258454
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,fp8,16383,0.016568000614643096
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,float16,1,0.008441600203514098
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,fp8,1,0.00841120034456253
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,float16,3,0.008419200032949447
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,fp8,3,0.008436799794435502
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,float16,7,0.008425600081682205
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,float16,1023,0.027603200078010558
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,fp8,7,0.008379200100898742
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,float16,15,0.008367999643087386
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,fp8,15,0.008374399691820144
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,float16,31,0.008432000130414962
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,fp8,31,0.008459199965000153
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,float16,63,0.008364800363779068
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,float16,127,0.008388800173997879
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,fp8,127,0.008372800052165985
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,float16,255,0.008363199979066848
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,fp8,255,0.008372800052165985
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,float16,511,0.008448000252246856
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,fp8,511,0.008350399881601333
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,float16,7,0.010143999755382539
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,fp8,1023,0.008444800227880477
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,float16,2047,0.00912960022687912
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,fp8,7,0.008747199922800064
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,float16,4095,0.010440000146627427
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,fp8,4095,0.010470400005578995
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,fp8,2047,0.008459199965000153
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,float16,8191,0.012467200309038163
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,fp8,8191,0.012489599734544754
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,float16,16383,0.016630400717258454
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,fp8,16383,0.016624000668525696
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,float16,1,0.008427199721336365
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,fp8,1,0.00846880003809929
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,float16,3,0.008371199667453765
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,float16,7,0.008459199965000153
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,fp8,7,0.008337599784135818
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,fp8,15,0.010516799986362457
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,float16,15,0.008486399799585343
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,fp8,15,0.00843999981880188
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,float16,31,0.008428800106048583
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,fp8,31,0.00843999981880188
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,fp8,63,0.008353599905967712
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,float16,127,0.008374399691820144
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,fp8,127,0.008433599770069123
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,float16,255,0.00840959995985031
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,fp8,255,0.008430399745702744
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,float16,511,0.00843840017914772
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,fp8,511,0.008348800241947174
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,float16,1023,0.008423999696969987
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,fp8,1023,0.008462399989366532
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,float16,2047,0.009595199674367904
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,fp8,2047,0.008462399989366532
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,fp8,63,0.008393599838018417
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,float16,4095,0.010463999956846238
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,float16,8191,0.012505599856376648
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,fp8,8191,0.012443199753761292
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,float16,16383,0.016235199570655823
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,fp8,16383,0.014679999649524688
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,float16,1,0.00841120034456253
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,float16,1023,0.008396799862384795
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,fp8,1,0.008425600081682205
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,float16,3,0.008369600027799606
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,fp8,3,0.008446399867534638
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,float16,7,0.008430399745702744
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,float16,15,0.008436799794435502
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,fp8,15,0.008432000130414962
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,float16,31,0.008449599891901017
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,fp8,31,0.008425600081682205
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,float16,63,0.008393599838018417
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,float16,31,0.008376000076532364
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,fp8,63,0.00835999995470047
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,float16,127,0.008430399745702744
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,fp8,127,0.008403199911117553
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,float16,255,0.008377599716186523
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,fp8,255,0.008366400003433227
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,float16,511,0.009380800276994705
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,fp8,511,0.00846560001373291
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,float16,1023,0.00835999995470047
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,fp8,1023,0.008481600135564805
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,fp8,2047,0.010438399761915207
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,fp8,3,0.008432000130414962
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,fp8,4095,0.01040479987859726
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,float16,8191,0.013540799915790557
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,fp8,8191,0.012462399899959564
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,float16,16383,0.015641599893569946
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,float16,1023,0.008534400165081025
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,fp8,16383,0.014441600441932679
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,float16,1,0.010494399815797806
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,float16,63,0.008448000252246856
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,fp8,1,0.008574400097131729
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,fp8,3,0.010478399693965912
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,float16,7,0.010395199805498124
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,fp8,7,0.008454400300979614
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,fp8,15,0.010390400141477584
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,float16,31,0.00987360030412674
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,fp8,31,0.010361599922180175
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,float16,63,0.009700799733400345
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,float16,127,0.008743999898433686
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,fp8,63,0.010424000024795533
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,fp8,127,0.00843999981880188
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,float16,255,0.01037919968366623
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,fp8,255,0.00843999981880188
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,float16,511,0.010391999781131745
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,fp8,511,0.010440000146627427
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,float16,1023,0.010422399640083313
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,fp8,1023,0.010424000024795533
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,float16,2047,0.010465600341558457
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,fp8,2047,0.010380800068378448
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,float16,4095,0.011716800183057785
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,fp8,4095,0.010503999888896942
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,fp8,4095,0.010396800190210342
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,float16,8191,0.014467200636863709
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,fp8,8191,0.012540799379348756
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,float16,16383,0.018593600392341612
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,float16,1,0.008420799672603608
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,fp8,16383,0.016651199758052827
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,fp8,1,0.008427199721336365
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,float16,3,0.008376000076532364
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,fp8,3,0.008449599891901017
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,float16,7,0.00841120034456253
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,float16,15,0.00846560001373291
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,fp8,7,0.008460800349712371
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,fp8,15,0.008476799726486206
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,float16,31,0.00844319984316826
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,fp8,7,0.008441600203514098
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,fp8,31,0.008446399867534638
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,float16,63,0.008462399989366532
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,fp8,63,0.008427199721336365
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,fp8,127,0.008396799862384795
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,float16,255,0.008428800106048583
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,fp8,255,0.008399999886751174
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,float16,511,0.008416000008583068
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,float16,1023,0.00841120034456253
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,fp8,1023,0.008350399881601333
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,float16,2047,0.010393600165843963
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,float16,2047,0.009188800305128097
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,float16,4095,0.010451199859380722
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,float16,4095,0.01048479974269867
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,fp8,4095,0.010446400195360184
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,float16,8191,0.012489599734544754
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,fp8,8191,0.01250080019235611
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,float16,16383,0.01600639969110489
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,fp8,16383,0.014532800018787383
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,float16,1,0.008452799916267396
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,float16,3,0.010414399951696397
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,fp8,1,0.008352000266313553
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,float16,3,0.008423999696969987
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,float16,15,0.010460799932479859
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,fp8,7,0.008406399935483932
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,float16,15,0.008430399745702744
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,fp8,15,0.008449599891901017
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,float16,31,0.008417599648237229
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,fp8,31,0.00844319984316826
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,float16,63,0.008395200222730636
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,fp8,63,0.008476799726486206
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,float16,127,0.00854720026254654
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,fp8,127,0.008473599702119828
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,float16,255,0.008347199857234954
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,fp8,255,0.0083856001496315
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,float16,511,0.008441600203514098
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,fp8,511,0.008432000130414962
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,float16,1023,0.008416000008583068
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,fp8,1023,0.008430399745702744
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,float16,2047,0.010395199805498124
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,fp8,2047,0.01027199998497963
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,fp8,4095,0.010507199913263321
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,float16,4095,0.010598400235176086
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,float16,8191,0.01316159963607788
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,fp8,8191,0.012505599856376648
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,float16,16383,0.01579039990901947
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,fp8,16383,0.014368000626564025
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,fp8,1,0.00843999981880188
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,float16,3,0.008414400368928909
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,fp8,3,0.008343999832868576
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,float16,7,0.008374399691820144
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,float16,127,0.008483199775218964
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,fp8,7,0.00844319984316826
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,float16,15,0.009284800291061402
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,fp8,15,0.00846880003809929
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,fp8,511,0.00841279998421669
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,float16,31,0.008417599648237229
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,float16,63,0.008419200032949447
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,fp8,31,0.008489599823951722
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,fp8,63,0.008449599891901017
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,float16,127,0.008455999940633774
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,fp8,127,0.008497600257396699
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,float16,255,0.008395200222730636
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,fp8,255,0.008441600203514098
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,float16,511,0.00833280012011528
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,fp8,511,0.008428800106048583
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,float16,1023,0.010356800258159637
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,fp8,1023,0.008460800349712371
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,float16,2047,0.010388799756765366
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,fp8,2047,0.008390399813652038
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,fp8,2047,0.010424000024795533
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,float16,4095,0.012417600303888322
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,float16,8191,0.014579200744628906
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,fp8,4095,0.010681600123643876
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,float16,16383,0.02901119887828827
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,float16,1,0.009683199971914292
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,fp8,16383,0.015067200362682342
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,fp8,1,0.00843999981880188
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,float16,3,0.008495999872684479
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,float16,7,0.008716800063848496
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,fp8,7,0.008463999629020691
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,fp8,3,0.008419200032949447
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,fp8,3,0.008401600271463394
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,fp8,15,0.008374399691820144
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,float16,63,0.009544000029563904
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,fp8,63,0.008566399663686752
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,float16,127,0.01040000021457672
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,fp8,127,0.00894080027937889
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,float16,255,0.008491200208663941
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,fp8,255,0.008478400111198426
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,float16,511,0.008367999643087386
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,fp8,511,0.008460800349712371
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,float16,1023,0.01043040007352829
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,fp8,1023,0.008497600257396699
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,float16,2047,0.010414399951696397
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,fp8,2047,0.010401599854230881
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,float16,1,0.008452799916267396
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,float16,4095,0.013033600151538849
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,fp8,4095,0.012545600533485413
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,float16,8191,0.02882719933986664
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,fp8,8191,0.015113599598407745
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,float16,16383,0.04690560102462769
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,fp8,16383,0.02940639853477478
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,float16,1,0.008422400057315826
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,fp8,1,0.00841279998421669
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,float16,3,0.008457600325345992
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,fp8,3,0.008432000130414962
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,fp8,7,0.008441600203514098
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,float16,15,0.008430399745702744
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,fp8,15,0.00843840017914772
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,float16,31,0.008470399677753449
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,fp8,8191,0.012513600289821625
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,fp8,31,0.008460800349712371
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,float16,63,0.008419200032949447
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,fp8,63,0.00843840017914772
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,float16,15,0.00840959995985031
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,float16,127,0.008414400368928909
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,fp8,127,0.008460800349712371
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,float16,7,0.00838399976491928
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,float16,31,0.010367999970912933
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,float16,511,0.0091839998960495
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,fp8,255,0.00835679993033409
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,fp8,31,0.00838399976491928
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,float16,2047,0.012462399899959564
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,fp8,1023,0.010409600287675857
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,fp8,2047,0.010422399640083313
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,fp8,4095,0.013910399377346038
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,float16,8191,0.045691201090812684
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,float16,4095,0.02767840027809143
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,fp8,8191,0.028705599904060363
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,float16,16383,0.08186079859733582
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,float16,1,0.00835840031504631
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,fp8,16383,0.0474047988653183
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,fp8,1,0.008339200168848038
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,float16,3,0.008371199667453765
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,float16,7,0.00835679993033409
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,fp8,7,0.008350399881601333
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,float16,15,0.008376000076532364
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,fp8,15,0.008350399881601333
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,float16,31,0.008374399691820144
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,fp8,31,0.00835840031504631
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,float16,63,0.008441600203514098
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,fp8,63,0.008340799808502197
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,float16,127,0.008444800227880477
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,fp8,127,0.008367999643087386
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,fp8,255,0.008376000076532364
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,float16,511,0.00957920029759407
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,fp8,511,0.00846719965338707
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,float16,1023,0.010982400178909302
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,fp8,1023,0.010396800190210342
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,float16,2047,0.02714880108833313
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,fp8,2047,0.012535999715328216
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,float16,4095,0.0451664000749588
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,fp8,4095,0.027291199564933775
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,float16,8191,0.08190720081329346
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,fp8,8191,0.04589920043945313
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,float16,7,0.008371199667453765
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,fp8,16383,0.08274239897727967
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,float16,1,0.009329599887132644
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,float16,16383,0.1542464017868042
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,float16,3,0.009761600196361542
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,fp8,1,0.008457600325345992
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,float16,7,0.010372799634933472
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,fp8,3,0.008446399867534638
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,fp8,7,0.008473599702119828
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,float16,15,0.010436800122261048
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,float16,31,0.010067199915647506
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,fp8,15,0.008427199721336365
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,fp8,31,0.00857120007276535
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,float16,63,0.010411199927330018
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,fp8,63,0.008561599999666214
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,fp8,127,0.008417599648237229
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,float16,255,0.010363200306892395
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,fp8,255,0.008446399867534638
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,float16,511,0.012537600100040435
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,fp8,511,0.010417599976062775
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,float16,1023,0.02704319953918457
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,fp8,1023,0.014155200123786927
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,float16,2047,0.04563519954681396
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,fp8,2047,0.028910401463508605
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,float16,4095,0.08295360207557678
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,float16,255,0.00843840017914772
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,fp8,4095,0.04796639978885651
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,float16,1023,0.010407999902963639
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,fp8,511,0.008396799862384795
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,float16,8191,0.156550395488739
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,fp8,8191,0.087772798538208
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,float16,1,0.010494399815797806
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,fp8,1,0.010401599854230881
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,float16,3,0.010475199669599533
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,fp8,3,0.010334400087594986
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,fp8,3,0.008323200047016144
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,float16,7,0.0105103999376297
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,fp8,7,0.010353600233793258
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,float16,15,0.010393600165843963
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,fp8,15,0.0104032002389431
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,fp8,31,0.010406400263309478
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,float16,63,0.010435199737548828
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,float16,255,0.008353599905967712
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,float16,127,0.010513599961996078
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,fp8,127,0.010384000092744827
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,float16,255,0.010462400317192078
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,fp8,255,0.010356800258159637
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,float16,511,0.010388799756765366
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,fp8,511,0.00936800017952919
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,float16,1023,0.010424000024795533
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,fp8,1023,0.01037440001964569
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,fp8,2047,0.010398399829864503
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,float16,4095,0.01244639977812767
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,fp8,4095,0.012401600182056428
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,float16,8191,0.014569599926471711
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,fp8,8191,0.01448799967765808
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,float16,16383,0.016625599563121797
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,fp8,16383,0.016523200273513793
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,float16,1,0.008454400300979614
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,fp8,1,0.008454400300979614
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,float16,3,0.008422400057315826
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,float16,127,0.010395199805498124
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,fp8,3,0.008423999696969987
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,float16,7,0.008448000252246856
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,fp8,7,0.00841120034456253
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,float16,15,0.008452799916267396
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,fp8,15,0.00833280012011528
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,float16,31,0.008459199965000153
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,fp8,31,0.008342400193214417
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,float16,63,0.008486399799585343
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,fp8,63,0.008428800106048583
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,float16,127,0.008454400300979614
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,fp8,127,0.008422400057315826
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,float16,255,0.008452799916267396
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,fp8,255,0.00841279998421669
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,float16,511,0.008478400111198426
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,fp8,511,0.008433599770069123
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,float16,1023,0.008417599648237229
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,fp8,1023,0.008403199911117553
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,float16,2047,0.010398399829864503
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,fp8,2047,0.010358399897813796
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,fp8,4095,0.010496000200510025
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,float16,8191,0.012856000661849975
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,fp8,8191,0.012516799569129943
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,float16,16383,0.01510079950094223
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,float16,16383,0.30210559368133544
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,fp8,16383,0.17344160079956056
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,float16,31,0.010441599786281586
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,fp8,1,0.008452799916267396
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,float16,3,0.00843840017914772
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,fp8,63,0.009481599926948548
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,fp8,3,0.008352000266313553
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,float16,7,0.008419200032949447
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,fp8,7,0.008457600325345992
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,fp8,15,0.008353599905967712
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,float16,31,0.008423999696969987
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,fp8,31,0.008419200032949447
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,float16,63,0.008479999750852585
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,fp8,63,0.00833280012011528
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,float16,127,0.008459199965000153
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,float16,2047,0.010444799810647965
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,fp8,127,0.008422400057315826
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,float16,255,0.008476799726486206
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,float16,511,0.008463999629020691
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,fp8,255,0.008455999940633774
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,fp8,511,0.008376000076532364
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,float16,1023,0.00886240005493164
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,float16,2047,0.01043199971318245
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,float16,4095,0.01255040019750595
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,fp8,1023,0.008430399745702744
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,fp8,4095,0.010427200049161912
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,float16,8191,0.014603200554847717
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,float16,16383,0.030243200063705445
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,fp8,16383,0.01581439971923828
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,float16,1,0.008425600081682205
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,fp8,1,0.00843520015478134
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,float16,3,0.008367999643087386
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,float16,7,0.008363199979066848
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,fp8,7,0.00841279998421669
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,float16,15,0.00835999995470047
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,fp8,15,0.0083856001496315
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,float16,31,0.008348800241947174
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,float16,4095,0.012331199645996094
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,fp8,31,0.008340799808502197
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,float16,63,0.00844319984316826
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,fp8,63,0.00835840031504631
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,float16,127,0.008342400193214417
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,fp8,127,0.00835840031504631
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,fp8,16383,0.013761599361896516
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,float16,255,0.008377599716186523
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,fp8,255,0.008337599784135818
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,float16,511,0.008374399691820144
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,fp8,511,0.008353599905967712
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,float16,1023,0.00846560001373291
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,float16,1,0.008495999872684479
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,fp8,1023,0.008459199965000153
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,float16,2047,0.010391999781131745
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,fp8,2047,0.010444799810647965
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,float16,4095,0.012785600125789642
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,fp8,4095,0.010497599840164185
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,float16,15,0.008427199721336365
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,float16,8191,0.028259199857711793
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,fp8,8191,0.014523200690746307
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,float16,16383,0.046316799521446225
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,float16,1,0.00843840017914772
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,fp8,1,0.008414400368928909
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,float16,3,0.00846560001373291
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,fp8,3,0.008432000130414962
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,float16,7,0.008347199857234954
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,fp8,7,0.008449599891901017
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,fp8,15,0.008372800052165985
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,float16,31,0.008390399813652038
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,fp8,31,0.00833439975976944
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,float16,63,0.008408000320196151
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,fp8,63,0.008310399949550629
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,fp8,2047,0.010425599664449692
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,float16,127,0.008339200168848038
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,float16,255,0.00840959995985031
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,fp8,255,0.008382400125265121
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,float16,511,0.010395199805498124
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,fp8,8191,0.012529599666595458
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,float16,1023,0.012508800625801087
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,fp8,1023,0.010507199913263321
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,float16,2047,0.028414401412010192
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,fp8,2047,0.014595200121402741
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,fp8,3,0.00841279998421669
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,float16,4095,0.046121600270271304
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,fp8,4095,0.02889760136604309
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,float16,1,0.00931520015001297
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,float16,8191,0.0818656027317047
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,fp8,1,0.008436799794435502
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,float16,3,0.009140799939632415
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,fp8,3,0.008441600203514098
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,float16,7,0.010424000024795533
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,fp8,7,0.008451200276613235
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,float16,15,0.010372799634933472
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,fp8,15,0.00840959995985031
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,fp8,31,0.008441600203514098
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,float16,63,0.0104032002389431
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,fp8,63,0.008473599702119828
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,float16,127,0.010470400005578995
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,fp8,127,0.00865280032157898
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,float16,255,0.010475199669599533
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,fp8,255,0.00843999981880188
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,float16,511,0.012529599666595458
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,fp8,511,0.010492800176143647
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,float16,1023,0.02717280089855194
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,fp8,1023,0.01327040046453476
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,float16,2047,0.04627839922904968
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,fp8,16383,0.028815999627113342
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,fp8,2047,0.02887359857559204
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,float16,4095,0.0820527970790863
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,fp8,8191,0.08499360084533691
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,float16,8191,0.15513759851455688
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,float16,1,0.012518399953842163
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,fp8,127,0.008329600095748901
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,fp8,1,0.010545600205659866
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,float16,3,0.012534399330615998
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,fp8,511,0.00955199971795082
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,fp8,3,0.010369600355625152
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,float16,7,0.012443199753761292
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,fp8,7,0.011383999884128571
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,float16,15,0.012465599924325943
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,fp8,31,0.010499200224876404
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,float16,63,0.012439999729394913
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,fp8,63,0.010526400059461594
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,float16,127,0.012465599924325943
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,fp8,127,0.010475199669599533
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,float16,255,0.012436799705028534
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,fp8,255,0.010820800065994262
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,float16,511,0.027267199754714967
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,fp8,8191,0.04719040095806122
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,fp8,511,0.01451520025730133
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,float16,1023,0.0471455991268158
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,fp8,1023,0.02902719974517822
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,float16,2047,0.08381279706954955
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,fp8,2047,0.04795359969139099
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,float16,31,0.010358399897813796
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,float16,4095,0.15790560245513915
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,fp8,4095,0.08425120115280152
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,fp8,1,0.014582400023937226
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,float16,3,0.016523200273513793
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,fp8,8191,0.15859199762344361
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,fp8,3,0.014603200554847717
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,float16,7,0.016649599373340606
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,fp8,7,0.01456640064716339
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,float16,15,0.017420800030231477
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,fp8,15,0.014646400511264802
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,float16,31,0.017083199322223665
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,fp8,31,0.014528000354766845
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,float16,63,0.01658719927072525
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,fp8,63,0.014569599926471711
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,float16,127,0.016784000396728515
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,fp8,127,0.014588800072669984
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,fp8,4095,0.0476175993680954
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,fp8,255,0.01509920060634613
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,float16,511,0.04625599980354309
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,fp8,511,0.02898559868335724
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,float16,1023,0.08273280262947083
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,fp8,15,0.01048320010304451
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,fp8,1023,0.04887360036373138
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,float16,2047,0.15494400262832642
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,fp8,2047,0.08687999844551086
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,fp8,4095,0.1673087954521179
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,float16,4095,0.29971680641174314
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,float16,1,0.01664000004529953
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,float16,1,0.010424000024795533
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,float16,8191,0.30921759605407717
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,fp8,1,0.010427200049161912
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,fp8,3,0.010422399640083313
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,float16,3,0.010521599650382995
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,float16,7,0.010760000348091126
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,fp8,8191,0.32498080730438234
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,fp8,7,0.010361599922180175
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,float16,8191,0.5898240089416504
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,float16,31,0.010601600259542465
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,fp8,15,0.010440000146627427
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,fp8,63,0.010518400371074677
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,fp8,31,0.010496000200510025
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,float16,127,0.010902400314807891
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,fp8,127,0.010438399761915207
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,fp8,255,0.010391999781131745
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,float16,511,0.013449600338935852
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,fp8,511,0.012392000108957291
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,float16,15,0.008387199789285659
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,float16,1023,0.02847520112991333
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,fp8,1023,0.014820800721645355
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,float16,2047,0.04684160053730011
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,fp8,2047,0.02948639988899231
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,fp8,1,0.010499200224876404
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,float16,1,0.012516799569129943
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,float16,3,0.012529599666595458
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,float16,7,0.012438400089740754
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,fp8,3,0.010449600219726563
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,fp8,7,0.010451199859380722
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,float16,15,0.012545600533485413
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,fp8,15,0.010470400005578995
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,float16,31,0.012503999471664428
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,fp8,31,0.010500799864530563
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,float16,63,0.012520000338554382
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,fp8,63,0.010503999888896942
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,float16,127,0.012537600100040435
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,fp8,127,0.010473600029945374
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,float16,255,0.0124719999730587
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,float16,511,0.028815999627113342
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,fp8,511,0.014529600739479065
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,float16,1023,0.04725280106067657
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,fp8,1023,0.02909280061721802
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,float16,2047,0.08397600054740906
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,float16,1,0.01703519970178604
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,fp8,1,0.014619199931621552
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,float16,3,0.016627199947834015
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,fp8,3,0.01451359987258911
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,float16,7,0.0166143998503685
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,fp8,7,0.014609600603580474
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,float16,15,0.016790400445461272
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,fp8,15,0.014561599493026734
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,float16,15,0.010751999914646149
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,float16,31,0.016944000124931337
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,fp8,31,0.014689600467681885
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,float16,63,0.01709440052509308
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,fp8,63,0.01456640064716339
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,float16,127,0.017241600155830383
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,fp8,127,0.014539200067520141
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,fp8,255,0.014569599926471711
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,float16,255,0.02841919958591461
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,float16,511,0.04671039879322052
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,float16,255,0.010494399815797806
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,float16,1023,0.08412160277366638
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,fp8,1023,0.04813120067119599
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,float16,255,0.027871999144554137
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,float16,2047,0.15604480504989623
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,float16,1,0.027526399493217467
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,fp8,1,0.022793599963188173
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,float16,3,0.027340799570083618
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,fp8,3,0.02279680073261261
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,float16,7,0.02731040120124817
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,fp8,7,0.022884799540042876
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,float16,15,0.027419200539588927
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,fp8,15,0.022910399734973906
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,float16,31,0.01244639977812767
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,float16,31,0.02775520086288452
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,fp8,31,0.022777600586414336
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,float16,63,0.027931201457977294
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,fp8,63,0.022759999334812164
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,fp8,255,0.011134400218725204
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,float16,127,0.030859199166297913
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,fp8,127,0.023491199314594268
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,fp8,255,0.02882240116596222
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,float16,255,0.047030401229858396
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,float16,511,0.08302879929542542
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,fp8,511,0.047491198778152464
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,fp8,2047,0.04761599898338318
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,fp8,1023,0.08613439798355102
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,float16,1,0.014620800316333771
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,fp8,1,0.012468799948692322
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,float16,3,0.014564800262451171
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,fp8,2047,0.16242719888687135
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,fp8,3,0.01252640038728714
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,fp8,7,0.01252799928188324
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,float16,7,0.014577600359916686
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,float16,15,0.014499199390411378
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,fp8,15,0.0125231996178627
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,float16,31,0.014588800072669984
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,fp8,31,0.012531200051307678
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,float16,63,0.014484800398349762
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,fp8,63,0.012464000284671784
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,float16,127,0.014505599439144135
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,fp8,127,0.012468799948692322
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,float16,255,0.014963200688362122
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,float16,63,0.010652799904346467
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,fp8,255,0.012521600723266602
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,float16,511,0.02905600070953369
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,fp8,511,0.016601599752902985
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,float16,1023,0.04739679992198944
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,fp8,1023,0.03081279993057251
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,float16,1,0.016569599509239197
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,fp8,1,0.014620800316333771
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,float16,3,0.017294399440288544
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,fp8,511,0.02892799973487854
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,fp8,3,0.014643199741840363
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,float16,7,0.017161600291728973
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,fp8,7,0.014662399888038635
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,fp8,15,0.01470080018043518
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,float16,15,0.016649599373340606
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,fp8,31,0.01467680037021637
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,float16,63,0.01839199960231781
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,float16,127,0.017342400550842286
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,fp8,127,0.0165120005607605
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,float16,255,0.02863360047340393
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,fp8,255,0.014705599844455719
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,fp8,2047,0.08483840227127075
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,float16,511,0.047251200675964354
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,fp8,511,0.028809601068496705
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,float16,1023,0.08315680027008057
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,fp8,1023,0.04863199889659882
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,fp8,1,0.023124800622463228
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,fp8,3,0.02280319929122925
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,float16,7,0.02720319926738739
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,fp8,7,0.02313600033521652
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,float16,15,0.02709920108318329
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,fp8,15,0.023265600204467773
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,float16,31,0.027395200729370118
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,fp8,31,0.023299199342727662
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,float16,63,0.027676799893379213
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,float16,1023,0.1556399941444397
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,fp8,63,0.022918400168418885
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,float16,127,0.03079040050506592
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,fp8,127,0.02276480048894882
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,float16,255,0.04723680019378662
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,float16,2047,0.30030241012573244
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,fp8,255,0.028937599062919615
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,float16,511,0.08411039710044861
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,fp8,511,0.04829759895801544
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,float16,1023,0.15771679878234862
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,fp8,1023,0.08521440029144287
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,fp8,1,0.03956480026245117
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,float16,3,0.04748800098896026
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,fp8,3,0.03927839994430542
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,float16,7,0.047312000393867494
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,fp8,7,0.03930720090866089
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,float16,15,0.04735040068626404
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,fp8,15,0.039166399836540224
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,float16,31,0.04845759868621826
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,float16,63,0.049369600415229795
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,fp8,63,0.03915199935436249
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,float16,127,0.05139679908752441
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,float16,31,0.01788319945335388
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,fp8,63,0.01465280055999756
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,float16,255,0.08420000076293946
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,fp8,255,0.05058559775352478
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,fp8,511,0.08592000007629394
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,float16,511,0.15607199668884278
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,float16,1,0.027496001124382018
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,float16,1,0.010441599786281586
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,fp8,1,0.01021919995546341
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,float16,1023,0.3004784107208252
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,float16,3,0.010518400371074677
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,fp8,3,0.010436800122261048
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,fp8,1023,0.16156959533691406
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,fp8,7,0.008752000331878663
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,float16,7,0.01045759990811348
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,float16,31,0.01034879982471466
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,fp8,31,0.00960479974746704
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,float16,63,0.010446400195360184
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,fp8,63,0.010380800068378448
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,float16,127,0.010393600165843963
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,fp8,127,0.01040479987859726
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,float16,255,0.010388799756765366
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,fp8,255,0.010385599732398988
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,float16,511,0.01037919968366623
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,fp8,511,0.010438399761915207
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,float16,1023,0.010414399951696397
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,fp8,1023,0.010385599732398988
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,float16,2047,0.010478399693965912
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,fp8,2047,0.010369600355625152
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,float16,4095,0.012404800206422806
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,float16,1,0.04799199998378754
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,fp8,4095,0.012452799826860428
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,fp8,8191,0.013755199313163758
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,float16,16383,0.016547200083732606
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,fp8,16383,0.01454080045223236
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,float16,1,0.008448000252246856
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,fp8,1,0.00836160033941269
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,fp8,31,0.039175999164581296
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,float16,3,0.00836160033941269
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,fp8,3,0.008444800227880477
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,float16,7,0.008449599891901017
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,fp8,7,0.008444800227880477
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,float16,15,0.00840959995985031
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,fp8,15,0.008364800363779068
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,float16,31,0.008342400193214417
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,fp8,31,0.00836160033941269
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,float16,63,0.008427199721336365
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,fp8,63,0.00843999981880188
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,float16,127,0.008843199908733368
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,fp8,127,0.008457600325345992
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,float16,255,0.008924800157546996
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,fp8,255,0.008366400003433227
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,float16,511,0.00896959975361824
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,fp8,511,0.008448000252246856
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,float16,1023,0.009641599655151368
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,fp8,1023,0.008406399935483932
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,float16,2047,0.010393600165843963
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,fp8,2047,0.010417599976062775
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,float16,4095,0.012529599666595458
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,fp8,4095,0.010424000024795533
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,float16,8191,0.014480000734329224
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,float16,15,0.0104032002389431
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,fp8,8191,0.012467200309038163
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,float16,16383,0.02889919877052307
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,float16,1,0.00833280012011528
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,fp8,1,0.008432000130414962
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,fp8,16383,0.014572800695896148
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,float16,3,0.008428800106048583
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,fp8,3,0.008313599973917007
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,float16,7,0.008401600271463394
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,fp8,7,0.008374399691820144
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,float16,15,0.008374399691820144
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,float16,31,0.008416000008583068
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,fp8,31,0.008416000008583068
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,float16,63,0.00836160033941269
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,fp8,63,0.008347199857234954
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,float16,127,0.008473599702119828
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,float16,255,0.008414400368928909
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,fp8,255,0.008350399881601333
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,float16,511,0.008427199721336365
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,fp8,15,0.00843520015478134
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,fp8,511,0.008372800052165985
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,float16,1023,0.008984000235795975
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,fp8,1023,0.008444800227880477
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,float16,2047,0.010369600355625152
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,fp8,2047,0.010371199995279311
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,float16,4095,0.01249760016798973
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,fp8,4095,0.010420800000429154
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,float16,8191,0.02725439965724945
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,fp8,8191,0.014108799397945404
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,float16,16383,0.046224001049995425
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,float16,1,0.008455999940633774
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,fp8,16383,0.028812798857688903
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,float16,3,0.008347199857234954
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,fp8,3,0.008430399745702744
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,float16,7,0.008364800363779068
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,fp8,7,0.00835679993033409
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,float16,15,0.00833119973540306
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,fp8,15,0.008449599891901017
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,float16,31,0.008425600081682205
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,fp8,31,0.008392000198364257
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,float16,63,0.008399999886751174
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,fp8,63,0.008448000252246856
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,fp8,127,0.008425600081682205
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,float16,8191,0.014486399292945863
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,float16,255,0.00846560001373291
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,float16,511,0.008420799672603608
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,fp8,511,0.008337599784135818
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,float16,1023,0.010367999970912933
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,fp8,1023,0.00840959995985031
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,float16,2047,0.012486399710178375
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,fp8,2047,0.010503999888896942
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,float16,4095,0.02712959945201874
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,fp8,4095,0.013915200531482697
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,float16,8191,0.046305599808692935
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,float16,3,0.02783679962158203
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,fp8,8191,0.028644800186157227
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,fp8,15,0.00941760018467903
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,fp8,16383,0.04716480076313019
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,float16,1,0.02284960001707077
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,fp8,127,0.00841120034456253
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,fp8,1,0.018731200695037843
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,float16,3,0.022710399329662324
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,fp8,3,0.018611200153827667
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,float16,7,0.02274399995803833
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,fp8,7,0.018705600500106813
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,float16,15,0.0229312002658844
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,fp8,15,0.01866399943828583
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,float16,31,0.022771200537681578
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,fp8,31,0.01867839992046356
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,fp8,1,0.008379200100898742
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,float16,63,0.02279199957847595
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,float16,127,0.024432000517845155
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,fp8,127,0.018615999817848207
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,float16,255,0.029241600632667543
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,fp8,255,0.018632000684738158
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,float16,511,0.04816800057888031
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,float16,127,0.008452799916267396
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,float16,1,0.0279664009809494
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,fp8,1,0.0247311994433403
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,float16,3,0.02794879972934723
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,fp8,3,0.024835200607776643
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,fp8,255,0.008374399691820144
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,float16,7,0.02767840027809143
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,fp8,7,0.024766400456428528
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,float16,15,0.027004799246788024
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,fp8,15,0.024751999974250795
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,float16,31,0.02787039875984192
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,fp8,31,0.024747200310230255
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,float16,63,0.027934399247169495
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,float16,127,0.030876800417900085
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,fp8,127,0.0247311994433403
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,float16,255,0.0472784012556076
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,fp8,127,0.041233599185943604
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,fp8,63,0.024779200553894043
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,float16,511,0.08431199789047242
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,fp8,511,0.048895999789237976
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,float16,1,0.04767520129680634
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,fp8,1,0.03967840075492859
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,float16,3,0.047366398572921756
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,fp8,3,0.04041439890861511
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,float16,7,0.04734399914741516
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,fp8,7,0.0394896000623703
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,float16,15,0.04737760126590729
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,fp8,15,0.03939360082149505
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,float16,31,0.04796639978885651
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,fp8,31,0.03974719941616058
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,float16,63,0.04933759868144989
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,fp8,63,0.03957439959049225
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,float16,127,0.05161759853363037
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,fp8,127,0.042475199699401854
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,float16,255,0.08492640256881714
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,float16,16383,0.08225439786911011
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,float16,511,0.15859999656677246
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,fp8,511,0.08639039993286132
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,float16,1,0.08832319974899291
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,fp8,1,0.0720624029636383
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,float16,3,0.08829759955406188
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,fp8,3,0.07194560170173644
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,float16,7,0.08749279975891114
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,fp8,7,0.07199040055274963
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,fp8,15,0.07218400239944459
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,float16,15,0.0887664020061493
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,float16,31,0.08850880265235901
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,fp8,31,0.07223039865493774
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,fp8,63,0.07342560291290283
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,float16,63,0.08946239948272705
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,float16,127,0.09261760115623474
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,fp8,127,0.07455199956893921
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,fp8,63,0.01867839992046356
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,float16,255,0.15905280113220216
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,float16,1,0.03934879899024964
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,float16,3,0.039889600872993466
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,float16,511,0.30365920066833496
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,fp8,3,0.028939199447631837
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,fp8,511,0.031001600623130798
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,float16,7,0.04013279974460602
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,float16,15,0.040115201473236085
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,fp8,15,0.028860801458358766
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,float16,31,0.039908799529075625
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,fp8,31,0.028918400406837463
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,float16,63,0.04096800088882446
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,float16,127,0.04126879870891571
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,fp8,255,0.030876800417900085
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,fp8,127,0.028856000304222106
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,float16,255,0.049420800805091855
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,fp8,255,0.03296639919281006
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,float16,1,0.04812960028648376
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,fp8,1,0.041196799278259276
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,float16,3,0.04789440035820007
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,fp8,3,0.0412416011095047
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,float16,7,0.04820640087127685
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,fp8,7,0.041176000237464906
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,float16,15,0.04767520129680634
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,float16,31,0.04941920042037964
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,fp8,15,0.041264000535011294
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,fp8,31,0.04120799899101257
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,float16,63,0.049395200610160825
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,fp8,63,0.041150400042533876
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,float16,127,0.05188959836959839
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,fp8,127,0.04515999853610993
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,float16,255,0.08618879914283753
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,fp8,255,0.053337597846984865
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,fp8,255,0.09287520051002503
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,float16,1,0.08731840252876281
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,fp8,1,0.028937599062919615
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,float16,3,0.08755040168762207
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,fp8,511,0.16172480583190918
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,fp8,3,0.07254080176353454
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,float16,7,0.08715199828147888
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,fp8,7,0.07305279970169068
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,float16,15,0.08861119747161865
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,fp8,15,0.07301120162010193
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,float16,31,0.0889136016368866
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,float16,63,0.08879839777946472
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,fp8,63,0.07401599884033203
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,float16,127,0.09439359903335572
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,fp8,127,0.07579360008239747
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,fp8,255,0.09147999882698059
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,float16,1,0.16705280542373657
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,fp8,1,0.13578239679336548
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,fp8,255,0.05071840286254883
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,fp8,3,0.13551039695739747
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,float16,7,0.16811200380325317
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,fp8,7,0.13546400070190429
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,float16,15,0.1683568000793457
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,fp8,15,0.13633439540863038
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,float16,31,0.16845279932022095
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,fp8,1,0.07230880260467529
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,fp8,31,0.1368175983428955
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,fp8,7,0.028932800889015196
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,float16,63,0.1684432029724121
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,fp8,63,0.13759360313415528
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,fp8,31,0.07313920259475708
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,float16,127,0.17658239603042603
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,fp8,127,0.1395792007446289
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,fp8,63,0.02884640097618103
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,float16,1,0.010409600287675857
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,fp8,1,0.010407999902963639
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,float16,3,0.010407999902963639
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,fp8,3,0.010388799756765366
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,float16,7,0.010377600044012069
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,fp8,7,0.010424000024795533
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,float16,15,0.010451199859380722
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,fp8,255,0.17628480195999147
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,fp8,15,0.010416000336408614
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,float16,31,0.010393600165843963
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,float16,255,0.30794079303741456
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,fp8,31,0.010412800312042236
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,float16,63,0.010355199873447418
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,float16,127,0.010497599840164185
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,fp8,127,0.009681600332260131
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,float16,255,0.16125600337982177
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,fp8,255,0.010407999902963639
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,fp8,63,0.00896959975361824
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,float16,511,0.010417599976062775
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,fp8,511,0.010382399708032609
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,float16,255,0.010356800258159637
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,fp8,1023,0.010417599976062775
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,fp8,2047,0.01043040007352829
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,fp8,4095,0.012511999905109405
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,float16,2047,0.010990399867296219
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,float16,4095,0.01250080019235611
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,float16,8191,0.01467680037021637
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,fp8,8191,0.014542399346828461
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,fp8,16383,0.016630400717258454
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,float16,16383,0.030859199166297913
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,fp8,3,0.008336000144481659
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,float16,7,0.008742400258779526
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,float16,15,0.008350399881601333
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,fp8,15,0.00841279998421669
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,float16,31,0.008363199979066848
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,fp8,31,0.00844319984316826
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,float16,63,0.008343999832868576
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,fp8,63,0.008430399745702744
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,float16,3,0.16709920167922973
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,float16,127,0.0083856001496315
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,fp8,127,0.008433599770069123
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,float16,255,0.00833439975976944
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,fp8,255,0.008379200100898742
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,float16,511,0.008459199965000153
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,fp8,511,0.008364800363779068
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,float16,1023,0.008462399989366532
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,fp8,1023,0.008433599770069123
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,float16,2047,0.010791999846696853
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,fp8,2047,0.009785600006580353
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,fp8,4095,0.010486400127410889
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,float16,8191,0.027476799488067628
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,fp8,8191,0.014473600685596466
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,float16,16383,0.04549919962882996
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,float16,1,0.008343999832868576
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,fp8,1,0.008348800241947174
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,float16,3,0.0083856001496315
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,fp8,3,0.008393599838018417
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,float16,7,0.008364800363779068
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,float16,1023,0.010411199927330018
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,fp8,7,0.008324799686670303
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,float16,15,0.00840959995985031
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,float16,1,0.008388800173997879
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,float16,3,0.008318399637937545
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,fp8,1,0.008366400003433227
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,fp8,7,0.008452799916267396
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,float16,31,0.008414400368928909
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,float16,63,0.008376000076532364
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,fp8,63,0.008544000238180161
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,float16,255,0.008374399691820144
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,fp8,255,0.008396799862384795
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,float16,511,0.008486399799585343
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,fp8,511,0.008432000130414962
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,float16,4095,0.012513600289821625
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,float16,1023,0.010427200049161912
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,fp8,1023,0.00851999968290329
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,float16,2047,0.0124208003282547
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,fp8,2047,0.010478399693965912
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,float16,4095,0.0275983989238739
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,fp8,4095,0.012531200051307678
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,fp8,16383,0.0288239985704422
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,float16,8191,0.04578239917755127
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,fp8,8191,0.0285616010427475
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,float16,1,0.008395200222730636
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,fp8,16383,0.04673280119895935
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,float16,16383,0.08228639960289001
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,fp8,1,0.008427199721336365
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,float16,3,0.008336000144481659
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,fp8,3,0.008382400125265121
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,float16,7,0.00843520015478134
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,fp8,7,0.008372800052165985
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,float16,15,0.008315200358629227
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,fp8,15,0.008416000008583068
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,float16,31,0.00843999981880188
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,fp8,31,0.008324799686670303
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,fp8,63,0.008347199857234954
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,float16,127,0.00843520015478134
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,fp8,127,0.00835999995470047
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,float16,255,0.008420799672603608
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,float16,511,0.010420800000429154
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,fp8,511,0.008452799916267396
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,float16,1023,0.012467200309038163
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,fp8,1023,0.010422399640083313
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,float16,2047,0.026955199241638184
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,fp8,2047,0.012529599666595458
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,float16,4095,0.045203199982643126
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,fp8,4095,0.027063998579978942
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,float16,8191,0.08192800283432007
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,fp8,8191,0.04600160121917725
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,fp8,15,0.00835679993033409
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,fp8,31,0.008355200290679932
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,float16,1,0.008427199721336365
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,fp8,127,0.00838399976491928
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,fp8,1,0.008497600257396699
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,float16,3,0.00836160033941269
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,fp8,3,0.008366400003433227
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,fp8,7,0.00835999995470047
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,float16,15,0.008432000130414962
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,fp8,15,0.008401600271463394
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,float16,31,0.008473599702119828
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,fp8,31,0.0083856001496315
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,float16,63,0.008459199965000153
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,fp8,63,0.008452799916267396
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,fp8,127,0.008382400125265121
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,float16,127,0.008369600027799606
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,float16,255,0.008390399813652038
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,fp8,255,0.008371199667453765
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,float16,511,0.008345600217580795
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,float16,1023,0.010356800258159637
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,fp8,1023,0.008777599781751633
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,float16,2047,0.01241919994354248
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,fp8,2047,0.010471999645233154
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,float16,4095,0.02815360128879547
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,fp8,4095,0.012703999876976013
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,float16,63,0.008473599702119828
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,float16,8191,0.04611999988555908
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,fp8,8191,0.02882719933986664
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,fp8,255,0.008339200168848038
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,float16,16383,0.08202080130577087
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,fp8,16383,0.046911999583244324
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,float16,1,0.00857120007276535
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,fp8,1,0.008404800295829773
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,float16,32767,0.1537984013557434
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,fp8,32767,0.08369759917259216
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,float16,3,0.008422400057315826
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,fp8,3,0.008345600217580795
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,float16,15,0.008350399881601333
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,fp8,15,0.008345600217580795
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,float16,31,0.00844319984316826
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,fp8,31,0.008326400071382523
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,float16,63,0.008404800295829773
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,fp8,63,0.008339200168848038
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,float16,127,0.008337599784135818
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,fp8,127,0.008329600095748901
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,float16,255,0.008451200276613235
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,fp8,255,0.008337599784135818
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,float16,511,0.010396800190210342
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,fp8,511,0.008376000076532364
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,float16,127,0.008516799658536911
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,float16,1023,0.01191679984331131
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,fp8,1023,0.010380800068378448
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,float16,7,0.008430399745702744
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,float16,2047,0.02682879865169525
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,fp8,2047,0.0126351997256279
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,float16,4095,0.04524640142917633
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,fp8,4095,0.028116801381111146
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,float16,8191,0.08167840242385864
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,fp8,8191,0.046107199788093564
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,fp8,511,0.008433599770069123
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,float16,16383,0.15320320129394532
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,fp8,16383,0.0823632001876831
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,float16,1,0.010398399829864503
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,fp8,1,0.008367999643087386
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,fp8,32767,0.15626720190048218
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,float16,3,0.009699200093746186
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,float16,32767,0.29630720615386963
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,fp8,3,0.008433599770069123
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,float16,7,0.010385599732398988
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,fp8,7,0.008433599770069123
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,fp8,15,0.008451200276613235
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,fp8,31,0.008423999696969987
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,float16,63,0.010407999902963639
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,fp8,63,0.008697599917650223
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,float16,127,0.010406400263309478
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,fp8,127,0.008419200032949447
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,float16,7,0.008399999886751174
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,float16,255,0.01037440001964569
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,fp8,255,0.008414400368928909
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,float16,511,0.012425599992275238
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,fp8,511,0.010478399693965912
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,float16,1023,0.027169600129127502
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,fp8,1023,0.013064000010490417
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,float16,2047,0.04562239944934845
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,fp8,2047,0.02877120077610016
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,float16,4095,0.08219839930534363
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,fp8,4095,0.04704160094261169
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,fp8,8191,0.0839680016040802
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,float16,8191,0.1557423949241638
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,float16,16383,0.15410879850387574
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,fp8,16383,0.16011040210723876
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,float16,16383,0.3014048099517822
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,float16,1,0.008377599716186523
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,float16,3,0.008352000266313553
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,fp8,3,0.00843840017914772
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,float16,7,0.008420799672603608
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,fp8,32767,0.3096656084060669
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,fp8,7,0.008446399867534638
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,fp8,15,0.008355200290679932
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,float16,32767,0.5984992027282715
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,float16,15,0.008419200032949447
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,float16,63,0.008406399935483932
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,fp8,63,0.00843520015478134
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,fp8,31,0.008342400193214417
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,float16,127,0.008364800363779068
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,fp8,127,0.008355200290679932
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,fp8,255,0.00830560028553009
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,float16,15,0.010211200267076493
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,float16,511,0.008416000008583068
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,float16,31,0.010417599976062775
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,float16,1023,0.008454400300979614
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,fp8,511,0.008343999832868576
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,fp8,1023,0.008388800173997879
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,float16,2047,0.009038399904966354
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,float16,4095,0.010401599854230881
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,fp8,4095,0.010300800204277039
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,float16,8191,0.012495999783277511
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,fp8,8191,0.012430399656295776
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,float16,16383,0.016539199650287627
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,fp8,16383,0.014616000652313232
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,fp8,7,0.008430399745702744
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,float16,32767,0.016571199893951415
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,fp8,32767,0.016545599699020384
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,float16,1,0.008436799794435502
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,float16,3,0.008342400193214417
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,fp8,3,0.008388800173997879
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,float16,7,0.008422400057315826
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,fp8,7,0.008404800295829773
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,float16,15,0.00835679993033409
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,fp8,15,0.008449599891901017
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,float16,31,0.00833119973540306
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,fp8,31,0.00843999981880188
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,fp8,63,0.008446399867534638
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,float16,127,0.008398400247097015
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,fp8,127,0.008427199721336365
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,float16,255,0.008505599945783615
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,fp8,255,0.008441600203514098
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,float16,511,0.008475200086832047
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,fp8,16383,0.08340960144996643
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,float16,1023,0.008355200290679932
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,fp8,1,0.008327999711036682
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,float16,2047,0.01040000021457672
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,fp8,2047,0.008436799794435502
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,float16,4095,0.010414399951696397
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,fp8,4095,0.010388799756765366
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,float16,31,0.008433599770069123
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,float16,8191,0.012425599992275238
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,fp8,8191,0.012443199753761292
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,float16,16383,0.016648000478744505
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,fp8,16383,0.0165120005607605
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,float16,32767,0.018568000197410582
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,fp8,32767,0.016641600430011748
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,float16,255,0.008436799794435502
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,float16,1,0.008423999696969987
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,fp8,1,0.008427199721336365
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,fp8,3,0.008486399799585343
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,float16,7,0.009014400094747544
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,fp8,2047,0.008441600203514098
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,fp8,7,0.008432000130414962
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,float16,15,0.008500800281763077
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,fp8,15,0.00841120034456253
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,float16,31,0.008395200222730636
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,fp8,31,0.008323200047016144
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,float16,63,0.008401600271463394
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,fp8,63,0.008396799862384795
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,float16,127,0.008488000184297562
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,fp8,127,0.008462399989366532
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,float16,255,0.008427199721336365
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,fp8,255,0.00838399976491928
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,float16,511,0.008406399935483932
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,fp8,511,0.008371199667453765
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,float16,1023,0.009177599847316743
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,fp8,1023,0.00835999995470047
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,float16,2047,0.010396800190210342
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,float16,63,0.008444800227880477
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,fp8,2047,0.008470399677753449
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,float16,4095,0.010491199791431427
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,fp8,4095,0.010460799932479859
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,float16,8191,0.0124719999730587
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,float16,16383,0.015348799526691437
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,fp8,16383,0.01448799967765808
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,float16,32767,0.017268800735473634
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,fp8,32767,0.016161599755287172
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,float16,1,0.008353599905967712
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,float16,3,0.008433599770069123
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,fp8,3,0.008363199979066848
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,fp8,511,0.008454400300979614
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,fp8,7,0.008428800106048583
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,float16,15,0.00843999981880188
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,fp8,15,0.00841120034456253
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,float16,31,0.008419200032949447
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,fp8,31,0.008475200086832047
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,float16,63,0.008528000116348267
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,fp8,1023,0.008371199667453765
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,fp8,63,0.008449599891901017
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,float16,127,0.008364800363779068
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,fp8,127,0.008436799794435502
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,float16,255,0.00830560028553009
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,fp8,255,0.00843999981880188
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,float16,511,0.008436799794435502
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,fp8,511,0.008497600257396699
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,float16,1023,0.008441600203514098
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,fp8,1023,0.00844319984316826
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,float16,2047,0.008774399757385254
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,fp8,2047,0.008452799916267396
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,float16,4095,0.010360000282526016
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,fp8,4095,0.010422399640083313
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,float16,8191,0.012470400333404541
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,fp8,8191,0.012534399330615998
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,fp8,16383,0.016624000668525696
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,float16,3,0.009382399916648864
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,float16,32767,0.018673600256443025
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,fp8,32767,0.017046399414539337
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,float16,1,0.008414400368928909
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,fp8,1,0.008425600081682205
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,float16,3,0.00843999981880188
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,fp8,3,0.008432000130414962
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,float16,7,0.008495999872684479
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,fp8,7,0.00835999995470047
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,float16,15,0.008463999629020691
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,fp8,15,0.008432000130414962
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,float16,31,0.00846560001373291
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,fp8,31,0.008363199979066848
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,fp8,63,0.00843840017914772
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,fp8,1,0.00846719965338707
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,float16,127,0.008457600325345992
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,fp8,127,0.008454400300979614
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,float16,255,0.008366400003433227
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,fp8,255,0.008353599905967712
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,float16,511,0.008452799916267396
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,fp8,511,0.008452799916267396
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,float16,1023,0.00844319984316826
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,float16,2047,0.010363200306892395
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,fp8,1023,0.008399999886751174
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,fp8,8191,0.012484800070524216
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,fp8,2047,0.00995199978351593
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,float16,4095,0.010451199859380722
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,fp8,4095,0.010478399693965912
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,float16,8191,0.014511999487876893
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,float16,16383,0.01639360040426254
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,fp8,1,0.00838399976491928
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,float16,32767,0.018632000684738158
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,fp8,1,0.008396799862384795
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,float16,1,0.008761599659919739
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,float16,3,0.008401600271463394
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,float16,7,0.008408000320196151
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,fp8,7,0.0083856001496315
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,float16,15,0.008579199761152267
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,fp8,15,0.00838399976491928
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,float16,31,0.009387200325727462
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,float16,16383,0.016620799899101257
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,fp8,31,0.008369600027799606
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,float16,63,0.008446399867534638
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,fp8,63,0.008436799794435502
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,float16,127,0.008416000008583068
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,fp8,127,0.00838399976491928
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,float16,255,0.00841279998421669
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,fp8,255,0.00846560001373291
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,float16,63,0.00841279998421669
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,float16,511,0.008428800106048583
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,fp8,511,0.008495999872684479
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,float16,1023,0.008448000252246856
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,fp8,1023,0.008968000113964082
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,float16,2047,0.010356800258159637
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,fp8,2047,0.01048479974269867
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,float16,4095,0.01051200032234192
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,fp8,4095,0.010521599650382995
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,float16,8191,0.012548799812793731
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,fp8,8191,0.012539200484752655
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,float16,16383,0.014571200311183929
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,fp8,16383,0.01454399973154068
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,float16,32767,0.03095200061798096
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,fp8,32767,0.01666879951953888
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,float16,1,0.008347199857234954
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,fp8,1,0.008392000198364257
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,float16,3,0.00833439975976944
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,fp8,16383,0.014564800262451171
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,fp8,3,0.008419200032949447
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,float16,7,0.008342400193214417
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,fp8,7,0.008459199965000153
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,float16,15,0.008432000130414962
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,fp8,15,0.008367999643087386
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,float16,31,0.00843840017914772
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,fp8,31,0.008363199979066848
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,fp8,32767,0.01661120057106018
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,float16,63,0.008401600271463394
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,fp8,63,0.008444800227880477
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,float16,127,0.008364800363779068
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,fp8,127,0.008287999778985977
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,fp8,255,0.008417599648237229
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,float16,511,0.008416000008583068
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,fp8,3,0.008390399813652038
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,float16,1023,0.008420799672603608
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,fp8,1023,0.008448000252246856
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,float16,2047,0.010390400141477584
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,fp8,2047,0.009243199974298478
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,float16,4095,0.012495999783277511
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,fp8,4095,0.010806400328874588
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,float16,8191,0.02768000066280365
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,fp8,8191,0.013582399487495423
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,float16,16383,0.04621599912643433
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,fp8,16383,0.028486400842666626
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,float16,32767,0.08205919861793518
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,float16,1,0.008423999696969987
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,fp8,32767,0.04723039865493774
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,float16,3,0.008455999940633774
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,fp8,1,0.008432000130414962
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,float16,7,0.008427199721336365
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,fp8,7,0.008366400003433227
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,float16,15,0.00851840004324913
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,fp8,15,0.008371199667453765
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,float16,31,0.008455999940633774
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,fp8,31,0.008441600203514098
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,float16,63,0.008416000008583068
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,fp8,63,0.008374399691820144
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,float16,127,0.008441600203514098
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,fp8,127,0.008366400003433227
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,float16,255,0.00844319984316826
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,fp8,255,0.00843840017914772
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,float16,511,0.008823999762535095
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,fp8,511,0.008436799794435502
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,float16,1023,0.010486400127410889
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,fp8,1023,0.008528000116348267
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,float16,2047,0.012540799379348756
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,float16,4095,0.027728000283241273
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,fp8,2047,0.010462400317192078
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,float16,8191,0.04549440145492554
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,fp8,8191,0.02885279953479767
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,float16,16383,0.08163999915122985
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,fp8,16383,0.0471455991268158
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,fp8,32767,0.08316959738731385
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,float16,1,0.008390399813652038
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,fp8,1,0.008323200047016144
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,float16,3,0.008444800227880477
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,fp8,3,0.008425600081682205
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,float16,7,0.008401600271463394
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,fp8,8191,0.012583999335765839
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,fp8,7,0.008393599838018417
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,float16,15,0.008408000320196151
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,fp8,15,0.008321599662303924
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,float16,31,0.008430399745702744
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,float16,63,0.008396799862384795
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,fp8,31,0.008388800173997879
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,float16,7,0.008398400247097015
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,fp8,63,0.008326400071382523
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,float16,127,0.008401600271463394
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,float16,255,0.008366400003433227
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,fp8,127,0.00833280012011528
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,fp8,255,0.008406399935483932
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,fp8,511,0.008326400071382523
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,float16,511,0.010396800190210342
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,float16,1023,0.012507200241088867
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,fp8,1023,0.010443200170993806
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,float16,2047,0.027423998713493346
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,fp8,2047,0.012537600100040435
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,float16,4095,0.04534080028533936
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,fp8,4095,0.027904000878334046
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,fp8,3,0.008382400125265121
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,float16,8191,0.08176000118255615
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,fp8,16383,0.0821071982383728
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,float16,16383,0.15342240333557128
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,fp8,32767,0.15542399883270264
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,float16,32767,0.29768478870391846
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,float16,1,0.00843840017914772
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,fp8,4095,0.012830400466918945
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,float16,3,0.008420799672603608
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,fp8,3,0.00841120034456253
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,fp8,1,0.008425600081682205
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,float16,7,0.008376000076532364
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,fp8,7,0.008451200276613235
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,float16,15,0.008396799862384795
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,fp8,15,0.008374399691820144
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,float16,31,0.008427199721336365
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,fp8,31,0.008377599716186523
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,float16,127,0.00939520001411438
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,fp8,127,0.008473599702119828
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,float16,32767,0.15361920595169068
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,fp8,63,0.00843999981880188
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,fp8,255,0.00846719965338707
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,float16,511,0.008479999750852585
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,fp8,511,0.008508799970149994
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,float16,1023,0.008446399867534638
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,fp8,1023,0.0084927998483181
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,float16,2047,0.00947680026292801
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,fp8,2047,0.00896959975361824
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,float16,4095,0.01048479974269867
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,fp8,4095,0.010401599854230881
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,float16,8191,0.01451680064201355
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,fp8,8191,0.012535999715328216
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,float16,16383,0.016627199947834015
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,fp8,16383,0.014507199823856353
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,float16,32767,0.01865759938955307
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,fp8,32767,0.016521599888801575
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,float16,1,0.00841120034456253
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,fp8,1,0.008355200290679932
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,fp8,3,0.00844319984316826
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,float16,7,0.008430399745702744
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,fp8,511,0.008315200358629227
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,fp8,7,0.008432000130414962
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,float16,15,0.00841120034456253
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,fp8,15,0.008436799794435502
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,float16,31,0.00844319984316826
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,fp8,31,0.00843840017914772
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,float16,63,0.00843520015478134
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,fp8,63,0.008433599770069123
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,float16,127,0.008641599863767623
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,fp8,127,0.008475200086832047
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,float16,255,0.008419200032949447
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,fp8,255,0.008425600081682205
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,fp8,8191,0.045444801449775696
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,fp8,511,0.008430399745702744
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,float16,1023,0.008367999643087386
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,fp8,1023,0.008417599648237229
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,float16,2047,0.010391999781131745
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,fp8,2047,0.010369600355625152
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,float16,4095,0.010583999752998351
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,fp8,4095,0.010463999956846238
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,float16,8191,0.013547199964523315
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,fp8,8191,0.012455999851226807
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,float16,16383,0.01650079935789108
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,fp8,16383,0.014511999487876893
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,fp8,32767,0.016612799465656282
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,float16,32767,0.031276801228523256
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,float16,1,0.008425600081682205
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,fp8,1,0.008481600135564805
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,float16,3,0.008455999940633774
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,fp8,3,0.008355200290679932
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,float16,7,0.008488000184297562
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,fp8,7,0.008323200047016144
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,float16,15,0.008432000130414962
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,fp8,15,0.008392000198364257
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,float16,31,0.00846719965338707
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,fp8,31,0.008425600081682205
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,float16,63,0.008393599838018417
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,fp8,63,0.008419200032949447
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,float16,127,0.00888959988951683
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,fp8,127,0.008433599770069123
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,float16,255,0.009131199866533279
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,fp8,255,0.008430399745702744
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,float16,255,0.008497600257396699
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,float16,511,0.008350399881601333
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,fp8,511,0.008342400193214417
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,float16,1023,0.00844319984316826
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,fp8,2047,0.01037919968366623
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,float16,2047,0.010447999835014344
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,fp8,1023,0.008422400057315826
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,fp8,4095,0.010983999818563461
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,float16,8191,0.014502400159835815
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,fp8,8191,0.012489599734544754
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,fp8,16383,0.015803200006484986
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,float16,16383,0.030884799361228944
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,float16,32767,0.04758880138397217
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,fp8,32767,0.03083199858665466
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,float16,1,0.008393599838018417
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,fp8,1,0.008367999643087386
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,float16,3,0.008414400368928909
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,fp8,3,0.008326400071382523
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,float16,7,0.008380799740552902
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,fp8,7,0.008355200290679932
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,float16,15,0.008417599648237229
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,fp8,15,0.008366400003433227
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,float16,31,0.00841120034456253
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,float16,3,0.008401600271463394
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,fp8,31,0.008428800106048583
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,float16,63,0.008374399691820144
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,fp8,63,0.008462399989366532
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,float16,127,0.008329600095748901
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,fp8,127,0.008395200222730636
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,fp8,255,0.008374399691820144
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,float16,511,0.010487999767065048
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,fp8,511,0.00843840017914772
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,float16,1023,0.012508800625801087
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,fp8,1023,0.010360000282526016
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,float16,2047,0.027190399169921876
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,fp8,2047,0.012451200187206269
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,float16,4095,0.04539999961853027
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,float16,511,0.008951999992132188
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,fp8,4095,0.028088000416755677
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,fp8,8191,0.04612640142440796
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,float16,8191,0.08180639743804932
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,float16,16383,0.1530735969543457
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,fp8,16383,0.08254560232162475
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,fp8,1,0.008449599891901017
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,float16,3,0.010371199995279311
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,float16,7,0.010382399708032609
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,fp8,7,0.00875839963555336
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,float16,15,0.010420800000429154
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,fp8,15,0.008423999696969987
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,float16,31,0.010364799946546554
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,fp8,31,0.008355200290679932
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,float16,63,0.01037919968366623
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,fp8,63,0.008657599985599517
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,float16,127,0.010441599786281586
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,fp8,127,0.008462399989366532
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,float16,255,0.010375999659299851
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,fp8,255,0.008448000252246856
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,float16,4095,0.012430399656295776
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,float16,511,0.012459199875593185
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,fp8,511,0.01048320010304451
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,float16,1023,0.027039998769760133
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,fp8,1023,0.013884800672531127
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,float16,2047,0.0455951988697052
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,fp8,2047,0.028832000494003297
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,float16,4095,0.08213919997215272
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,fp8,4095,0.04775680005550385
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,float16,255,0.00836160033941269
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,float16,8191,0.15386719703674318
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,fp8,8191,0.08468639850616455
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,float16,255,0.008449599891901017
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,float16,1,0.012507200241088867
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,fp8,1,0.010996799916028976
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,float16,3,0.012556800246238708
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,fp8,3,0.010907199978828431
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,float16,7,0.012459199875593185
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,fp8,16383,0.1598207950592041
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,fp8,7,0.010860799998044967
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,float16,15,0.012436799705028534
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,float16,16383,0.2987360000610352
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,float16,31,0.01252799928188324
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,fp8,15,0.010494399815797806
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,fp8,31,0.010468800365924836
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,float16,127,0.012449599802494049
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,float16,63,0.012465599924325943
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,float16,255,0.012439999729394913
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,fp8,255,0.011105599999427795
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,float16,511,0.027459201216697694
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,fp8,511,0.014535999298095703
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,float16,1,0.010438399761915207
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,fp8,63,0.010473600029945374
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,fp8,1023,0.029190400242805482
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,float16,2047,0.08367519974708557
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,fp8,4095,0.08428959846496582
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,fp8,3,0.008441600203514098
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,float16,4095,0.15810079574584962
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,float16,8191,0.30708160400390627
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,fp8,8191,0.15813920497894288
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,float16,63,0.009094399958848953
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,fp8,1,0.008582399785518646
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,fp8,3,0.008417599648237229
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,fp8,16383,0.30582399368286134
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,float16,16383,0.6042592048645019
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,fp8,7,0.009388799965381622
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,fp8,15,0.008556800335645676
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,float16,15,0.010425599664449692
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,float16,31,0.010361599922180175
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,fp8,31,0.008495999872684479
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,float16,63,0.010387200117111205
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,fp8,63,0.008423999696969987
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,float16,127,0.010419200360774993
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,fp8,127,0.008430399745702744
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,float16,255,0.010406400263309478
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,fp8,255,0.008446399867534638
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,float16,511,0.012451200187206269
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,fp8,511,0.010494399815797806
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,float16,1023,0.027673599123954774
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,fp8,1023,0.014478400349617004
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,float16,2047,0.04688639938831329
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,fp8,2047,0.028891199827194215
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,float16,4095,0.0820847988128662
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,fp8,127,0.01040000021457672
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,fp8,4095,0.047366398572921756
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,float16,1,0.012515200674533844
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,fp8,1,0.010516799986362457
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,float16,3,0.012479999661445617
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,fp8,3,0.010531199723482132
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,float16,7,0.012516799569129943
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,float16,1023,0.04694719910621643
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,fp8,7,0.01048320010304451
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,float16,15,0.012452799826860428
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,float16,31,0.012403199821710587
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,fp8,31,0.010443200170993806
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,float16,63,0.012428800016641617
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,fp8,63,0.010526400059461594
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,fp8,2047,0.04745599925518036
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,fp8,127,0.010780800133943558
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,float16,255,0.0125231996178627
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,fp8,255,0.010806400328874588
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,float16,511,0.027476799488067628
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,fp8,511,0.014470399916172027
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,float16,1023,0.046188798546791074
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,float16,1,0.010387200117111205
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,float16,3,0.010412800312042236
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,fp8,2047,0.04760800004005432
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,float16,7,0.010475199669599533
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,float16,1,0.01663520038127899
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,fp8,1,0.01462399959564209
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,float16,3,0.01661919951438904
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,fp8,3,0.014553600549697876
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,float16,7,0.01658719927072525
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,fp8,7,0.014582400023937226
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,float16,15,0.017228800058364867
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,fp8,15,0.014575999975204468
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,float16,31,0.01685120016336441
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,fp8,31,0.014620800316333771
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,float16,63,0.017020800709724428
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,fp8,63,0.014679999649524688
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,float16,127,0.017347200214862822
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,fp8,127,0.01459999978542328
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,float16,255,0.02735520005226135
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,fp8,255,0.014585599303245544
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,float16,511,0.045974400639533994
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,fp8,511,0.02828960120677948
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,float16,1023,0.08276000022888183
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,fp8,1023,0.047646400332450864
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,fp8,15,0.011444800347089768
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,float16,2047,0.15633440017700195
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,fp8,2047,0.08526399731636047
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,float16,1,0.01255040019750595
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,fp8,1,0.010791999846696853
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,fp8,4095,0.1603360056877136
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,float16,4095,0.3011215925216675
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,float16,3,0.012468799948692322
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,fp8,3,0.0110384002327919
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,fp8,7,0.012452799826860428
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,float16,7,0.012441600114107132
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,float16,15,0.01252480000257492
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,fp8,15,0.012390399724245072
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,float16,31,0.012468799948692322
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,fp8,31,0.011383999884128571
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,fp8,63,0.010795199871063232
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,fp8,4095,0.08512160181999207
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,fp8,127,0.012417600303888322
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,float16,255,0.012470400333404541
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,fp8,255,0.011524800211191177
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,float16,511,0.02886880040168762
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,fp8,511,0.014567999541759491
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,float16,1023,0.046998399496078494
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,fp8,1023,0.029311999678611755
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,float16,2047,0.08436959981918335
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,fp8,2047,0.04763840138912201
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,float16,1,0.01679999977350235
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,fp8,1,0.014580799639225006
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,float16,3,0.01735360026359558
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,fp8,3,0.01462399959564209
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,float16,7,0.017399999499320983
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,fp8,7,0.014601600170135499
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,float16,15,0.018187199532985688
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,fp8,15,0.015523199737071992
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,float16,31,0.017785599827766417
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,fp8,31,0.015022400021553039
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,float16,63,0.017795200645923614
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,fp8,63,0.0151296004652977
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,float16,127,0.018211199343204497
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,fp8,127,0.015440000593662262
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,float16,255,0.027611199021339416
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,fp8,255,0.014670400321483612
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,float16,511,0.04610239863395691
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,fp8,511,0.02882080078125
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,float16,1023,0.08217759728431702
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,fp8,1023,0.048870399594306946
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,float16,2047,0.1545632004737854
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,fp8,2047,0.08460000157356262
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,float16,127,0.01252799928188324
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,float16,1,0.027799999713897704
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,float16,3,0.027561599016189577
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,fp8,3,0.023287999629974365
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,float16,7,0.02687999904155731
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,fp8,7,0.022767999768257143
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,fp8,1023,0.028995200991630554
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,float16,15,0.02689119875431061
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,float16,2047,0.0833407998085022
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,fp8,15,0.022777600586414336
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,float16,31,0.02688319981098175
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,float16,63,0.027153599262237548
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,fp8,63,0.02279839962720871
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,float16,63,0.012467200309038163
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,float16,4095,0.1574496030807495
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,float16,255,0.04703199863433838
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,fp8,255,0.02924480140209198
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,float16,511,0.08265280127525329
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,float16,1023,0.15809119939804078
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,fp8,1023,0.08453440070152282
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,float16,2047,0.3014688014984131
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,fp8,2047,0.15812959671020507
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,float16,1,0.008446399867534638
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,fp8,1,0.008347199857234954
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,float16,3,0.008430399745702744
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,fp8,3,0.00836160033941269
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,float16,7,0.008416000008583068
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,fp8,7,0.00843840017914772
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,float16,15,0.00835840031504631
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,fp8,15,0.008428800106048583
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,float16,31,0.008441600203514098
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,fp8,31,0.008448000252246856
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,float16,63,0.008433599770069123
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,fp8,1,0.023119999468326567
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,float16,127,0.008390399813652038
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,fp8,127,0.008455999940633774
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,float16,255,0.008345600217580795
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,fp8,255,0.00846880003809929
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,float16,511,0.008428800106048583
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,fp8,511,0.008459199965000153
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,float16,1023,0.009369599819183349
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,fp8,1023,0.008452799916267396
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,float16,2047,0.010496000200510025
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,fp8,2047,0.0104032002389431
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,float16,4095,0.01242239996790886
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,fp8,4095,0.0104592002928257
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,fp8,31,0.022776000201702118
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,fp8,8191,0.0124208003282547
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,float16,8191,0.013748799264431
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,fp8,127,0.02280319929122925
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,float16,127,0.012564800679683685
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,fp8,16383,0.014494399726390838
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,float16,32767,0.030958399176597595
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,fp8,32767,0.016575999557971954
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,float16,1,0.008449599891901017
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,fp8,1,0.008379200100898742
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,float16,3,0.008430399745702744
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,fp8,3,0.008363199979066848
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,fp8,7,0.008462399989366532
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,float16,15,0.00846880003809929
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,fp8,15,0.008396799862384795
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,float16,31,0.00835999995470047
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,fp8,31,0.008377599716186523
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,fp8,511,0.047460800409317015
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,float16,63,0.008446399867534638
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,fp8,63,0.008374399691820144
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,float16,127,0.00844319984316826
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,fp8,127,0.008491200208663941
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,fp8,255,0.008366400003433227
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,float16,511,0.008408000320196151
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,fp8,511,0.008454400300979614
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,float16,1023,0.008561599999666214
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,fp8,1023,0.008430399745702744
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,float16,2047,0.010420800000429154
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,fp8,2047,0.01040479987859726
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,float16,4095,0.012457600235939026
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,float16,8191,0.014582400023937226
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,fp8,8191,0.012555199861526489
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,float16,16383,0.030044800043106078
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,fp8,16383,0.015281599760055543
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,float16,32767,0.04824639856815338
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,float16,1,0.008369600027799606
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,fp8,1,0.008347199857234954
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,float16,3,0.008303999900817871
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,fp8,3,0.008323200047016144
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,float16,7,0.008425600081682205
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,fp8,7,0.008355200290679932
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,float16,15,0.008403199911117553
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,fp8,15,0.00833280012011528
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,float16,31,0.008355200290679932
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,fp8,31,0.008423999696969987
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,float16,63,0.00841120034456253
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,fp8,63,0.008428800106048583
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,float16,127,0.008433599770069123
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,fp8,127,0.00841279998421669
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,float16,255,0.008417599648237229
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,fp8,63,0.008390399813652038
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,fp8,255,0.00835840031504631
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,float16,511,0.008355200290679932
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,fp8,511,0.008345600217580795
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,float16,1023,0.008844800293445587
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,fp8,1023,0.008475200086832047
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,float16,2047,0.01048159971833229
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,fp8,2047,0.00931520015001297
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,float16,4095,0.012441600114107132
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,fp8,4095,0.010406400263309478
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,float16,16383,0.01650079935789108
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,float16,127,0.030856001377105712
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,float16,8191,0.02715199887752533
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,fp8,8191,0.014560000598430633
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,fp8,16383,0.02874560058116913
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,float16,7,0.00841120034456253
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,float16,32767,0.0820688009262085
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,float16,1,0.01661760061979294
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,fp8,32767,0.04724319875240326
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,float16,3,0.016616000235080718
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,fp8,3,0.01653439998626709
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,float16,255,0.008364800363779068
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,float16,7,0.01764640063047409
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,fp8,7,0.01653600037097931
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,float16,15,0.016651199758052827
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,fp8,15,0.016663999855518342
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,float16,31,0.018377600610256194
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,fp8,31,0.01645279973745346
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,float16,63,0.01786559969186783
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,fp8,4095,0.011023999750614166
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,fp8,63,0.016603200137615202
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,float16,127,0.016862399876117706
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,fp8,127,0.01658560037612915
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,fp8,255,0.01653600037097931
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,float16,511,0.047056001424789426
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,fp8,511,0.030326399207115173
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,float16,1,0.02739039957523346
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,float16,1023,0.08382880091667175
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,fp8,1023,0.04901280105113983
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,fp8,32767,0.031176000833511353
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,fp8,1,0.0247296005487442
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,float16,3,0.02687999904155731
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,fp8,3,0.024719999730587007
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,fp8,7,0.024835200607776643
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,float16,15,0.026878398656845093
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,fp8,15,0.024747200310230255
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,float16,31,0.026984000205993654
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,fp8,31,0.02441119998693466
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,float16,63,0.028774398565292358
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,fp8,63,0.02422720044851303
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,fp8,127,0.024751999974250795
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,float16,255,0.04710240066051483
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,fp8,255,0.03017120063304901
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,float16,511,0.08297439813613891
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,fp8,511,0.048764801025390624
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,fp8,1023,0.08615360260009766
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,float16,1023,0.1554703950881958
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,float16,1,0.047804799675941465
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,fp8,1,0.03978880047798157
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,float16,3,0.04748480021953583
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,fp8,3,0.03937279880046844
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,float16,7,0.04742560088634491
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,fp8,7,0.03971680104732513
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,float16,15,0.047363200783729555
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,fp8,15,0.040300801396369934
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,float16,31,0.04841119945049286
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,fp8,31,0.04037440121173859
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,float16,63,0.04937120079994202
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,fp8,63,0.04050720036029816
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,float16,127,0.051553601026535036
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,fp8,127,0.0418287992477417
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,float16,255,0.08419679999351501
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,fp8,255,0.050267201662063596
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,float16,16383,0.04612320065498352
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,float16,511,0.1566815972328186
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,fp8,1,0.016606399416923524
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,float16,1,0.027585598826408386
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,float16,1023,0.30476961135864256
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,fp8,1,0.024851199984550477
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,fp8,3,0.024804799258708952
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,float16,255,0.028814399242401124
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,float16,3,0.028859201073646545
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,float16,7,0.02792479991912842
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,fp8,7,0.024831999838352204
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,float16,15,0.028755199909210206
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,fp8,15,0.024774399399757386
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,fp8,31,0.024772800505161285
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,float16,31,0.02821600139141083
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,float16,63,0.02890239953994751
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,float16,7,0.027198401093482972
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,fp8,63,0.024796800315380098
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,fp8,127,0.02481279969215393
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,float16,255,0.047491198778152464
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,fp8,255,0.030902400612831116
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,float16,127,0.030854400992393494
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,float16,1,0.047547200322151185
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,float16,511,0.0843936026096344
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,fp8,3,0.04119200110435486
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,float16,3,0.04779039919376373
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,float16,7,0.04753440022468567
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,fp8,1,0.04113920032978058
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,fp8,7,0.04120480120182037
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,float16,15,0.04737440049648285
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,fp8,15,0.04116320013999939
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,float16,31,0.04872959852218628
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,float16,63,0.04940800070762634
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,fp8,63,0.041124799847602846
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,float16,127,0.051611202955245974
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,fp8,127,0.04315840005874634
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,float16,255,0.08439520001411438
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,fp8,255,0.05215200185775757
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,float16,511,0.15732159614562988
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,float16,1,0.08797439932823181
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,fp8,1,0.07247200012207031
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,float16,3,0.0876688003540039
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,fp8,3,0.07240319848060608
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,fp8,511,0.0855455994606018
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,float16,7,0.08740800023078918
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,float16,15,0.0887008011341095
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,fp8,15,0.0722656011581421
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,fp8,1023,0.1593616008758545
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,float16,31,0.08867999911308289
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,fp8,31,0.07344480156898499
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,float16,63,0.08907840251922608
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,float16,127,0.030888000130653383
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,float16,127,0.09339519739151
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,fp8,127,0.07513759732246399
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,fp8,511,0.04969440102577209
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,float16,255,0.15877120494842528
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,fp8,255,0.09097920060157776
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,float16,1,0.008483199775218964
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,fp8,1,0.00843840017914772
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,fp8,31,0.041198399662971494
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,fp8,3,0.008353599905967712
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,fp8,511,0.16134719848632811
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,float16,7,0.008449599891901017
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,fp8,7,0.008399999886751174
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,float16,511,0.3036623954772949
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,fp8,15,0.008444800227880477
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,float16,31,0.008326400071382523
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,float16,15,0.00843520015478134
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,float16,63,0.008408000320196151
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,fp8,63,0.00846560001373291
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,float16,127,0.008406399935483932
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,fp8,31,0.008371199667453765
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,fp8,127,0.008380799740552902
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,float16,511,0.008444800227880477
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,fp8,511,0.008403199911117553
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,float16,1023,0.008448000252246856
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,fp8,1023,0.008432000130414962
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,float16,2047,0.010414399951696397
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,fp8,2047,0.010393600165843963
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,fp8,511,0.08708320260047912
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,float16,4095,0.012443199753761292
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,fp8,4095,0.010467199981212616
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,fp8,8191,0.012516799569129943
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,float16,16383,0.028977599740028382
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,fp8,16383,0.014571200311183929
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,float16,32767,0.047460800409317015
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,float16,1,0.00833119973540306
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,fp8,32767,0.03056640028953552
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,fp8,1,0.008327999711036682
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,float16,3,0.008345600217580795
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,fp8,3,0.00833280012011528
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,fp8,7,0.00835679993033409
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,float16,15,0.008371199667453765
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,fp8,15,0.00836160033941269
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,float16,31,0.008339200168848038
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,fp8,31,0.008382400125265121
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,fp8,7,0.0725920021533966
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,float16,63,0.00840959995985031
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,fp8,63,0.008401600271463394
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,float16,127,0.008353599905967712
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,float16,255,0.00843840017914772
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,fp8,255,0.008355200290679932
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,float16,511,0.008392000198364257
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,fp8,511,0.008350399881601333
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,float16,1023,0.008417599648237229
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,fp8,1023,0.008454400300979614
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,float16,2047,0.010492800176143647
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,fp8,2047,0.01040000021457672
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,float16,4095,0.012515200674533844
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,fp8,4095,0.010425599664449692
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,float16,8191,0.02748799920082092
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,fp8,8191,0.013484799861907959
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,float16,16383,0.04529440104961395
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,float16,3,0.008441600203514098
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,fp8,16383,0.028782400488853454
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,float16,32767,0.08194239735603333
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,fp8,32767,0.0474480003118515
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,float16,1,0.008401600271463394
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,fp8,1,0.008343999832868576
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,float16,255,0.008441600203514098
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,fp8,3,0.008347199857234954
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,fp8,255,0.008425600081682205
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,fp8,7,0.008446399867534638
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,fp8,15,0.008364800363779068
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,float16,31,0.00840959995985031
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,fp8,31,0.008422400057315826
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,float16,63,0.00843999981880188
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,fp8,63,0.008423999696969987
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,float16,127,0.008408000320196151
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,float16,7,0.008382400125265121
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,fp8,127,0.008353599905967712
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,float16,255,0.00843999981880188
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,fp8,255,0.00844319984316826
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,float16,511,0.008348800241947174
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,fp8,511,0.00835679993033409
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,float16,1023,0.010360000282526016
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,fp8,127,0.008470399677753449
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,fp8,1023,0.008457600325345992
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,float16,2047,0.012511999905109405
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,fp8,2047,0.010454399883747101
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,float16,4095,0.028921601176261903
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,fp8,4095,0.012566399574279786
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,fp8,63,0.07402560114860535
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,float16,8191,0.046412798762321475
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,fp8,8191,0.02861759960651398
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,fp8,16383,0.04720639884471893
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,float16,16383,0.08205119967460632
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,float16,32767,0.15565760135650636
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,fp8,32767,0.08299840092658997
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,1,0.008371199667453765
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,3,0.008380799740552902
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,7,0.008454400300979614
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,1,0.008353599905967712
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,7,0.008363199979066848
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,15,0.008472000062465668
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,31,0.008475200086832047
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,31,0.00835840031504631
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,63,0.00843999981880188
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,63,0.008448000252246856
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,float16,3,0.0083856001496315
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,127,0.008441600203514098
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,127,0.008396799862384795
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,float16,15,0.008345600217580795
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,255,0.008475200086832047
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,255,0.008364800363779068
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,511,0.008763200044631958
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,511,0.00844319984316826
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,1023,0.010452800244092942
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,2047,0.012692800164222718
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,2047,0.010487999767065048
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,4095,0.02884480059146881
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,float16,8191,0.014455999433994293
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,4095,0.014460800588130951
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,8191,0.045921599864959715
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,16383,0.08205599784851074
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,32767,0.15365920066833497
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,32767,0.08295999765396118
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,65535,0.29667038917541505
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,65535,0.15487040281295777
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,1,0.008425600081682205
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,1,0.008392000198364257
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,3,0.008367999643087386
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,3,0.00838399976491928
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,7,0.008380799740552902
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,7,0.00841279998421669
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,15,0.008337599784135818
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,15,0.008316799998283386
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,31,0.00835840031504631
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,31,0.00838399976491928
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,15,0.00835840031504631
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,63,0.008395200222730636
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,63,0.008379200100898742
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,127,0.008336000144481659
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,127,0.008321599662303924
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,255,0.008425600081682205
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,255,0.008343999832868576
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,511,0.00841120034456253
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,1023,0.011025600135326385
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,1023,0.010345599800348281
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,2047,0.02696479856967926
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,2047,0.012483199685811996
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,4095,0.04521439969539642
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,4095,0.028755199909210206
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,1023,0.010377600044012069
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,8191,0.08042240142822266
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,8191,0.04627679884433746
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,16383,0.15341919660568237
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,8191,0.028884801268577575
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,16383,0.08257279992103576
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,16383,0.04715839922428131
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,1,0.008430399745702744
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,32767,0.29631359577178956
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,1,0.00840959995985031
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,3,0.008436799794435502
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,3,0.008422400057315826
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,7,0.008432000130414962
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,7,0.008414400368928909
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,15,0.008446399867534638
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,15,0.008420799672603608
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,65535,0.3027328014373779
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,65535,0.5820271968841553
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,31,0.008347199857234954
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,127,0.008372800052165985
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,63,0.008352000266313553
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,127,0.00843520015478134
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,63,0.008392000198364257
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,255,0.008350399881601333
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,255,0.008312000334262848
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,511,0.008326400071382523
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,511,0.008340799808502197
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,1023,0.008420799672603608
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,1023,0.00840959995985031
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,2047,0.008715199679136277
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,2047,0.00883840024471283
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,4095,0.01043199971318245
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,8191,0.012539200484752655
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,4095,0.009996800124645234
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,8191,0.012513600289821625
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,16383,0.014571200311183929
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,16383,0.014593599736690522
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,32767,0.016523200273513793
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,32767,0.016631999611854555
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,65535,0.01858399957418442
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,3,0.00844319984316826
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,1,0.008366400003433227
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,1,0.008513599634170532
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,3,0.00835679993033409
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,3,0.008452799916267396
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,7,0.00841279998421669
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,7,0.008419200032949447
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,15,0.008353599905967712
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,15,0.008425600081682205
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,31,0.00836160033941269
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,31,0.008427199721336365
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,63,0.008425600081682205
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,63,0.008462399989366532
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,127,0.008417599648237229
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,127,0.008463999629020691
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,255,0.008432000130414962
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,255,0.008433599770069123
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,511,0.00883679986000061
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,511,0.008470399677753449
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,1023,0.008403199911117553
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,2047,0.010462400317192078
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,2047,0.008462399989366532
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,4095,0.01043199971318245
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,4095,0.010443200170993806
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,8191,0.012505599856376648
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,8191,0.012529599666595458
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,16383,0.01664319932460785
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,32767,0.1559376001358032
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,32767,0.01860000044107437
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,32767,0.01661919951438904
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,65535,0.020627200603485107
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,31,0.008371199667453765
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,65535,0.018648000061511995
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,1,0.008460800349712371
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,1,0.008329600095748901
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,3,0.008430399745702744
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,3,0.008364800363779068
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,7,0.00843840017914772
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,7,0.008352000266313553
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,15,0.008425600081682205
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,15,0.008420799672603608
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,31,0.00844319984316826
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,31,0.008353599905967712
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,63,0.008452799916267396
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,63,0.008337599784135818
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,127,0.008444800227880477
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,127,0.008350399881601333
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,255,0.00836160033941269
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,255,0.00835840031504631
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,511,0.008422400057315826
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,1023,0.008446399867534638
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,1023,0.008459199965000153
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,2047,0.010385599732398988
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,2047,0.00844319984316826
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,4095,0.010377600044012069
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,4095,0.010382399708032609
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,8191,0.012521600723266602
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,8191,0.012491200119256973
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,16383,0.0166143998503685
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,float16,7,0.00838399976491928
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,16383,0.016596800088882445
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,1023,0.008422400057315826
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,32767,0.016956800222396852
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,65535,0.020262399315834047
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,1,0.008348800241947174
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,1,0.008347199857234954
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,3,0.008473599702119828
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,3,0.008427199721336365
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,7,0.008444800227880477
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,7,0.008449599891901017
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,15,0.008436799794435502
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,15,0.008425600081682205
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,31,0.008399999886751174
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,63,0.008432000130414962
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,16383,0.01659200042486191
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,63,0.008436799794435502
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,127,0.008433599770069123
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,255,0.008497600257396699
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,255,0.008449599891901017
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,511,0.00843999981880188
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,511,0.008463999629020691
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,1023,0.008544000238180161
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,1023,0.008451200276613235
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,2047,0.009779199957847595
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,2047,0.00851840004324913
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,4095,0.010529600083827972
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,511,0.008455999940633774
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,4095,0.0104592002928257
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,8191,0.014473600685596466
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,8191,0.012555199861526489
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,65535,0.016607999801635742
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,16383,0.016518400609493257
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,16383,0.014595200121402741
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,32767,0.018590399622917177
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,32767,0.016676799952983858
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,65535,0.03475199937820435
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,1,0.008396799862384795
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,1,0.00836160033941269
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,511,0.010123199969530105
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,3,0.008364800363779068
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,7,0.008430399745702744
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,3,0.008374399691820144
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,7,0.008436799794435502
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,15,0.008374399691820144
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,32767,0.018559999763965607
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,15,0.00836160033941269
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,31,0.008363199979066848
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,31,0.00835679993033409
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,63,0.008462399989366532
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,65535,0.018646399676799773
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,127,0.008422400057315826
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,127,0.00844319984316826
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,63,0.008342400193214417
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,255,0.0083856001496315
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,511,0.00841120034456253
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,511,0.008369600027799606
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,1023,0.00835840031504631
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,1023,0.008380799740552902
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,2047,0.010385599732398988
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,4095,0.012875199317932129
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,4095,0.010471999645233154
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,8191,0.02754560112953186
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,8191,0.014112000167369843
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,31,0.008432000130414962
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,16383,0.045281600952148435
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,16383,0.02882080078125
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,127,0.008432000130414962
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,32767,0.08175519704818726
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,32767,0.047056001424789426
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,1,0.008353599905967712
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,1,0.008432000130414962
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,65535,0.15391360521316527
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,3,0.008427199721336365
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,3,0.008342400193214417
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,7,0.008396799862384795
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,15,0.008392000198364257
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,7,0.008448000252246856
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,15,0.008327999711036682
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,31,0.008427199721336365
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,31,0.008486399799585343
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,63,0.008455999940633774
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,63,0.008430399745702744
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,127,0.008449599891901017
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,255,0.008336000144481659
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,127,0.008451200276613235
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,255,0.008455999940633774
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,511,0.008968000113964082
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,511,0.00841120034456253
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,1023,0.010420800000429154
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,1023,0.009974399954080582
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,2047,0.012444800138473511
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,2047,0.010494399815797806
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,4095,0.027987200021743774
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,4095,0.013867199420928955
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,8191,0.04549440145492554
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,8191,0.028838399052619933
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,16383,0.08184319734573364
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,16383,0.04643679857254028
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,32767,0.15360959768295288
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,32767,0.08394399881362916
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,1,0.008711999654769898
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,1,0.008416000008583068
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,65535,0.2970479965209961
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,3,0.008451200276613235
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,65535,0.15659840106964112
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,7,0.00843999981880188
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,65535,0.018643200397491455
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,3,0.00846719965338707
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,15,0.008399999886751174
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,7,0.008430399745702744
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,31,0.008367999643087386
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,63,0.008433599770069123
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,63,0.008374399691820144
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,127,0.00843520015478134
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,127,0.008369600027799606
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,255,0.008460800349712371
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,255,0.008345600217580795
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,511,0.00846719965338707
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,511,0.008390399813652038
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,1023,0.008436799794435502
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,1023,0.008377599716186523
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,2047,0.010390400141477584
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,2047,0.008588799834251403
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,4095,0.010425599664449692
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,2047,0.0095551997423172
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,8191,0.013459199666976928
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,8191,0.012468799948692322
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,16383,0.014620800316333771
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,16383,0.014561599493026734
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,32767,0.018555200099945067
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,32767,0.016539199650287627
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,65535,0.03497599959373474
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,65535,0.017689600586891174
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,65535,0.0837664008140564
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,1,0.008337599784135818
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,1,0.00843999981880188
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,3,0.008433599770069123
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,3,0.008454400300979614
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,7,0.008459199965000153
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,7,0.008419200032949447
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,15,0.008452799916267396
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,15,0.008404800295829773
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,31,0.00843840017914772
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,63,0.00846880003809929
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,63,0.008473599702119828
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,127,0.008481600135564805
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,127,0.00843520015478134
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,255,0.008638399839401244
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,255,0.00835679993033409
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,511,0.009137599915266036
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,511,0.008432000130414962
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,1023,0.008433599770069123
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,1023,0.00843520015478134
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,2047,0.010497599840164185
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,2047,0.010316800326108932
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,4095,0.0104592002928257
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,8191,0.012865599989891053
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,8191,0.012516799569129943
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,16383,0.015620799362659454
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,16383,0.014190399646759033
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,32767,0.030934399366378783
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,32767,0.01690399944782257
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,31,0.008449599891901017
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,65535,0.04930559992790222
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,float16,1,0.00833439975976944
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,65535,0.032548800110816956
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,fp8,1,0.008372800052165985
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,float16,3,0.008350399881601333
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,fp8,3,0.00833439975976944
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,float16,7,0.008329600095748901
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,fp8,7,0.00833119973540306
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,255,0.008350399881601333
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,fp8,15,0.00835679993033409
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,4095,0.010548800230026245
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,float16,31,0.008376000076532364
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,fp8,31,0.008320000022649765
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,float16,63,0.008372800052165985
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,fp8,63,0.008363199979066848
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,float16,127,0.008337599784135818
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,fp8,127,0.00835840031504631
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,float16,255,0.008454400300979614
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,fp8,255,0.008300799876451492
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,fp8,511,0.008387199789285659
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,float16,1023,0.01120320037007332
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,fp8,1023,0.010384000092744827
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,float16,2047,0.026849600672721862
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,fp8,2047,0.012547199428081513
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,fp8,4095,0.02873600125312805
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,float16,4095,0.045203199982643126
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,float16,8191,0.08137279748916626
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,float16,16383,0.15302560329437256
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,31,0.008446399867534638
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,fp8,16383,0.08228160142898559
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,float16,32767,0.29657440185546874
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,fp8,32767,0.15435839891433717
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,fp8,1,0.008769600093364716
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,float16,1,0.010395199805498124
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,4095,0.011934400349855424
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,float16,3,0.010046400129795074
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,fp8,3,0.008683200180530547
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,float16,15,0.010398399829864503
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,fp8,7,0.008551999926567078
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,float16,31,0.010444799810647965
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,fp8,31,0.0084927998483181
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,float16,63,0.010390400141477584
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,fp8,63,0.008428800106048583
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,float16,127,0.010387200117111205
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,fp8,127,0.00841120034456253
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,float16,255,0.010470400005578995
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,float16,511,0.01247360035777092
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,15,0.008422400057315826
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,fp8,511,0.01043040007352829
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,float16,1023,0.0267984002828598
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,fp8,1023,0.014260800182819366
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,fp8,2047,0.02879520058631897
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,float16,4095,0.08215680122375488
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,float16,15,0.008321599662303924
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,fp8,4095,0.04763999879360199
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,fp8,8191,0.08452640175819397
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,float16,8191,0.1540832042694092
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,float16,511,0.00968639999628067
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,fp8,16383,0.15965440273284912
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,fp8,8191,0.045824000239372255
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,float16,1,0.009433600306510925
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,fp8,1,0.008448000252246856
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,float16,32767,0.5896383762359619
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,float16,3,0.009884800016880035
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,fp8,32767,0.31120638847351073
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,fp8,3,0.008488000184297562
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,fp8,7,0.008433599770069123
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,float16,7,0.010409600287675857
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,float16,15,0.01043360009789467
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,fp8,15,0.008408000320196151
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,float16,31,0.010377600044012069
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,fp8,31,0.009068799763917923
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,float16,63,0.01040000021457672
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,fp8,63,0.008724799752235413
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,float16,127,0.010395199805498124
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,fp8,127,0.00840959995985031
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,float16,255,0.01048479974269867
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,fp8,15,0.009379199892282485
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,fp8,255,0.008460800349712371
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,float16,511,0.012515200674533844
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,fp8,511,0.010384000092744827
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,float16,1023,0.02741599977016449
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,fp8,1023,0.01430879980325699
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,float16,2047,0.045491200685501096
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,fp8,2047,0.02871519923210144
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,float16,4095,0.08210240006446838
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,fp8,255,0.008441600203514098
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,float16,1,0.012559999525547028
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,float16,8191,0.1546512007713318
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,fp8,1,0.01045759990811348
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,fp8,8191,0.08388159871101379
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,float16,2047,0.04581280052661896
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,float16,7,0.012435200065374375
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,float16,15,0.012401600182056428
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,fp8,15,0.010502400249242783
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,float16,31,0.012452799826860428
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,fp8,31,0.01048159971833229
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,float16,63,0.01247519999742508
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,fp8,63,0.010476800054311753
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,float16,127,0.012460800260305405
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,fp8,127,0.011502400040626526
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,float16,255,0.012494400143623352
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,fp8,255,0.011444800347089768
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,float16,16383,0.29967679977416994
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,float16,511,0.02789599895477295
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,fp8,511,0.014537599682807923
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,float16,1023,0.046312001347541806
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,fp8,1023,0.028892800211906433
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,float16,2047,0.08232640027999878
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,fp8,2047,0.04728800058364868
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,float16,4095,0.15579040050506593
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,fp8,4095,0.0841488003730774
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,fp8,8191,0.15843199491500853
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,float16,7,0.01037440001964569
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,float16,8191,0.30317599773406984
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,float16,1,0.012481600046157837
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,fp8,1,0.01239359974861145
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,fp8,3,0.012459199875593185
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,float16,3,0.012539200484752655
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,fp8,7,0.012476799637079239
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,float16,15,0.012436799705028534
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,fp8,15,0.012464000284671784
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,fp8,31,0.011115200072526931
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,fp8,4095,0.047152000665664676
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,float16,63,0.012491200119256973
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,float16,127,0.0124719999730587
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,fp8,127,0.011240000277757645
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,fp8,3,0.010451199859380722
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,fp8,7,0.010689599812030793
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,fp8,255,0.010486400127410889
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,float16,511,0.028233599662780762
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,fp8,511,0.01459839940071106
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,fp8,1023,0.029150399565696716
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,float16,2047,0.08314719796180725
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,fp8,2047,0.04766719937324524
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,fp8,4095,0.08394560217857361
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,float16,4095,0.15677119493484498
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,float16,1,0.016820800304412842
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,fp8,1,0.014608000218868256
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,float16,3,0.016601599752902985
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,fp8,3,0.014691199362277984
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,float16,7,0.016612799465656282
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,fp8,7,0.014603200554847717
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,float16,15,0.016628800332546233
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,float16,31,0.016624000668525696
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,fp8,31,0.014668799936771393
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,float16,63,0.017553600668907165
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,fp8,63,0.014606399834156037
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,float16,127,0.01793439984321594
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,fp8,127,0.01456640064716339
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,float16,255,0.027559998631477355
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,fp8,255,0.0147599995136261
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,float16,511,0.045396798849105836
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,float16,7,0.012545600533485413
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,fp8,511,0.028591999411582948
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,float16,1023,0.08215680122375488
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,fp8,63,0.010684800148010255
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,float16,2047,0.15513600111007692
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,float16,255,0.012454400211572647
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,fp8,2047,0.0857200026512146
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,1,0.0083856001496315
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,1,0.00835840031504631
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,float16,4095,0.3002608060836792
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,fp8,4095,0.15942879915237426
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,3,0.008441600203514098
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,float16,1023,0.0468176007270813
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,7,0.00851840004324913
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,3,0.00833439975976944
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,15,0.008454400300979614
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,7,0.008406399935483932
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,31,0.008427199721336365
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,31,0.008441600203514098
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,63,0.008430399745702744
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,63,0.008500800281763077
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,127,0.008345600217580795
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,127,0.008427199721336365
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,255,0.008446399867534638
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,255,0.00844319984316826
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,fp8,15,0.014590400457382201
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,511,0.008379200100898742
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,1023,0.008540800213813782
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,1023,0.008462399989366532
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,2047,0.010384000092744827
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,2047,0.010438399761915207
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,4095,0.011112000048160552
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,4095,0.010446400195360184
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,8191,0.012464000284671784
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,8191,0.012495999783277511
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,16383,0.014608000218868256
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,16383,0.014295999705791474
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,32767,0.016631999611854555
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,32767,0.03097119927406311
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,65535,0.05130559802055359
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,65535,0.031171199679374696
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,1,0.008694399893283845
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,3,0.008780799806118011
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,3,0.008504000306129456
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,7,0.008399999886751174
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,float16,31,0.012435200065374375
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,7,0.008420799672603608
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,15,0.008449599891901017
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,15,0.008406399935483932
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,31,0.009099200367927551
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,31,0.008376000076532364
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,63,0.009344000369310379
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,63,0.008473599702119828
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,float16,3,0.01247519999742508
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,127,0.00881119966506958
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,255,0.008337599784135818
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,255,0.008615999668836593
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,511,0.009009599685668945
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,1023,0.009809599816799163
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,1023,0.009140799939632415
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,2047,0.010436800122261048
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,15,0.00835679993033409
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,2047,0.010384000092744827
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,4095,0.012534399330615998
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,4095,0.010496000200510025
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,8191,0.014494399726390838
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,8191,0.012462399899959564
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,16383,0.030795198678970338
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,16383,0.015094399452209473
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,32767,0.04885599911212921
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,32767,0.030766400694847106
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,65535,0.0841871976852417
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,511,0.008454400300979614
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,65535,0.04921759963035584
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,float16,1,0.017310400307178498
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,fp8,1,0.015652799606323244
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,float16,3,0.016651199758052827
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,fp8,3,0.01653600037097931
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,float16,7,0.01735839992761612
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,float16,15,0.01838880032300949
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,fp8,15,0.016550399363040924
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,float16,31,0.016731199622154237
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,fp8,31,0.016553600132465363
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,float16,63,0.01685280054807663
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,fp8,63,0.016523200273513793
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,float16,127,0.01676799952983856
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,fp8,127,0.01656000018119812
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,float16,255,0.027561599016189577
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,fp8,255,0.016540800034999848
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,float16,511,0.04642559885978699
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,fp8,511,0.029105600714683533
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,float16,1023,0.08220000267028808
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,fp8,1023,0.04774560034275055
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,1,0.008455999940633774
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,float16,2047,0.15559840202331543
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,float16,1,0.027102398872375488
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,fp8,1,0.024201600253582
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,fp8,2047,0.08523039817810059
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,float16,3,0.026982399821281432
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,float16,7,0.02688319981098175
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,fp8,3,0.02463040053844452
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,fp8,7,0.024779200553894043
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,fp8,15,0.024403199553489685
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,float16,31,0.026878398656845093
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,fp8,31,0.024751999974250795
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,float16,63,0.028812798857688903
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,float16,127,0.03078559935092926
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,fp8,127,0.024716800451278685
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,127,0.010089600086212158
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,float16,255,0.047177600860595706
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,fp8,255,0.030638399720191955
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,float16,511,0.08256959915161133
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,fp8,511,0.04845280051231384
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,float16,1023,0.15424799919128418
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,fp8,1023,0.08546720147132873
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,fp8,2047,0.15819040536880494
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,float16,2047,0.29913599491119386
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,float16,1,0.027771198749542238
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,float16,3,0.027521601319313048
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,fp8,3,0.024820800125598907
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,float16,7,0.028255999088287354
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,fp8,7,0.024884800612926482
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,fp8,7,0.01645440012216568
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,float16,15,0.02847520112991333
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,fp8,15,0.024859200417995452
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,float16,31,0.028563201427459717
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,float16,63,0.028702399134635924
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,fp8,31,0.024827200174331664
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,fp8,63,0.02480800002813339
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,float16,127,0.03096800148487091
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,fp8,127,0.024849599599838255
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,float16,255,0.04733920097351074
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,fp8,255,0.03096800148487091
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,float16,511,0.08379520177841186
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,fp8,511,0.049184000492095946
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,float16,1023,0.1557919979095459
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,fp8,1023,0.047727999091148374
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,float16,15,0.027296000719070436
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,float16,1,0.04779039919376373
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,fp8,1,0.041196799278259276
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,fp8,63,0.02444480061531067
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,float16,7,0.04756479859352112
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,fp8,3,0.0412304013967514
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,float16,15,0.04808320105075836
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,fp8,15,0.04111199975013733
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,fp8,31,0.04115520119667053
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,511,0.00846560001373291
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,float16,63,0.04937599897384644
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,fp8,63,0.04116640090942383
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,float16,127,0.051148802042007446
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,fp8,127,0.04329439997673035
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,float16,255,0.0837552011013031
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,fp8,255,0.051844799518585206
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,float16,511,0.1556447982788086
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,fp8,511,0.0864512026309967
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,fp8,1,0.024833600223064422
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,1,0.008448000252246856
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,1,0.008327999711036682
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,float16,1023,0.29886560440063475
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,3,0.008476799726486206
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,7,0.008432000130414962
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,7,0.008364800363779068
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,15,0.008457600325345992
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,15,0.008457600325345992
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,31,0.008774399757385254
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,63,0.00843840017914772
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,63,0.008366400003433227
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,127,0.008436799794435502
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,127,0.008376000076532364
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,255,0.008451200276613235
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,255,0.00843840017914772
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,511,0.00843999981880188
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,fp8,1023,0.08616480231285095
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,511,0.008376000076532364
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,1023,0.008433599770069123
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,1023,0.0083856001496315
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,float16,3,0.047572800517082216
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,2047,0.010491199791431427
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,2047,0.01043360009789467
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,4095,0.01244800016283989
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,8191,0.014572800695896148
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,fp8,7,0.04113920032978058
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,8191,0.012483199685811996
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,16383,0.030768001079559328
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,16383,0.014830400049686433
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,32767,0.04803839921951294
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,32767,0.03105599880218506
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,float16,31,0.04877600073814392
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,65535,0.08440319895744323
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,65535,0.04953919947147369
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,3,0.008342400193214417
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,1,0.008419200032949447
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,3,0.008367999643087386
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,7,0.00833439975976944
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,7,0.00846560001373291
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,15,0.008425600081682205
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,15,0.008339200168848038
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,31,0.008459199965000153
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,31,0.00844319984316826
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,63,0.00844319984316826
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,127,0.008417599648237229
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,127,0.00841279998421669
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,255,0.008433599770069123
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,255,0.008427199721336365
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,511,0.008363199979066848
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,511,0.00841120034456253
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,1023,0.009467200189828873
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,1023,0.008455999940633774
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,2047,0.01034879982471466
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,2047,0.009454400092363358
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,4095,0.012464000284671784
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,4095,0.010489600151777268
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,8191,0.02739039957523346
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,fp8,1023,0.15994880199432374
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,8191,0.0144896000623703
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,16383,0.045263999700546266
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,16383,0.02882719933986664
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,3,0.008423999696969987
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,32767,0.08171520233154297
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,32767,0.04756639897823334
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,1,0.008433599770069123
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,31,0.008369600027799606
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,65535,0.1540511965751648
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,65535,0.08565279841423035
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,3,0.008388800173997879
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,3,0.008462399989366532
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,7,0.008420799672603608
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,7,0.008353599905967712
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,15,0.008476799726486206
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,15,0.008414400368928909
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,31,0.008423999696969987
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,31,0.008352000266313553
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,63,0.0083856001496315
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,63,0.008380799740552902
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,127,0.008454400300979614
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,127,0.00846560001373291
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,255,0.008463999629020691
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,511,0.008591999858617782
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,511,0.008481600135564805
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,1023,0.010441599786281586
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,1023,0.010427200049161912
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,2047,0.012454400211572647
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,2047,0.010435199737548828
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,4095,0.011020799726247787
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,4095,0.02787359952926636
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,4095,0.014305600523948669
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,8191,0.045691201090812684
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,8191,0.028860801458358766
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,16383,0.08206239938735962
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,16383,0.04642400145530701
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,1,0.008342400193214417
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,32767,0.15380159616470337
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,32767,0.08309760093688964
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,63,0.008441600203514098
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,65535,0.1552448034286499
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,65535,0.296398401260376
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,1,0.008884800225496292
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,3,0.008436799794435502
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,1,0.00836160033941269
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,3,0.0083856001496315
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,7,0.008825600147247314
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,7,0.008366400003433227
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,31,0.00843999981880188
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,15,0.008430399745702744
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,31,0.008444800227880477
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,63,0.008727999776601792
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,131071,0.29857120513916013
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,63,0.008393599838018417
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,131071,0.5829760074615479
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,255,0.00844319984316826
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,127,0.008403199911117553
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,511,0.00836160033941269
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,1023,0.0083856001496315
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,1023,0.008324799686670303
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,2047,0.008993600308895112
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,2047,0.008484800159931184
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,4095,0.01045759990811348
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,4095,0.010417599976062775
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,8191,0.01069599986076355
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,8191,0.010436800122261048
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,16383,0.014547200500965118
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,16383,0.014511999487876893
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,255,0.008348800241947174
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,32767,0.016524800658226015
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,32767,0.014635199308395385
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,65535,0.018611200153827667
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,65535,0.016628800332546233
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,131071,0.021777600049972534
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,1,0.008363199979066848
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,3,0.008377599716186523
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,1,0.008454400300979614
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,3,0.008382400125265121
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,7,0.008446399867534638
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,7,0.008355200290679932
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,15,0.008462399989366532
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,15,0.008329600095748901
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,31,0.008948799967765809
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,31,0.008367999643087386
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,63,0.00846560001373291
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,63,0.008367999643087386
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,127,0.008441600203514098
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,127,0.008345600217580795
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,255,0.008428800106048583
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,255,0.008355200290679932
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,511,0.008393599838018417
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,511,0.008432000130414962
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,1023,0.00851840004324913
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,1023,0.008377599716186523
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,2047,0.01037919968366623
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,2047,0.008985599875450135
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,4095,0.01051200032234192
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,15,0.008355200290679932
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,4095,0.010523200035095215
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,8191,0.01241919994354248
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,8191,0.012574400007724761
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,127,0.00896959975361824
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,16383,0.0166143998503685
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,16383,0.01663679927587509
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,32767,0.01860000044107437
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,32767,0.016648000478744505
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,65535,0.020718400180339814
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,65535,0.01860959976911545
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,131071,0.036692801117897036
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,1,0.008337599784135818
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,131071,0.02072480022907257
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,1,0.00835679993033409
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,3,0.00833439975976944
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,3,0.008472000062465668
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,7,0.008336000144481659
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,7,0.008446399867534638
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,15,0.00841120034456253
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,15,0.008342400193214417
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,31,0.008388800173997879
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,31,0.008460800349712371
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,63,0.008455999940633774
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,63,0.008433599770069123
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,127,0.008455999940633774
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,127,0.00835840031504631
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,255,0.008348800241947174
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,1,0.008352000266313553
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,255,0.00843999981880188
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,511,0.008382400125265121
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,1023,0.008419200032949447
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,2047,0.01040000021457672
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,1023,0.008451200276613235
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,4095,0.012476799637079239
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,2047,0.00947519987821579
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,4095,0.010487999767065048
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,8191,0.02757279872894287
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,16383,0.04535520076751709
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,16383,0.028835201263427736
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,32767,0.08161919713020324
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,32767,0.04667679965496063
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,131071,0.019363200664520262
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,65535,0.15370559692382812
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,65535,0.08339679837226868
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,1,0.008388800173997879
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,1,0.008371199667453765
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,3,0.008846399933099746
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,131071,0.2980639934539795
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,131071,0.1567792057991028
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,7,0.00846880003809929
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,7,0.008427199721336365
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,15,0.008460800349712371
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,15,0.008459199965000153
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,31,0.008347199857234954
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,31,0.008459199965000153
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,63,0.008337599784135818
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,63,0.008528000116348267
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,127,0.00841279998421669
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,127,0.00846719965338707
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,255,0.008452799916267396
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,255,0.00843520015478134
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,511,0.008417599648237229
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,511,0.00841120034456253
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,1023,0.008427199721336365
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,1023,0.008427199721336365
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,2047,0.009382399916648864
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,2047,0.008516799658536911
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,4095,0.010427200049161912
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,4095,0.010492800176143647
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,8191,0.01252799928188324
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,511,0.008425600081682205
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,8191,0.012566399574279786
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,16383,0.016422399878501893
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,32767,0.018695999681949616
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,16383,0.014584000408649444
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,255,0.008364800363779068
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,65535,0.035006400942802426
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,32767,0.016232000291347505
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,65535,0.018571199476718904
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,131071,0.053110402822494504
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,1,0.008423999696969987
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,1,0.008420799672603608
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,131071,0.03515200018882751
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,7,0.008355200290679932
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,3,0.008390399813652038
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,15,0.008403199911117553
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,15,0.008340799808502197
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,31,0.00835840031504631
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,31,0.008419200032949447
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,63,0.008350399881601333
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,63,0.008345600217580795
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,127,0.00840959995985031
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,127,0.00835840031504631
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,255,0.008376000076532364
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,255,0.00835840031504631
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,511,0.010080000013113022
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,511,0.008534400165081025
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,1023,0.011097600311040878
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,1023,0.010420800000429154
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,8191,0.013991999626159667
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,2047,0.026956799626350402
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,2047,0.012467200309038163
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,4095,0.04524160027503967
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,4095,0.027456000447273254
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,8191,0.08071039915084839
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,8191,0.04560959935188293
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,16383,0.153164803981781
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,16383,0.0820688009262085
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,32767,0.2960736036300659
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,32767,0.15404800176620484
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,3,0.008451200276613235
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,float16,1,0.009759999811649323
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,fp8,1,0.008433599770069123
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,float16,3,0.010119999945163726
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,fp8,3,0.008737599849700928
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,float16,7,0.010412800312042236
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,fp8,7,0.008859200030565261
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,float16,15,0.010436800122261048
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,fp8,15,0.008782400190830231
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,65535,0.29879519939422605
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,float16,31,0.010428799688816071
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,fp8,31,0.008867199718952178
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,fp8,63,0.00835999995470047
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,float16,127,0.010388799756765366
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,float16,63,0.010391999781131745
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,fp8,127,0.00851999968290329
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,fp8,255,0.008985599875450135
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,float16,511,0.012593600153923034
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,fp8,511,0.01048479974269867
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,float16,1023,0.027636799216270446
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,fp8,1023,0.01454399973154068
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,float16,2047,0.04635840058326721
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,float16,4095,0.08214719891548157
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,fp8,4095,0.04700640141963959
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,3,0.008369600027799606
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,7,0.008353599905967712
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,float16,8191,0.1544592022895813
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,fp8,8191,0.08420479893684388
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,float16,1,0.012459199875593185
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,float16,16383,0.2984639883041382
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,fp8,1,0.012433599680662155
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,fp8,3,0.012417600303888322
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,float16,3,0.012451200187206269
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,float16,7,0.012531200051307678
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,fp8,7,0.0124208003282547
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,float16,15,0.01247360035777092
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,fp8,15,0.010908800363540649
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,float16,31,0.012452799826860428
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,fp8,31,0.010807999968528747
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,float16,63,0.012468799948692322
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,fp8,63,0.010796800255775452
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,float16,127,0.012492799758911132
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,fp8,127,0.011121600121259689
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,fp8,255,0.010463999956846238
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,float16,255,0.012534399330615998
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,float16,511,0.027153599262237548
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,fp8,511,0.01451359987258911
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,float16,1023,0.04622080028057098
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,fp8,1023,0.028951999545097352
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,float16,2047,0.08286399841308593
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,fp8,2047,0.04734239876270294
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,float16,4095,0.15694400072097778
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,65535,0.583571195602417
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,float16,8191,0.30254240036010743
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,1,0.00843520015478134
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,float16,255,0.010387200117111205
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,3,0.008654399961233138
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,3,0.008430399745702744
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,7,0.008451200276613235
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,7,0.008436799794435502
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,15,0.008448000252246856
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,fp8,2047,0.02861280143260956
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,31,0.00843999981880188
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,31,0.008382400125265121
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,63,0.008455999940633774
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,63,0.008472000062465668
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,127,0.008451200276613235
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,127,0.00846560001373291
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,255,0.00841120034456253
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,255,0.00843999981880188
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,511,0.008380799740552902
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,1023,0.009348800033330917
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,511,0.008441600203514098
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,1023,0.008446399867534638
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,2047,0.010406400263309478
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,2047,0.009860800206661224
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,511,0.00833439975976944
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,4095,0.010486400127410889
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,8191,0.012465599924325943
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,8191,0.012444800138473511
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,16383,0.014555199444293976
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,16383,0.01284639984369278
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,32767,0.01653600037097931
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,fp8,16383,0.15796480178833008
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,65535,0.04999839961528778
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,65535,0.03235360085964203
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,131071,0.08626719713211059
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,131071,0.05148640275001526
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,float16,1,0.01744000017642975
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,fp8,1,0.016518400609493257
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,float16,3,0.017030400037765504
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,fp8,3,0.016516800224781036
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,float16,7,0.0174575999379158
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,fp8,7,0.01652960032224655
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,float16,15,0.016630400717258454
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,fp8,15,0.01650079935789108
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,fp8,4095,0.08278239965438842
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,float16,31,0.016627199947834015
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,float16,63,0.016628800332546233
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,fp8,31,0.01648000031709671
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,fp8,63,0.01652960032224655
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,float16,127,0.018564799427986146
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,fp8,127,0.01653600037097931
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,fp8,8191,0.15795680284500122
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,fp8,255,0.015510399639606477
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,1,0.008544000238180161
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,fp8,511,0.029116800427436827
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,float16,511,0.04608800113201141
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,float16,1023,0.08225759863853455
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,fp8,1023,0.048393601179122926
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,float16,2047,0.1569808006286621
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,fp8,2047,0.08563359975814819
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,float16,1,0.028302401304244995
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,fp8,1,0.024817599356174468
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,float16,4095,0.2979568004608154
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,fp8,4095,0.15966399908065795
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,fp8,3,0.024828800559043886
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,4095,0.012135999649763108
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,fp8,7,0.024825599789619446
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,float16,15,0.02874400019645691
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,fp8,15,0.024806399643421174
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,float16,31,0.028870400786399842
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,fp8,31,0.02481600046157837
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,float16,63,0.02882719933986664
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,32767,0.030926400423049928
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,float16,127,0.03086400032043457
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,fp8,127,0.025176000595092774
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,float16,255,0.04654400050640106
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,fp8,255,0.030959999561309813
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,float16,511,0.08245919942855835
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,fp8,511,0.04928480088710785
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,float16,1023,0.15399520397186278
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,fp8,1023,0.08608959913253784
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,1,0.008721599727869034
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,float16,2047,0.29999840259552
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,fp8,2047,0.1586176037788391
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,1,0.008577600121498108
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,3,0.00846880003809929
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,7,0.008913599699735642
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,7,0.008419200032949447
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,15,0.008500800281763077
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,15,0.008451200276613235
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,31,0.008404800295829773
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,float16,255,0.027819201350212097
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,31,0.00843999981880188
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,63,0.00846560001373291
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,127,0.008372800052165985
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,127,0.008609600365161896
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,255,0.00835679993033409
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,15,0.008382400125265121
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,511,0.008432000130414962
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,511,0.008366400003433227
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,1023,0.008377599716186523
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,1023,0.008460800349712371
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,2047,0.010416000336408614
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,2047,0.010358399897813796
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,4095,0.01096000000834465
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,8191,0.014574399590492249
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,8191,0.01252640038728714
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,float16,3,0.028092798590660096
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,16383,0.03049919903278351
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,16383,0.014595200121402741
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,float16,7,0.028833600878715514
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,32767,0.04928320050239563
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,32767,0.030632001161575318
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,65535,0.0846176028251648
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,65535,0.049403199553489686
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,fp8,63,0.024798400700092316
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,131071,0.08658239841461182
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,3,0.008463999629020691
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,63,0.008449599891901017
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,255,0.008484800159931184
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,4095,0.012521600723266602
SGLang,0.5.10,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,131071,0.1559440016746521
