framework,version,device,op_name,kernel_source,batch_size,isl,num_heads,num_key_value_heads,head_dim,beam_width,attn_dtype,kv_cache_dtype,step,latency
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,float16,1,0.048897600173950194
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,float16,3,0.0489984005689621
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,float16,7,0.04824639856815338
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,fp8,1,0.039499199390411376
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,fp8,3,0.03969280123710632
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,float16,15,0.04866400063037872
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,fp8,15,0.03956800103187561
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,float16,31,0.05141440033912659
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,fp8,31,0.04030880033969879
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,fp8,63,0.03945440053939819
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,float16,127,0.05439839959144592
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,fp8,7,0.03924640119075775
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,fp8,127,0.041335999965667725
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,fp8,255,0.04782719910144806
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,float16,511,0.15874240398406983
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,fp8,511,0.08438720107078553
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,float16,255,0.08595679998397827
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,fp8,1023,0.15990560054779052
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,float16,1023,0.30288960933685305
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,float16,7,0.010480000078678131
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,float16,63,0.05260639786720276
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,float16,15,0.010414399951696397
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,fp8,3,0.008393599838018417
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,float16,31,0.010454399883747101
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,fp8,31,0.008382400125265121
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,float16,63,0.010419200360774993
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,fp8,7,0.008417599648237229
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,fp8,63,0.008459199965000153
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,float16,127,0.010369600355625152
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,float16,255,0.010452800244092942
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,fp8,255,0.00844319984316826
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,float16,511,0.010427200049161912
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,fp8,511,0.00835999995470047
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,fp8,127,0.008470399677753449
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,float16,1023,0.012140800058841706
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,float16,2047,0.012639999389648438
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,float16,1,0.010393600165843963
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,fp8,2047,0.010407999902963639
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,fp8,1023,0.0091279998421669
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,float16,1,0.010449600219726563
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,float16,3,0.010440000146627427
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,fp8,3,0.00835840031504631
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,float16,7,0.010406400263309478
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,fp8,7,0.008310399949550629
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,float16,15,0.010416000336408614
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,fp8,15,0.008329600095748901
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,float16,31,0.010438399761915207
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,fp8,31,0.008292800188064576
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,float16,63,0.010460799932479859
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,fp8,63,0.008313599973917007
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,float16,127,0.01043040007352829
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,fp8,2047,0.31195840835571287
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,fp8,127,0.008347199857234954
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,float16,511,0.010356800258159637
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,float16,3,0.010475199669599533
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,float16,255,0.010440000146627427
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,float16,2047,0.5916719913482666
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,fp8,255,0.00833280012011528
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,float16,1023,0.012459199875593185
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,fp8,1023,0.010424000024795533
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,float16,1,0.028836798667907716
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,fp8,2047,0.012454400211572647
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,float16,2047,0.030939200520515443
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,fp8,1,0.02280000001192093
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,float16,3,0.02884640097618103
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,fp8,3,0.023131200671195985
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,fp8,7,0.02282720059156418
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,float16,7,0.02889760136604309
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,float16,31,0.028886398673057555
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,float16,63,0.028911998867988585
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,fp8,63,0.022808000445365906
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,float16,15,0.028939199447631837
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,fp8,15,0.02327360063791275
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,float16,127,0.033164799213409424
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,fp8,255,0.028233599662780762
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,float16,511,0.08627359867095948
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,float16,255,0.04885919988155365
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,float16,1023,0.15850239992141724
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,fp8,511,0.008419200032949447
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,fp8,15,0.00843999981880188
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,fp8,1023,0.08549919724464417
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,float16,2047,0.30278398990631106
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,fp8,1,0.008432000130414962
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,float16,3,0.011032000184059143
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,fp8,3,0.008481600135564805
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,float16,7,0.01080000028014183
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,fp8,2047,0.16016800403594972
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,fp8,7,0.00843999981880188
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,fp8,15,0.008420799672603608
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,float16,31,0.011401599645614624
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,fp8,31,0.008388800173997879
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,float16,63,0.011430399864912033
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,float16,15,0.010926400125026704
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,fp8,63,0.008417599648237229
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,fp8,127,0.00841120034456253
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,float16,127,0.01122720018029213
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,float16,255,0.010409600287675857
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,fp8,511,0.04736000001430511
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,fp8,255,0.008955200016498566
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,fp8,1,0.008382400125265121
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,float16,511,0.013516800105571746
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,fp8,1023,0.012488000094890594
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,float16,2047,0.05006080269813538
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,fp8,2047,0.028676798939704894
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,float16,1,0.08743519783020019
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,fp8,1,0.07263039946556091
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,float16,3,0.08769599795341491
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,fp8,3,0.07266079783439636
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,float16,7,0.08768320083618164
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,fp8,7,0.0722208023071289
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,float16,15,0.09064800143241883
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,float16,1,0.010507199913263321
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,float16,31,0.09157599806785584
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,fp8,31,0.07395520210266113
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,float16,63,0.09246240258216858
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,fp8,63,0.07395039796829224
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,float16,127,0.09651679992675781
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,fp8,1,0.00835999995470047
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,fp8,127,0.0739184021949768
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,float16,1023,0.030687999725341798
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,fp8,255,0.08633919954299926
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,fp8,511,0.1594640016555786
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,float16,511,0.30362401008605955
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,fp8,15,0.0722208023071289
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,float16,1023,0.5913951873779297
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,float16,1,0.16456960439682006
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,fp8,1,0.13743679523468016
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,fp8,3,0.13783040046691894
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,float16,3,0.1661087989807129
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,fp8,1023,0.3091056108474731
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,fp8,7,0.13749920129776
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,fp8,31,0.022780799865722658
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,float16,15,0.16924480199813843
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,fp8,127,0.02439039945602417
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,float16,31,0.16892800331115723
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,fp8,31,0.13813120126724243
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,float16,63,0.16976799964904785
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,fp8,63,0.1383936047554016
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,float16,7,0.1684720039367676
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,fp8,127,0.1393391966819763
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,fp8,15,0.13905760049819946
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,64,128,1,float16,fp8,1,0.26711359024047854
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,float16,127,0.1800976037979126
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,float16,255,0.3063472032546997
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,fp8,511,0.010470400005578995
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,64,128,1,float16,float16,1,0.32332959175109866
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,64,128,1,float16,float16,7,0.32457919120788575
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,64,128,1,float16,float16,3,0.3253216028213501
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,64,128,1,float16,fp8,7,0.2687743902206421
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,64,128,1,float16,float16,15,0.3237423896789551
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,64,128,1,float16,fp8,3,0.2683183908462524
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,64,128,1,float16,float16,31,0.3242000102996826
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,64,128,1,float16,float16,63,0.3258016109466553
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,64,128,1,float16,fp8,15,0.26851840019226075
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,float16,1,0.014473600685596466
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,64,128,1,float16,fp8,31,0.26802239418029783
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,64,128,1,float16,fp8,127,0.26917760372161864
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,64,128,1,float16,float16,127,0.3442575931549072
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,float16,7,0.014454400539398194
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,float16,3,0.014481599628925323
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,float16,15,0.01449120044708252
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,fp8,3,0.010500799864530563
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,float16,31,0.014457599818706512
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,fp8,31,0.010540799796581268
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,float16,63,0.014531199634075165
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,fp8,63,0.010502400249242783
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,float16,127,0.014475199580192565
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,fp8,127,0.011409600079059602
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,float16,255,0.01446560025215149
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,fp8,255,0.010996799916028976
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,float16,511,0.031083199381828307
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,fp8,15,0.010977599769830704
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,fp8,511,0.014569599926471711
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,float16,1023,0.049804800748825075
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,fp8,1023,0.028811201453208923
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,float16,2047,0.08778240084648133
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,fp8,2047,0.047244799137115476
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,64,128,1,float16,float16,1,0.6387231826782227
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,64,128,1,float16,float16,3,0.6371583938598633
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,fp8,255,0.1640671968460083
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,64,128,1,float16,fp8,3,0.5288159847259521
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,64,128,1,float16,float16,7,0.6370736122131347
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,float16,255,0.15868799686431884
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,64,128,1,float16,fp8,7,0.5285327911376954
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,64,128,1,float16,float16,15,0.637174415588379
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,64,128,1,float16,fp8,15,0.5282479763031006
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,64,128,1,float16,fp8,63,0.2681583881378174
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,64,128,1,float16,float16,31,0.6361968040466308
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,fp8,1,0.01119839996099472
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,64,128,1,float16,float16,63,0.6387296199798584
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,64,128,1,float16,fp8,31,0.5269199848175049
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,64,128,1,float16,fp8,63,0.5270639896392822
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,fp8,7,0.01056319996714592
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,64,128,1,float16,float16,1,1.2683631896972656
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,64,128,1,float16,fp8,3,1.0492048263549805
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,64,128,1,float16,fp8,7,1.0491840362548828
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,64,128,1,float16,fp8,1,0.5283792018890381
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,float16,1,0.018675200641155243
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,fp8,1,0.014511999487876893
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,64,128,1,float16,float16,3,1.2681167602539063
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,64,128,1,float16,float16,7,1.269264030456543
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,float16,3,0.018667200207710268
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,64,128,1,float16,fp8,15,1.0479711532592773
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,fp8,7,0.014884799718856812
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,64,128,1,float16,float16,15,1.2668560028076172
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,float16,7,0.018595199286937713
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,fp8,3,0.01480640023946762
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,float16,15,0.0186256006360054
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,64,128,1,float16,fp8,31,1.0465439796447753
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,64,128,1,float16,float16,31,1.2694944381713866
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,fp8,63,0.014703999459743499
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,fp8,15,0.014979200065135955
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,float16,31,0.018595199286937713
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,fp8,31,0.014531199634075165
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,fp8,127,0.014584000408649444
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,float16,127,0.018648000061511995
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,fp8,511,0.028798401355743408
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,float16,1023,0.08629760146141052
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,fp8,1023,0.0481440007686615
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,float16,255,0.03065119981765747
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,float16,511,0.049348801374435425
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,fp8,2047,0.08777599930763244
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,float16,2047,0.15985120534896852
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,float16,3,0.0386352002620697
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,fp8,3,0.03229120075702667
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,float16,7,0.03887040019035339
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,float16,1,0.03880960047245026
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,fp8,7,0.03190400004386902
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,float16,15,0.03904640078544617
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,fp8,1,0.031193599104881287
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,float16,31,0.03912639915943146
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,fp8,31,0.03188000023365021
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,fp8,63,0.03188160061836243
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,float16,127,0.04334560036659241
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,fp8,127,0.03298079967498779
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,fp8,15,0.031046399474143983
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,fp8,255,0.03759520053863526
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,float16,63,0.0411327987909317
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,fp8,511,0.06630240082740783
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,float16,511,0.12100000381469726
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,64,128,1,float16,fp8,1,1.0492848396301269
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,fp8,1023,0.12297439575195312
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,float16,1023,0.22948160171508789
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,float16,1,0.010454399883747101
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,float16,3,0.010414399951696397
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,float16,2047,0.4453711986541748
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,float16,7,0.010422399640083313
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,float16,15,0.010492800176143647
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,fp8,2047,0.23632640838623048
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,float16,63,0.018612800538539885
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,float16,31,0.010419200360774993
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,fp8,255,0.014510400593280792
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,float16,63,0.010380800068378448
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,float16,127,0.010476800054311753
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,fp8,1,0.008472000062465668
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,fp8,3,0.008561599999666214
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,float16,255,0.0667519986629486
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,float16,255,0.010497599840164185
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,fp8,255,0.009041599929332733
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,fp8,7,0.008478400111198426
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,float16,1023,0.01241919994354248
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,fp8,1023,0.010465600341558457
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,float16,2047,0.013835200667381286
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,fp8,2047,0.010465600341558457
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,float16,1,0.008432000130414962
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,fp8,15,0.00841120034456253
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,fp8,1,0.007782399654388428
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,float16,3,0.008422400057315826
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,float16,7,0.008444800227880477
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,fp8,7,0.007816000282764435
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,float16,15,0.008417599648237229
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,fp8,15,0.008012799918651581
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,float16,31,0.008416000008583068
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,fp8,31,0.008193600177764892
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,float16,63,0.008499199897050858
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,fp8,63,0.008195199817419053
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,float16,127,0.008392000198364257
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,fp8,127,0.008337599784135818
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,float16,255,0.008388800173997879
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,fp8,31,0.008419200032949447
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,fp8,255,0.008313599973917007
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,fp8,511,0.008452799916267396
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,float16,1023,0.012377600371837615
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,fp8,1023,0.010360000282526016
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,float16,2047,0.02234079986810684
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,fp8,2047,0.012507200241088867
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,float16,1,0.024007999897003175
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,fp8,1,0.01876640021800995
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,fp8,3,0.01868479996919632
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,fp8,63,0.00846880003809929
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,float16,7,0.022942399978637694
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,fp8,7,0.0202224001288414
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,float16,15,0.023443199694156647
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,fp8,15,0.02051679939031601
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,float16,31,0.02356639951467514
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,fp8,31,0.019985599815845488
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,float16,63,0.023532800376415253
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,fp8,63,0.019808000326156615
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,float16,127,0.02680639922618866
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,fp8,127,0.019915199279785155
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,float16,255,0.03908640146255493
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,fp8,255,0.020678399503231047
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,float16,511,0.06632159948348999
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,fp8,511,0.03808479905128479
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,float16,1023,0.12098560333251954
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,fp8,1023,0.06611359715461732
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,fp8,127,0.008841600269079208
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,fp8,511,0.010372799634933472
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,float16,1,0.010388799756765366
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,fp8,2047,0.12318400144577027
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,fp8,1,0.008348800241947174
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,float16,2047,0.2298640012741089
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,fp8,3,0.008396799862384795
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,float16,3,0.010385599732398988
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,fp8,3,0.00833280012011528
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,float16,7,0.010460799932479859
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,fp8,7,0.008367999643087386
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,float16,15,0.010398399829864503
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,fp8,15,0.008366400003433227
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,float16,31,0.010489600151777268
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,float16,63,0.010335999727249145
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,fp8,31,0.00838399976491928
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,float16,127,0.01045759990811348
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,fp8,127,0.008393599838018417
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,float16,255,0.010419200360774993
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,float16,511,0.012481600046157837
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,fp8,511,0.010412800312042236
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,float16,511,0.010411199927330018
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,fp8,1023,0.012545600533485413
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,float16,2047,0.04321599900722504
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,fp8,2047,0.020212799310684204
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,float16,3,0.02407840043306351
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,float16,1,0.06790879964828492
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,fp8,1,0.0565775990486145
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,float16,3,0.06810719966888427
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,fp8,7,0.05687680244445801
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,fp8,3,0.05623999834060669
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,float16,15,0.06945919990539551
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,float16,7,0.067958402633667
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,float16,31,0.07178400158882141
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,float16,63,0.07178879976272583
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,fp8,15,0.05623679757118225
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,fp8,63,0.0575872004032135
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,fp8,31,0.05636159777641296
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,fp8,127,0.0575872004032135
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,float16,127,0.0740447998046875
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,float16,255,0.12258399724960327
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,float16,511,0.010367999970912933
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,float16,511,0.22971839904785157
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,float16,1023,0.4444608211517334
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,fp8,63,0.008433599770069123
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,fp8,511,0.12281440496444702
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,fp8,255,0.008399999886751174
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,float16,1023,0.023336000740528107
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,float16,3,0.12538559436798097
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,fp8,1023,0.23533279895782472
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,fp8,3,0.10519039630889893
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,fp8,1,0.10501919984817505
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,float16,7,0.12719520330429077
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,float16,15,0.12926559448242186
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,fp8,15,0.1051408052444458
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,float16,31,0.1293280005455017
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,float16,63,0.12932159900665283
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,fp8,7,0.10479680299758912
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,float16,127,0.13585439920425416
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,fp8,127,0.10673760175704956
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,float16,255,0.23171041011810303
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,fp8,63,0.10576640367507935
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,48,48,128,1,float16,float16,1,0.2423583984375
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,fp8,255,0.12600640058517457
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,48,48,128,1,float16,float16,7,0.24591200351715087
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,48,48,128,1,float16,fp8,3,0.2029968023300171
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,48,48,128,1,float16,float16,15,0.24557280540466309
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,48,48,128,1,float16,fp8,7,0.20299038887023926
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,48,48,128,1,float16,float16,31,0.24570879936218262
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,fp8,255,0.06751840114593506
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,48,48,128,1,float16,fp8,15,0.20315840244293212
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,48,48,128,1,float16,float16,63,0.24592480659484864
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,48,48,128,1,float16,fp8,63,0.2029616117477417
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,float16,1,0.0124208003282547
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,48,48,128,1,float16,fp8,127,0.2042720079421997
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,fp8,1,0.01037919968366623
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,float16,3,0.01250240057706833
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,fp8,3,0.010494399815797806
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,float16,7,0.012481600046157837
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,fp8,7,0.010416000336408614
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,48,48,128,1,float16,float16,127,0.2594320058822632
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,float16,15,0.012510399520397186
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,fp8,15,0.010452800244092942
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,fp8,31,0.010443200170993806
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,float16,63,0.012414400279521943
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,fp8,63,0.010447999835014344
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,float16,127,0.012451200187206269
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,fp8,127,0.010451199859380722
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,float16,1,0.12586079835891723
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,fp8,255,0.010428799688816071
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,float16,511,0.020931200683116914
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,float16,31,0.01249760016798973
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,float16,1023,0.039375999569892885
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,float16,255,0.012451200187206269
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,float16,2047,0.0669983983039856
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,fp8,2047,0.03780319988727569
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,fp8,1023,0.018611200153827667
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,48,48,128,1,float16,fp8,1,0.20299999713897704
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,48,48,128,1,float16,float16,1,0.4804863929748535
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,48,48,128,1,float16,fp8,1,0.39766879081726075
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,48,48,128,1,float16,float16,3,0.4796304225921631
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,48,48,128,1,float16,float16,3,0.2444528102874756
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,48,48,128,1,float16,float16,7,0.4792352199554443
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,48,48,128,1,float16,fp8,7,0.39792160987854003
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,48,48,128,1,float16,fp8,15,0.3977839946746826
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,48,48,128,1,float16,float16,15,0.4792384147644043
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,48,48,128,1,float16,float16,31,0.4796112060546875
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,48,48,128,1,float16,fp8,31,0.39743680953979493
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,48,48,128,1,float16,fp8,63,0.3974384069442749
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,48,48,128,1,float16,float16,63,0.4797696113586426
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,fp8,511,0.01255040019750595
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,48,48,128,1,float16,float16,1,0.9520560264587402
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,fp8,31,0.10543520450592041
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,48,48,128,1,float16,fp8,1,0.7891295909881592
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,48,48,128,1,float16,fp8,3,0.7889488220214844
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,48,48,128,1,float16,float16,3,0.9523568153381348
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,48,48,128,1,float16,fp8,7,0.789086389541626
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,48,48,128,1,float16,float16,7,0.9523743629455567
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,float16,1,0.01658080071210861
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,48,48,128,1,float16,float16,15,0.9522704124450684
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,48,48,128,1,float16,fp8,3,0.39927361011505125
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,float16,3,0.01650879979133606
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,fp8,3,0.013031999766826629
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,float16,7,0.016651199758052827
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,48,48,128,1,float16,fp8,15,0.7883391857147217
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,fp8,7,0.01289760023355484
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,float16,15,0.01655679941177368
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,fp8,15,0.014521600306034088
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,float16,31,0.01652639955282211
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,fp8,31,0.014528000354766845
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,float16,63,0.016603200137615202
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,48,48,128,1,float16,fp8,31,0.786846399307251
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,48,48,128,1,float16,float16,31,0.9529232025146485
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,fp8,127,0.014558400213718414
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,float16,255,0.021038399636745454
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,float16,511,0.03995679914951324
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,fp8,255,0.012484800070524216
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,fp8,511,0.020606400072574617
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,fp8,1023,0.0391072005033493
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,float16,1023,0.06765919923782349
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,48,48,128,1,float16,fp8,31,0.20290238857269288
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,float16,2047,0.12454400062561036
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,fp8,2047,0.06679040193557739
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,float16,1,0.03298079967498779
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,float16,3,0.032995200157165526
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,float16,15,0.033024001121521
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,fp8,1,0.01451520025730133
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,float16,31,0.033032000064849854
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,float16,127,0.01656000018119812
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,fp8,63,0.012768000364303589
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,float16,63,0.035043200850486754
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,float16,127,0.037108799815177916
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,float16,7,0.03304159939289093
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,float16,255,0.057548797130584715
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,fp8,1,0.028297600150108338
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,float16,511,0.10248479843139649
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,fp8,255,0.03442240059375763
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,fp8,511,0.05704479813575745
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,float16,1023,0.1926192045211792
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,fp8,1023,0.10396640300750733
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,fp8,15,0.027489599585533143
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,float16,1,0.008393599838018417
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,fp8,2047,0.19897760152816774
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,float16,2047,0.3728784084320068
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,float16,3,0.009062399715185165
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,float16,7,0.008723200112581254
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,fp8,31,0.02881760001182556
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,fp8,63,0.02885119915008545
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,float16,15,0.008881600201129913
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,float16,31,0.008852799981832504
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,fp8,3,0.02874400019645691
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,fp8,7,0.028798401355743408
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,float16,63,0.008788800239562989
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,float16,127,0.010230399668216705
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,fp8,1,0.008336000144481659
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,float16,255,0.008390399813652038
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,fp8,255,0.008318399637937545
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,float16,511,0.0094480000436306
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,fp8,7,0.008420799672603608
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,fp8,511,0.008408000320196151
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,fp8,3,0.008377599716186523
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,float16,1023,0.010572800040245056
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,fp8,1023,0.00947519987821579
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,float16,2047,0.01244800016283989
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,fp8,2047,0.0104032002389431
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,fp8,1,0.007739199697971344
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,float16,3,0.00841120034456253
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,fp8,3,0.0077632002532482145
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,float16,7,0.008430399745702744
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,fp8,7,0.007569599896669388
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,float16,15,0.008340799808502197
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,fp8,15,0.007823999971151352
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,float16,31,0.00840959995985031
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,fp8,31,0.007524800300598144
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,float16,63,0.008406399935483932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,fp8,63,0.0074143998324871065
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,float16,127,0.008392000198364257
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,fp8,15,0.008427199721336365
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,fp8,127,0.00830880030989647
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,fp8,31,0.008336000144481659
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,float16,511,0.010424000024795533
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,fp8,511,0.008454400300979614
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,float16,1023,0.012273599952459335
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,fp8,63,0.008324799686670303
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,fp8,1023,0.009497600048780442
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,float16,2047,0.016068799793720244
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,fp8,2047,0.012452799826860428
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,float16,1,0.020747199654579163
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,fp8,1,0.016622400283813475
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,fp8,127,0.008363199979066848
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,float16,3,0.02070080041885376
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,float16,7,0.020684799551963805
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,fp8,3,0.016595199704170227
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,fp8,7,0.016663999855518342
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,float16,15,0.020664000511169435
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,fp8,15,0.016652800142765045
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,float16,31,0.020686399936676026
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,fp8,31,0.016684800386428833
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,float16,63,0.02067680060863495
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,fp8,63,0.016620799899101257
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,float16,127,0.02272160053253174
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,float16,255,0.03479360044002533
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,fp8,255,0.018193599581718446
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,float16,511,0.057494401931762695
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,fp8,511,0.03326559960842133
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,fp8,127,0.016575999557971954
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,float16,1023,0.10256160497665405
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,fp8,1023,0.05662400126457214
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,fp8,127,0.02892960011959076
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,float16,1,0.010465600341558457
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,fp8,1,0.008446399867534638
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,float16,3,0.010451199859380722
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,fp8,2047,0.10456000566482544
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,float16,7,0.010422399640083313
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,fp8,7,0.008425600081682205
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,float16,15,0.010387200117111205
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,fp8,15,0.008441600203514098
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,float16,31,0.010449600219726563
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,fp8,31,0.008379200100898742
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,float16,2047,0.19293919801712037
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,fp8,63,0.008423999696969987
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,float16,1,0.008460800349712371
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,fp8,127,0.008372800052165985
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,float16,63,0.010462400317192078
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,float16,255,0.010446400195360184
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,fp8,255,0.008371199667453765
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,fp8,511,0.010435199737548828
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,float16,1023,0.016908800601959227
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,float16,127,0.010422399640083313
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,fp8,1023,0.012495999783277511
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,float16,2047,0.041119998693466185
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,float16,1,0.05756480097770691
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,float16,511,0.012465599924325943
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,fp8,1,0.047491198778152464
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,float16,3,0.05758399963378906
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,fp8,3,0.04828000068664551
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,fp8,7,0.04763999879360199
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,fp8,2047,0.018705600500106813
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,float16,15,0.05781279802322388
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,fp8,15,0.04760800004005432
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,fp8,31,0.0479856014251709
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,fp8,255,0.00729919970035553
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,float16,63,0.060278397798538205
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,float16,7,0.0578000009059906
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,float16,255,0.008476799726486206
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,fp8,63,0.04936639964580536
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,float16,127,0.06367520093917847
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,float16,255,0.1028480052947998
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,fp8,255,0.0588703989982605
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,float16,511,0.19324640035629273
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,fp8,511,0.10335359573364258
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,fp8,127,0.049465599656105044
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,float16,1,0.10697120428085327
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,fp8,1,0.08951359987258911
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,float16,1023,0.37247838973999026
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,fp8,3,0.0889087975025177
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,float16,7,0.10762560367584229
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,fp8,7,0.08871039748191833
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,float16,15,0.10911519527435302
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,fp8,15,0.08922079801559449
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,float16,31,0.10893440246582031
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,fp8,31,0.08922560214996338
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,float16,63,0.10973279476165772
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,fp8,63,0.09035519957542419
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,fp8,127,0.09052000045776368
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,float16,127,0.11491520404815674
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,fp8,3,0.008510400354862214
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,fp8,255,0.10758559703826905
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,40,40,128,1,float16,float16,1,0.20350079536437987
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,float16,255,0.19472639560699462
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,40,40,128,1,float16,float16,3,0.2050015926361084
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,40,40,128,1,float16,fp8,3,0.1704256057739258
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,40,40,128,1,float16,float16,7,0.20591359138488768
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,40,40,128,1,float16,fp8,7,0.17073440551757812
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,40,40,128,1,float16,float16,15,0.20602719783782958
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,40,40,128,1,float16,float16,31,0.20564160346984864
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,40,40,128,1,float16,fp8,31,0.17063679695129394
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,40,40,128,1,float16,float16,63,0.20699679851531982
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,40,40,128,1,float16,fp8,63,0.17059680223464965
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,40,40,128,1,float16,float16,127,0.2173504114151001
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,40,40,128,1,float16,fp8,127,0.1722048044204712
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,float16,1,0.011241599917411804
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,fp8,1,0.010486400127410889
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,float16,3,0.01244639977812767
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,fp8,3,0.010390400141477584
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,float16,31,0.06053280234336853
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,float16,15,0.010819199681282043
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,float16,3,0.10573920011520385
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,fp8,1023,0.19783040285110473
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,fp8,15,0.010391999781131745
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,float16,31,0.011468800157308579
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,float16,63,0.010707200318574906
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,fp8,31,0.010419200360774993
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,float16,127,0.010844799876213073
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,fp8,127,0.010444799810647965
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,float16,255,0.012142399698495865
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,fp8,255,0.01043040007352829
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,float16,511,0.016542400419712066
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,fp8,511,0.012486399710178375
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,float16,1023,0.036934399604797365
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,fp8,1023,0.016571199893951415
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,float16,2047,0.06168799996376038
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,fp8,2047,0.0350847989320755
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,40,40,128,1,float16,fp8,1,0.17039040327072144
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,40,40,128,1,float16,float16,1,0.401204776763916
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,40,40,128,1,float16,fp8,1,0.33341279029846194
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,40,40,128,1,float16,fp8,15,0.17084640264511108
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,40,40,128,1,float16,float16,3,0.40161919593811035
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,40,40,128,1,float16,fp8,3,0.3338624000549316
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,40,40,128,1,float16,float16,7,0.4014416217803955
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,40,40,128,1,float16,fp8,7,0.3338047981262207
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,40,40,128,1,float16,fp8,15,0.33390560150146487
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,40,40,128,1,float16,float16,15,0.4010335922241211
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,40,40,128,1,float16,float16,31,0.40166401863098145
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,40,40,128,1,float16,fp8,31,0.332367992401123
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,40,40,128,1,float16,float16,63,0.40195040702819823
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,40,40,128,1,float16,fp8,63,0.33277120590209963
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,float16,7,0.012516799569129943
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,fp8,7,0.010396800190210342
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,40,40,128,1,float16,float16,1,0.7958127975463867
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,40,40,128,1,float16,fp8,1,0.6594016075134277
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,40,40,128,1,float16,fp8,3,0.6594848155975341
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,40,40,128,1,float16,float16,3,0.794377613067627
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,float16,1,0.014523200690746307
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,40,40,128,1,float16,fp8,7,0.6592448234558106
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,fp8,63,0.010332799702882766
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,fp8,1,0.01244639977812767
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,float16,3,0.014572800695896148
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,fp8,3,0.01249919980764389
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,40,40,128,1,float16,fp8,15,0.6579504013061523
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,float16,15,0.01454080045223236
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,40,40,128,1,float16,float16,15,0.79519681930542
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,fp8,15,0.012449599802494049
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,float16,31,0.01454399973154068
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,float16,63,0.014510400593280792
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,fp8,63,0.012439999729394913
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,float16,7,0.014519999921321868
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,fp8,127,0.012532800436019897
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,40,40,128,1,float16,fp8,31,0.6574416160583496
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,40,40,128,1,float16,float16,31,0.7948480129241944
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,fp8,255,0.012403199821710587
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,float16,511,0.03494240045547485
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,fp8,511,0.01658080071210861
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,float16,1023,0.057441598176956175
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,fp8,1023,0.033580800890922545
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,float16,127,0.014567999541759491
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,float16,255,0.01647839993238449
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,float16,1,0.02887200117111206
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,float16,2047,0.10465760231018066
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,float16,3,0.028112000226974486
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,float16,7,0.028193598985671996
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,float16,15,0.028697600960731505
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,fp8,3,0.02356960028409958
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,float16,31,0.02893120050430298
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,fp8,31,0.02356480062007904
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,fp8,1,0.022782400250434875
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,float16,63,0.028883200883865357
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,float16,127,0.031651198863983154
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,fp8,127,0.02293439954519272
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,float16,255,0.04730879962444305
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,fp8,255,0.028854399919509888
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,float16,511,0.08386880159378052
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,fp8,511,0.047635200619697574
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,float16,1023,0.15594079494476318
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,fp8,1023,0.08624320030212403
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,40,40,128,1,float16,float16,7,0.795243215560913
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,fp8,7,0.012521600723266602
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,float16,2047,0.30087521076202395
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,fp8,31,0.012529599666595458
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,fp8,2047,0.16112639904022216
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,float16,1,0.008419200032949447
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,float16,3,0.008396799862384795
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,fp8,4095,0.3164128065109253
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,fp8,2047,0.05735359787940979
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,float16,4095,0.5915984153747559
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,float16,7,0.00835840031504631
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,float16,15,0.008420799672603608
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,fp8,15,0.02276960015296936
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,float16,31,0.00894559994339943
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,fp8,63,0.022950400412082673
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,float16,63,0.009256000071763993
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,fp8,1,0.008403199911117553
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,float16,127,0.008404800295829773
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,fp8,127,0.008366400003433227
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,float16,255,0.008379200100898742
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,fp8,255,0.008355200290679932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,float16,511,0.00900000035762787
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,fp8,511,0.008414400368928909
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,float16,1023,0.010396800190210342
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,fp8,1023,0.008376000076532364
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,float16,2047,0.0108815997838974
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,fp8,7,0.008392000198364257
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,fp8,2047,0.01040479987859726
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,float16,4095,0.012567999958992004
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,fp8,4095,0.010462400317192078
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,float16,1,0.010444799810647965
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,fp8,31,0.008433599770069123
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,fp8,1,0.008427199721336365
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,float16,3,0.010382399708032609
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,fp8,3,0.008457600325345992
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,fp8,7,0.008403199911117553
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,float16,7,0.008710400015115739
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,fp8,15,0.008457600325345992
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,float16,31,0.009020800143480301
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,fp8,31,0.00843520015478134
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,fp8,63,0.008379200100898742
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,float16,127,0.010471999645233154
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,fp8,127,0.008377599716186523
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,fp8,7,0.0227183997631073
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,float16,255,0.009424000233411788
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,fp8,255,0.008352000266313553
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,float16,511,0.010262399911880493
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,fp8,511,0.008387199789285659
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,float16,1023,0.010494399815797806
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,fp8,1023,0.009065599739551544
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,float16,2047,0.012510399520397186
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,fp8,2047,0.010513599961996078
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,float16,4095,0.029110398888587952
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,fp8,4095,0.012750400602817536
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,float16,1,0.018596799671649934
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,fp8,1,0.014820800721645355
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,float16,3,0.018636800348758698
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,float16,7,0.01865919977426529
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,fp8,7,0.014561599493026734
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,float16,15,0.018310399353504182
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,fp8,15,0.014532800018787383
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,float16,31,0.01866399943828583
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,fp8,31,0.014601600170135499
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,float16,63,0.01839520037174225
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,float16,127,0.018585599958896637
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,fp8,3,0.008371199667453765
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,fp8,63,0.014534400403499603
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,float16,255,0.028825598955154418
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,fp8,127,0.015124799311161041
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,float16,511,0.047276800870895384
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,fp8,511,0.028907200694084166
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,float16,1023,0.08365280032157899
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,fp8,1023,0.048548799753189084
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,fp8,15,0.008419200032949447
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,float16,2047,0.15646719932556152
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,float16,4095,0.30236001014709474
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,float16,15,0.010428799688816071
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,fp8,4095,0.16208479404449463
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,fp8,63,0.008425600081682205
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,fp8,1,0.008336000144481659
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,float16,1,0.008433599770069123
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,fp8,3,0.008423999696969987
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,fp8,7,0.008316799998283386
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,float16,63,0.008993600308895112
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,float16,15,0.008339200168848038
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,fp8,15,0.008321599662303924
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,float16,31,0.008820799738168716
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,fp8,31,0.00830560028553009
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,float16,63,0.008399999886751174
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,fp8,63,0.008379200100898742
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,float16,127,0.008948799967765809
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,fp8,127,0.008353599905967712
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,float16,255,0.009019199758768082
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,float16,511,0.010369600355625152
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,fp8,511,0.008382400125265121
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,float16,1023,0.012417600303888322
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,fp8,1023,0.010468800365924836
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,float16,2047,0.028790399432182312
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,fp8,2047,0.012478400021791458
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,float16,4095,0.046430400013923644
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,fp8,4095,0.027187201380729675
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,float16,1,0.04774399995803833
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,fp8,3,0.0149167999625206
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,float16,3,0.04745759963989258
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,fp8,3,0.040140798687934874
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,float16,7,0.04774079918861389
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,fp8,7,0.039683198928833006
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,float16,15,0.04782080054283142
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,fp8,15,0.03925760090351105
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,float16,31,0.04951040148735046
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,fp8,31,0.04038560092449188
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,float16,63,0.05133600234985351
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,fp8,63,0.04096480011940003
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,float16,127,0.05344319939613342
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,fp8,127,0.041201600432395936
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,float16,255,0.08431360125541687
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,fp8,255,0.014552000164985656
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,fp8,255,0.049035200476646425
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,fp8,2047,0.08729439973831177
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,fp8,511,0.08473920226097106
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,float16,511,0.15656479597091674
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,float16,1023,0.30015199184417723
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,float16,7,0.00886240005493164
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,float16,1,0.08657280206680298
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,fp8,255,0.008284799754619598
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,float16,3,0.08648959994316101
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,fp8,2047,0.31164960861206054
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,fp8,3,0.07230880260467529
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,fp8,7,0.07234399914741516
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,float16,7,0.08680160045623779
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,float16,15,0.0888271987438202
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,fp8,15,0.07210080027580261
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,float16,2047,0.5893648147583008
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,float16,31,0.08869760036468506
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,fp8,31,0.07241439819335938
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,fp8,63,0.0739359974861145
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,float16,63,0.08980640172958373
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,float16,127,0.09343360066413879
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,fp8,127,0.07399359941482545
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,fp8,255,0.08853920102119446
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,float16,511,0.3014271974563599
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,fp8,511,0.1587455987930298
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,float16,1,0.16465760469436647
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,fp8,1,0.13756959438323973
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,float16,3,0.16433440446853637
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,fp8,3,0.13736480474472046
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,fp8,7,0.13742560148239136
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,float16,15,0.16619360446929932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,fp8,1023,0.16025919914245607
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,float16,3,0.00852160006761551
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,fp8,15,0.13766880035400392
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,float16,31,0.16650400161743165
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,fp8,31,0.13803679943084718
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,float16,63,0.16620320081710815
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,fp8,1,0.0724399983882904
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,fp8,63,0.1375872015953064
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,float16,127,0.17455040216445922
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,float16,1,0.010416000336408614
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,float16,3,0.010420800000429154
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,fp8,1,0.008420799672603608
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,fp8,255,0.16946239471435548
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,float16,255,0.3038399934768677
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,fp8,1,0.0394351989030838
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,fp8,7,0.008419200032949447
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,fp8,15,0.008393599838018417
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,float16,31,0.010385599732398988
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,fp8,31,0.00846719965338707
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,float16,63,0.010387200117111205
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,fp8,63,0.00846880003809929
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,float16,127,0.01045759990811348
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,fp8,127,0.00843520015478134
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,float16,255,0.010417599976062775
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,fp8,255,0.008399999886751174
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,float16,511,0.012532800436019897
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,fp8,511,0.010436800122261048
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,float16,1023,0.02890239953994751
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,fp8,1023,0.01342719942331314
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,float16,2047,0.047225600481033324
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,fp8,2047,0.028856000304222106
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,fp8,4095,0.047737601399421695
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,float16,4095,0.08378720283508301
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,32,128,1,float16,float16,1,0.321616005897522
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,32,128,1,float16,fp8,1,0.2684895992279053
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,float16,7,0.16646560430526733
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,32,128,1,float16,float16,3,0.32209761142730714
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,32,128,1,float16,fp8,3,0.26843359470367434
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,32,128,1,float16,float16,7,0.32357120513916016
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,32,128,1,float16,fp8,7,0.2688416004180908
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,32,128,1,float16,float16,15,0.3219055891036987
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,32,128,1,float16,fp8,15,0.2684319972991943
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,32,128,1,float16,float16,31,0.32371840476989744
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,fp8,127,0.13950719833374023
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,32,128,1,float16,float16,63,0.32340478897094727
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,fp8,3,0.008460800349712371
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,float16,7,0.010462400317192078
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,32,128,1,float16,fp8,63,0.2684959888458252
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,float16,15,0.01048479974269867
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,32,128,1,float16,float16,127,0.33931679725646974
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,32,128,1,float16,fp8,127,0.26966879367828367
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,32,128,1,float16,fp8,1,0.5296783924102784
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,32,128,1,float16,float16,1,0.6374447822570801
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,32,128,1,float16,float16,3,0.6374479770660401
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,32,128,1,float16,fp8,3,0.5292880058288574
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,float16,255,0.1578976035118103
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,32,128,1,float16,fp8,7,0.5288095951080323
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,32,128,1,float16,float16,7,0.6385200023651123
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,float16,1,0.01266079992055893
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,32,128,1,float16,float16,15,0.6388160228729248
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,32,128,1,float16,fp8,15,0.5285168170928956
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,fp8,1,0.010908800363540649
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,fp8,3,0.010518400371074677
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,32,128,1,float16,fp8,31,0.52816801071167
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,32,128,1,float16,float16,31,0.6370960235595703
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,fp8,7,0.010454399883747101
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,float16,15,0.012540799379348756
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,float16,31,0.012476799637079239
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,32,128,1,float16,float16,63,0.6387919902801513
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,fp8,31,0.010449600219726563
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,fp8,15,0.011417599767446518
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,32,128,1,float16,fp8,63,0.5276527881622315
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,float16,63,0.01257600039243698
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,float16,127,0.012481600046157837
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,fp8,127,0.01144160032272339
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,fp8,63,0.011348800361156463
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,float16,511,0.028798401355743408
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,fp8,511,0.014542399346828461
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,fp8,1023,0.028815999627113342
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,float16,2047,0.0858128011226654
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,fp8,2047,0.0474368005990982
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,float16,4095,0.15904799699783326
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,float16,1,0.02268960028886795
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,32,128,1,float16,fp8,31,0.26846239566802976
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,float16,3,0.022806400060653688
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,fp8,3,0.020638400316238405
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,float16,7,0.022731199860572815
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,fp8,7,0.02032800018787384
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,float16,15,0.02268480062484741
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,fp8,1,0.01897760033607483
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,fp8,15,0.01887200027704239
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,float16,31,0.022840000689029694
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,fp8,31,0.01887039989233017
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,float16,63,0.02269600033760071
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,float16,127,0.024747200310230255
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,fp8,127,0.018622399866580965
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,float16,255,0.03835839927196503
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,fp8,255,0.021049599349498748
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,float16,3,0.012492799758911132
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,float16,511,0.06564000248908997
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,float16,7,0.012510399520397186
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,float16,1023,0.11972960233688354
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,fp8,511,0.038838401436805725
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,fp8,255,0.01088000014424324
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,fp8,2047,0.12416800260543823
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,float16,1023,0.048747199773788455
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,float16,2047,0.22798080444335939
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,float16,1,0.01043360009789467
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,float16,3,0.010420800000429154
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,fp8,4095,0.08530719876289368
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,float16,4095,0.4453999996185303
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,fp8,4095,0.23908159732818604
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,float16,7,0.010375999659299851
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,float16,15,0.010355199873447418
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,float16,31,0.01034879982471466
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,fp8,63,0.018987199664115904
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,float16,63,0.010375999659299851
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,fp8,1,0.008417599648237229
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,fp8,3,0.008507200330495835
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,float16,127,0.010396800190210342
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,fp8,127,0.008398400247097015
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,fp8,255,0.00838399976491928
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,fp8,31,0.008404800295829773
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,float16,511,0.010367999970912933
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,fp8,1023,0.0666208028793335
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,fp8,511,0.008420799672603608
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,fp8,7,0.008423999696969987
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,float16,2047,0.0125231996178627
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,fp8,1023,0.008371199667453765
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,fp8,2047,0.010441599786281586
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,float16,4095,0.012470400333404541
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,fp8,4095,0.01072319969534874
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,float16,1,0.01045759990811348
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,float16,3,0.010121600329875946
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,fp8,3,0.008479999750852585
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,float16,7,0.009864000231027603
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,fp8,7,0.008366400003433227
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,float16,15,0.00979359969496727
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,fp8,15,0.008382400125265121
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,float16,31,0.010185600072145463
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,fp8,31,0.008382400125265121
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,float16,63,0.00968799963593483
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,fp8,63,0.008406399935483932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,float16,127,0.010387200117111205
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,fp8,127,0.008423999696969987
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,float16,255,0.010411199927330018
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,fp8,15,0.00846560001373291
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,float16,511,0.010438399761915207
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,fp8,511,0.008671999722719193
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,fp8,63,0.008408000320196151
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,fp8,1023,0.010443200170993806
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,float16,1023,0.012455999851226807
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,float16,2047,0.01295199990272522
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,fp8,2047,0.010505600273609162
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,fp8,4095,0.012753599882125854
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,float16,4095,0.020182399451732634
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,float16,1,0.015148800611495972
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,fp8,1,0.014496000111103058
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,float16,3,0.016128000617027283
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,float16,7,0.016305600106716157
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,fp8,7,0.014519999921321868
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,float16,15,0.016206400096416475
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,fp8,15,0.014537599682807923
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,float16,31,0.015043200552463531
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,fp8,31,0.01302880048751831
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,float16,63,0.015087999403476715
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,fp8,63,0.012726399302482604
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,float16,127,0.015060800313949584
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,fp8,127,0.012908799946308136
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,float16,255,0.01887039989233017
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,fp8,255,0.01265919953584671
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,float16,511,0.03912639915943146
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,fp8,511,0.02059199959039688
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,float16,255,0.010419200360774993
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,float16,1023,0.06696320176124573
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,fp8,1023,0.03907040059566498
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,float16,255,0.012824000418186187
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,float16,2047,0.122980797290802
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,fp8,2047,0.06753119826316833
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,float16,1,0.008367999643087386
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,fp8,1,0.007760000228881836
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,fp8,4095,0.12591359615325928
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,float16,4095,0.23480799198150634
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,float16,3,0.008371199667453765
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,fp8,3,0.008297599852085114
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,fp8,7,0.008399999886751174
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,float16,15,0.008433599770069123
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,float16,31,0.008416000008583068
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,fp8,31,0.008062399923801422
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,float16,63,0.008337599784135818
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,fp8,255,0.008392000198364257
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,float16,127,0.008377599716186523
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,fp8,127,0.008379200100898742
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,float16,255,0.008372800052165985
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,fp8,255,0.0083856001496315
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,fp8,3,0.012744000554084778
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,float16,511,0.010427200049161912
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,fp8,511,0.008339200168848038
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,fp8,1023,0.010407999902963639
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,float16,2047,0.01995840072631836
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,fp8,2047,0.012415999919176102
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,float16,4095,0.03716480135917664
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,fp8,4095,0.019041599333286287
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,float16,1,0.037222400307655334
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,fp8,1,0.031651198863983154
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,float16,3,0.03714239895343781
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,fp8,3,0.031548801064491275
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,fp8,7,0.03153600096702576
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,float16,15,0.037415999174118045
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,fp8,15,0.031171199679374696
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,float16,31,0.03780480027198792
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,fp8,31,0.031041601300239564
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,float16,63,0.039103999733924866
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,fp8,63,0.031067198514938353
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,float16,127,0.04127359986305237
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,float16,1023,0.010478399693965912
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,fp8,127,0.03287039995193482
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,float16,255,0.06572480201721191
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,fp8,1,0.008776000142097473
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,float16,511,0.11927039623260498
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,float16,7,0.008419200032949447
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,fp8,15,0.008416000008583068
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,float16,1023,0.2278991937637329
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,fp8,1023,0.12341599464416504
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,fp8,63,0.008318399637937545
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,float16,1,0.06765440106391907
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,fp8,1,0.05608000159263611
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,float16,2047,0.4438928127288818
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,float16,3,0.06764000058174133
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,fp8,3,0.057068800926208495
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,fp8,7,0.05655519962310791
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,fp8,2047,0.23858559131622314
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,float16,7,0.06744639873504639
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,float16,15,0.06788640022277832
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,float16,1023,0.011860799789428712
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,fp8,31,0.05602719783782959
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,float16,63,0.07000640034675598
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,fp8,63,0.05761119723320007
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,float16,127,0.072188800573349
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,float16,255,0.12099360227584839
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,fp8,255,0.06816160082817077
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,fp8,127,0.05776320099830627
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,fp8,511,0.1219167947769165
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,float16,7,0.037150400876998904
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,float16,511,0.22784159183502198
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,float16,1,0.12543840408325196
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,fp8,1,0.10516159534454346
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,float16,3,0.12538559436798097
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,float16,7,0.126363205909729
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,fp8,7,0.10506080389022827
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,float16,15,0.12751200199127197
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,fp8,15,0.10508960485458374
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,float16,31,0.1273136019706726
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,fp8,31,0.10630559921264648
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,float16,63,0.1275696039199829
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,fp8,63,0.10662879943847656
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,fp8,127,0.10667519569396973
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,fp8,255,0.039017599821090695
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,float16,255,0.2306704044342041
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,float16,1,0.010235200077295304
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,fp8,1,0.008371199667453765
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,float16,3,0.01037440001964569
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,fp8,511,0.06638720035552978
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,float16,7,0.010345599800348281
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,fp8,7,0.00843520015478134
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,float16,15,0.01008159965276718
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,fp8,3,0.00835999995470047
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,fp8,15,0.008327999711036682
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,fp8,31,0.008366400003433227
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,float16,63,0.009830400347709656
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,fp8,63,0.00843999981880188
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,float16,127,0.010363200306892395
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,float16,31,0.010358399897813796
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,fp8,127,0.008403199911117553
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,fp8,255,0.008427199721336365
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,float16,511,0.012465599924325943
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,fp8,15,0.05597440004348755
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,fp8,511,0.010467199981212616
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,float16,255,0.010363200306892395
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,float16,1023,0.021060800552368163
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,float16,31,0.07004960179328919
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,fp8,2047,0.020392000675201416
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,float16,4095,0.07198879718780518
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,fp8,4095,0.04189760088920593
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,24,24,128,1,float16,fp8,1,0.20296320915222169
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,24,24,128,1,float16,float16,1,0.24211359024047852
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,24,24,128,1,float16,float16,3,0.24388480186462402
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,fp8,3,0.10545120239257813
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,24,24,128,1,float16,float16,7,0.24438879489898682
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,24,24,128,1,float16,fp8,7,0.20304319858551026
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,24,24,128,1,float16,float16,15,0.24418559074401855
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,24,24,128,1,float16,fp8,15,0.20342559814453126
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,float16,127,0.1334704041481018
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,24,24,128,1,float16,float16,31,0.24388480186462402
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,24,24,128,1,float16,fp8,31,0.20377440452575685
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,fp8,255,0.12723679542541505
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,24,24,128,1,float16,fp8,63,0.20314719676971435
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,24,24,128,1,float16,float16,127,0.25672159194946287
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,24,24,128,1,float16,fp8,127,0.20504961013793946
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,24,24,128,1,float16,float16,1,0.479369592666626
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,24,24,128,1,float16,fp8,1,0.3995743989944458
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,24,24,128,1,float16,float16,3,0.47931680679321287
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,24,24,128,1,float16,fp8,3,0.39933760166168214
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,fp8,1023,0.012683199346065521
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,24,24,128,1,float16,fp8,7,0.3998016119003296
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,float16,2047,0.04105280041694641
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,24,24,128,1,float16,float16,15,0.47934880256652834
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,24,24,128,1,float16,fp8,15,0.3995599985122681
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,24,24,128,1,float16,float16,31,0.47971358299255373
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,24,24,128,1,float16,fp8,31,0.39761919975280763
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,float16,1,0.011915200203657151
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,fp8,1,0.010407999902963639
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,24,24,128,1,float16,float16,63,0.48057122230529786
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,fp8,3,0.010451199859380722
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,fp8,7,0.010396800190210342
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,24,24,128,1,float16,fp8,63,0.39913599491119384
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,float16,15,0.012435200065374375
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,fp8,15,0.010489600151777268
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,fp8,31,0.01043199971318245
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,float16,63,0.012417600303888322
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,fp8,63,0.01035040020942688
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,float16,127,0.011689600348472596
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,fp8,127,0.010361599922180175
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,float16,255,0.011561600118875503
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,float16,511,0.01863359957933426
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,24,24,128,1,float16,float16,63,0.24445600509643556
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,fp8,511,0.012441600114107132
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,float16,1023,0.037694400548934935
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,float16,2047,0.06577439904212952
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,fp8,2047,0.03846240043640137
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,fp8,1023,0.018593600392341612
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,fp8,4095,0.06770880222320556
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,float16,4095,0.12054719924926757
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,float16,1,0.018145599961280824
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,float16,3,0.017759999632835387
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,fp8,1,0.014611199498176575
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,fp8,3,0.014740799367427827
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,24,24,128,1,float16,float16,7,0.47956480979919436
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,float16,15,0.018115200102329254
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,fp8,15,0.014646400511264802
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,float16,31,0.018105599284172057
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,fp8,31,0.01478240042924881
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,float16,63,0.016841599345207216
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,24,24,128,1,float16,fp8,3,0.20305280685424804
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,float16,3,0.01231200024485588
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,fp8,63,0.014528000354766845
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,float16,127,0.016710400581359863
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,float16,255,0.027449598908424376
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,float16,7,0.01048479974269867
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,float16,511,0.0463344007730484
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,float16,31,0.012398400157690049
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,float16,1023,0.0829200029373169
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,fp8,1023,0.04810400009155273
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,float16,2047,0.15519200563430785
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,fp8,2047,0.08725600242614746
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,fp8,255,0.010414399951696397
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,float16,4095,0.30066239833831787
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,fp8,4095,0.16448320150375367
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,float16,1,0.010364799946546554
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,float16,8191,0.5896016120910644
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,fp8,8191,0.32131199836730956
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,float16,3,0.010417599976062775
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,float16,7,0.016654400527477263
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,fp8,7,0.014752000570297241
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,fp8,127,0.014961600303649902
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,float16,15,0.010286399722099304
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,fp8,1,0.008388800173997879
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,float16,31,0.010444799810647965
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,fp8,31,0.008343999832868576
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,float16,63,0.008848000317811966
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,fp8,63,0.008414400368928909
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,float16,127,0.008723200112581254
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,fp8,127,0.00840959995985031
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,float16,255,0.010339199751615524
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,fp8,255,0.008417599648237229
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,float16,511,0.010320000350475311
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,fp8,511,0.008419200032949447
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,float16,1023,0.010360000282526016
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,fp8,1023,0.008449599891901017
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,float16,2047,0.010467199981212616
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,fp8,2047,0.010332799702882766
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,float16,7,0.008667200058698653
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,fp8,15,0.008460800349712371
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,fp8,4095,0.010518400371074677
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,float16,8191,0.014547200500965118
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,fp8,8191,0.012558400630950928
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,float16,4095,0.01249760016798973
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,float16,1,0.008428800106048583
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,fp8,1,0.00835999995470047
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,float16,3,0.008371199667453765
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,fp8,3,0.00836160033941269
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,float16,7,0.00838399976491928
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,fp8,7,0.008299200236797333
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,float16,15,0.008408000320196151
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,fp8,15,0.008387199789285659
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,float16,31,0.008408000320196151
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,fp8,31,0.008337599784135818
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,float16,63,0.008446399867534638
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,float16,127,0.008377599716186523
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,fp8,127,0.008387199789285659
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,fp8,255,0.008425600081682205
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,float16,511,0.008369600027799606
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,fp8,511,0.008367999643087386
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,float16,1023,0.00854720026254654
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,fp8,255,0.014558400213718414
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,fp8,1023,0.00841279998421669
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,fp8,511,0.028870400786399842
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,fp8,2047,0.008459199965000153
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,float16,4095,0.012555199861526489
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,fp8,4095,0.010497599840164185
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,float16,8191,0.02810559868812561
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,fp8,8191,0.014083200693130493
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,float16,1,0.012508800625801087
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,fp8,1,0.010843200236558914
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,float16,3,0.012492799758911132
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,fp8,3,0.01090560033917427
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,float16,7,0.012510399520397186
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,fp8,7,0.01043360009789467
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,float16,15,0.012532800436019897
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,fp8,15,0.010486400127410889
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,float16,31,0.012489599734544754
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,fp8,31,0.010521599650382995
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,float16,63,0.012486399710178375
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,fp8,63,0.010518400371074677
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,float16,127,0.01281760036945343
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,fp8,7,0.008428800106048583
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,fp8,127,0.01050880029797554
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,float16,255,0.01249919980764389
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,fp8,255,0.010505600273609162
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,float16,511,0.028207999467849732
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,fp8,511,0.014542399346828461
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,float16,1023,0.0470223993062973
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,fp8,1023,0.028856000304222106
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,float16,2047,0.08438400030136109
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,fp8,2047,0.04764800071716309
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,float16,4095,0.15952320098876954
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,fp8,4095,0.0858735978603363
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,float16,1,0.008414400368928909
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,fp8,1,0.0083856001496315
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,float16,3,0.008420799672603608
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,float16,8191,0.3053407907485962
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,fp8,3,0.008398400247097015
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,fp8,7,0.008441600203514098
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,float16,7,0.008475200086832047
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,float16,15,0.008505599945783615
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,fp8,15,0.008427199721336365
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,fp8,31,0.008336000144481659
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,float16,63,0.008455999940633774
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,float16,127,0.008481600135564805
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,fp8,127,0.008444800227880477
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,float16,255,0.008430399745702744
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,fp8,255,0.008441600203514098
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,float16,255,0.00841279998421669
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,float16,511,0.008959999680519104
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,fp8,511,0.00843520015478134
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,float16,1023,0.010436800122261048
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,fp8,1023,0.008395200222730636
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,float16,2047,0.01244639977812767
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,fp8,2047,0.010471999645233154
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,float16,4095,0.028944000601768494
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,fp8,4095,0.012881599366664886
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,float16,8191,0.046433600783348086
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,fp8,8191,0.028207999467849732
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,float16,1,0.027316799759864806
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,fp8,1,0.022801600396633148
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,float16,3,0.02688640058040619
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,fp8,3,0.022811199724674224
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,float16,7,0.026900801062583923
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,fp8,7,0.022793599963188173
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,fp8,3,0.00841279998421669
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,float16,15,0.02691200077533722
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,float16,31,0.026915198564529418
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,fp8,31,0.022767999768257143
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,float16,63,0.028443199396133424
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,fp8,63,0.022921599447727203
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,float16,127,0.03086400032043457
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,fp8,127,0.024780799448490144
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,float16,255,0.047225600481033324
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,fp8,255,0.028815999627113342
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,fp8,8191,0.16443840265274048
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,float16,511,0.08257279992103576
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,fp8,511,0.04733439981937408
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,float16,31,0.008455999940633774
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,float16,1023,0.1556815981864929
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,fp8,1023,0.08476960062980651
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,fp8,63,0.008427199721336365
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,float16,2047,0.2996543884277344
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,fp8,63,0.00841120034456253
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,fp8,2047,0.16260960102081298
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,fp8,1,0.04056800007820129
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,float16,1,0.04739679992198944
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,float16,3,0.04735200107097626
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,fp8,3,0.04015839993953705
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,float16,7,0.04735040068626404
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,float16,15,0.04732159972190857
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,fp8,4095,0.3149375915527344
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,float16,4095,0.5904640197753906
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,fp8,15,0.03953920006752014
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,float16,2047,0.010467199981212616
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,float16,31,0.047619199752807616
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,fp8,63,0.04120959937572479
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,fp8,31,0.041012799739837645
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,fp8,127,0.04127359986305237
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,float16,255,0.08375359773635864
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,fp8,255,0.048916798830032346
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,fp8,511,0.08486239910125733
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,float16,511,0.1558608055114746
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,float16,1,0.08658239841461182
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,float16,1023,0.29921119213104247
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,fp8,1023,0.15911680459976196
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,fp8,15,0.023158399760723113
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,float16,3,0.08672959804534912
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,float16,7,0.08650559782981873
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,fp8,3,0.07361279726028443
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,fp8,7,0.07269759774208069
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,fp8,15,0.07320320010185241
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,float16,31,0.0885263979434967
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,fp8,31,0.07292159795761108
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,fp8,63,0.07400640249252319
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,float16,63,0.08830559849739075
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,fp8,127,0.07462720274925232
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,fp8,255,0.08915680050849914
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,float16,511,0.30089919567108153
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,fp8,511,0.15900800228118897
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,float16,1,0.00880960002541542
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,fp8,7,0.03960480093955994
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,fp8,1,0.008380799740552902
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,float16,3,0.008355200290679932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,fp8,3,0.008363199979066848
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,float16,7,0.008457600325345992
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,fp8,7,0.008428800106048583
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,float16,15,0.008355200290679932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,fp8,15,0.008382400125265121
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,float16,31,0.00835999995470047
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,float16,127,0.05147839784622192
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,float16,63,0.049737599492073056
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,float16,63,0.008420799672603608
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,fp8,63,0.00841279998421669
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,float16,127,0.00841279998421669
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,float16,255,0.008379200100898742
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,fp8,255,0.008337599784135818
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,float16,511,0.010473600029945374
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,fp8,511,0.008326400071382523
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,float16,1023,0.012468799948692322
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,fp8,1023,0.01043040007352829
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,float16,2047,0.027987200021743774
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,float16,4095,0.045798400044441225
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,fp8,1,0.07281759977340699
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,fp8,4095,0.02685759961605072
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,fp8,8191,0.046716800332069396
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,float16,15,0.08780959844589234
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,float16,1,0.16422879695892334
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,fp8,1,0.13766560554504395
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,float16,3,0.16448960304260254
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,float16,127,0.0922976016998291
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,fp8,3,0.13788479566574097
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,float16,7,0.16620960235595703
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,fp8,7,0.13775039911270143
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,float16,255,0.15718239545822144
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,float16,15,0.16621919870376586
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,fp8,15,0.13932960033416747
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,float16,31,0.16605279445648194
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,fp8,31,0.13856639862060546
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,float16,63,0.1663807988166809
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,fp8,63,0.13937599658966066
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,float16,127,0.17426719665527343
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,fp8,127,0.13940320014953614
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,float16,255,0.30339200496673585
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,16,128,1,float16,float16,1,0.3215359926223755
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,16,128,1,float16,float16,3,0.32240478992462157
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,16,128,1,float16,fp8,1,0.2689440011978149
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,fp8,31,0.008374399691820144
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,16,128,1,float16,fp8,3,0.2688944101333618
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,fp8,127,0.008329600095748901
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,16,128,1,float16,float16,7,0.3231679916381836
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,16,128,1,float16,fp8,7,0.2687376022338867
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,fp8,2047,0.012503999471664428
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,16,128,1,float16,float16,15,0.32325279712677
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,16,128,1,float16,float16,31,0.32298879623413085
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,16,128,1,float16,fp8,31,0.2690000057220459
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,16,128,1,float16,fp8,63,0.2692255973815918
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,16,128,1,float16,float16,63,0.323636794090271
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,float16,1,0.01040479987859726
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,fp8,1,0.00836160033941269
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,fp8,3,0.008427199721336365
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,float16,7,0.010375999659299851
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,fp8,7,0.008395200222730636
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,float16,15,0.0103472001850605
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,fp8,15,0.008479999750852585
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,float16,31,0.01045600026845932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,fp8,31,0.008806400001049042
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,float16,63,0.010351999849081039
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,fp8,63,0.008859200030565261
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,float16,127,0.010454399883747101
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,fp8,127,0.008630400151014328
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,float16,255,0.010372799634933472
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,fp8,255,0.008376000076532364
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,float16,511,0.012503999471664428
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,fp8,511,0.010465600341558457
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,float16,1023,0.027827200293540955
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,fp8,255,0.16987839937210084
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,fp8,1023,0.013622400164604188
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,fp8,2047,0.028779199719429015
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,float16,4095,0.08284959793090821
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,fp8,4095,0.04773919880390167
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,float16,8191,0.15664960145950318
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,fp8,8191,0.08713600039482117
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,float16,1,0.015377600491046906
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,16,128,1,float16,fp8,15,0.268558406829834
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,float16,3,0.014609600603580474
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,float16,8191,0.08204799890518188
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,float16,7,0.016407999396324157
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,16,128,1,float16,float16,127,0.33812479972839354
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,16,128,1,float16,fp8,127,0.2707711935043335
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,float16,15,0.0161423996090889
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,float16,31,0.016139200329780577
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,float16,63,0.016495999693870545
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,float16,2047,0.04598560035228729
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,float16,127,0.01631679981946945
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,float16,3,0.010451199859380722
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,float16,255,0.018668800592422485
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,fp8,1,0.013041600584983826
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,float16,511,0.0381520003080368
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,fp8,511,0.02067199945449829
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,float16,1023,0.06583840250968934
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,fp8,1023,0.03883999884128571
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,float16,2047,0.12215839624404908
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,fp8,63,0.014529600739479065
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,fp8,2047,0.06842079758644104
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,float16,4095,0.23294880390167236
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,fp8,4095,0.12820160388946533
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,fp8,127,0.01454399973154068
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,float16,1,0.009521599858999252
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,fp8,8191,0.24835360050201416
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,float16,8191,0.4575664043426514
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,float16,3,0.010406400263309478
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,float16,7,0.010377600044012069
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,fp8,3,0.013089600205421447
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,float16,15,0.008444800227880477
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,fp8,15,0.014529600739479065
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,fp8,7,0.014478400349617004
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,float16,31,0.010428799688816071
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,float16,63,0.009459199756383896
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,fp8,1,0.008399999886751174
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,float16,127,0.008451200276613235
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,fp8,127,0.008414400368928909
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,float16,255,0.008423999696969987
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,fp8,255,0.008367999643087386
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,fp8,7,0.008462399989366532
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,fp8,3,0.008425600081682205
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,fp8,511,0.008449599891901017
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,float16,1023,0.010452800244092942
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,fp8,1023,0.00846719965338707
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,float16,2047,0.010436800122261048
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,fp8,2047,0.010414399951696397
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,fp8,15,0.008425600081682205
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,fp8,4095,0.010454399883747101
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,float16,8191,0.013977600634098053
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,fp8,8191,0.012408000230789185
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,float16,1,0.009587199985980987
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,fp8,1,0.008420799672603608
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,float16,3,0.010393600165843963
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,fp8,3,0.008441600203514098
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,float16,7,0.010406400263309478
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,fp8,7,0.008433599770069123
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,fp8,31,0.008423999696969987
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,fp8,63,0.008404800295829773
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,float16,15,0.01043199971318245
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,fp8,15,0.008451200276613235
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,float16,31,0.009900800138711929
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,fp8,31,0.008430399745702744
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,float16,63,0.010375999659299851
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,fp8,63,0.0083856001496315
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,float16,127,0.0104032002389431
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,fp8,127,0.00841279998421669
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,fp8,255,0.008448000252246856
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,fp8,511,0.008452799916267396
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,float16,1023,0.010398399829864503
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,fp8,1023,0.008537600189447403
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,float16,2047,0.012443199753761292
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,fp8,255,0.012503999471664428
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,float16,4095,0.012596799433231354
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,fp8,4095,0.012455999851226807
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,fp8,31,0.014478400349617004
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,float16,8191,0.02067680060863495
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,float16,1,0.011798399686813354
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,fp8,8191,0.01446239948272705
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,fp8,2047,0.010407999902963639
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,fp8,1,0.010422399640083313
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,float16,3,0.010856000334024429
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,fp8,3,0.010393600165843963
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,fp8,7,0.010380800068378448
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,float16,15,0.01072480008006096
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,fp8,15,0.010412800312042236
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,float16,31,0.011219199746847153
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,fp8,31,0.010424000024795533
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,float16,63,0.012430399656295776
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,fp8,63,0.010380800068378448
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,float16,127,0.012428800016641617
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,fp8,127,0.010385599732398988
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,float16,255,0.012336000055074691
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,fp8,255,0.01037440001964569
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,float16,511,0.01857759952545166
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,fp8,511,0.012460800260305405
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,float16,1023,0.03802559971809387
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,fp8,1023,0.018619200587272643
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,float16,511,0.008479999750852585
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,fp8,2047,0.03895840048789978
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,float16,2047,0.06554399728775025
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,fp8,4095,0.0662671983242035
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,float16,4095,0.012488000094890594
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,float16,1,0.010496000200510025
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,float16,8191,0.23064000606536866
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,fp8,1,0.008432000130414962
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,fp8,8191,0.12493760585784912
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,float16,3,0.010076799988746643
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,fp8,3,0.008470399677753449
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,float16,7,0.010390400141477584
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,fp8,7,0.008502399921417237
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,fp8,15,0.008711999654769898
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,float16,31,0.010371199995279311
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,fp8,31,0.0089199997484684
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,float16,63,0.010380800068378448
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,fp8,63,0.008633600175380706
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,fp8,127,0.008406399935483932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,float16,255,0.010369600355625152
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,float16,255,0.010443200170993806
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,float16,511,0.010360000282526016
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,float16,511,0.010468800365924836
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,fp8,511,0.00888959988951683
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,fp8,1023,0.010444799810647965
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,float16,2047,0.012511999905109405
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,fp8,2047,0.010486400127410889
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,float16,4095,0.018667200207710268
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,fp8,4095,0.01252799928188324
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,float16,8191,0.03903839886188507
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,fp8,8191,0.018537600338459016
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,float16,7,0.01250080019235611
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,fp8,1,0.01926880031824112
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,float16,3,0.022793599963188173
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,float16,7,0.022708800435066224
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,fp8,3,0.019472000002861024
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,fp8,7,0.020287999510765077
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,float16,15,0.02268960028886795
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,fp8,15,0.019067199528217317
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,float16,31,0.0227743998169899
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,fp8,31,0.01911199986934662
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,float16,63,0.02276960015296936
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,fp8,63,0.019036799669265747
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,fp8,127,0.01918399930000305
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,float16,127,0.025496000051498414
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,float16,255,0.037990400195121767
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,fp8,255,0.021404799818992615
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,float16,511,0.06559039950370789
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,fp8,511,0.03880159854888916
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,float16,4095,0.1202672004699707
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,float16,1023,0.11929440498352051
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,fp8,1023,0.06678720116615296
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,float16,2047,0.22802720069885254
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,float16,15,0.010435199737548828
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,float16,4095,0.44480161666870116
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,fp8,4095,0.24006080627441406
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,fp8,1,0.03298079967498779
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,fp8,255,0.00840959995985031
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,float16,1023,0.012459199875593185
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,float16,3,0.03781279921531677
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,fp8,3,0.031246399879455565
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,float16,7,0.03746080100536346
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,fp8,7,0.03272159993648529
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,float16,15,0.037508800625801086
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,fp8,15,0.03298560082912445
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,float16,31,0.037990400195121767
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,fp8,31,0.03245759904384613
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,float16,63,0.04045119881629944
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,fp8,63,0.03267680108547211
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,float16,127,0.04158399999141693
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,fp8,127,0.032950401306152344
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,fp8,255,0.03914400041103363
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,float16,511,0.1199679970741272
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,float16,1,0.022745600342750548
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,fp8,511,0.0659824013710022
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,float16,1,0.0669920027256012
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,float16,1023,0.22786400318145753
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,float16,3,0.06741920113563538
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,fp8,3,0.05714719891548157
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,float16,7,0.06714239716529846
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,fp8,7,0.057123202085495
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,float16,15,0.06793760061264038
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,fp8,2047,0.12313120365142823
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,fp8,15,0.05724160075187683
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,float16,31,0.0698144018650055
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,fp8,31,0.05725600123405457
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,float16,63,0.06991199851036071
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,fp8,63,0.057555198669433594
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,float16,1,0.03775359988212586
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,float16,127,0.07247679829597473
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,fp8,127,0.057871997356414795
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,float16,255,0.12095999717712402
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,fp8,255,0.06999040246009827
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,float16,1,0.00840959995985031
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,fp8,1,0.007926400005817413
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,float16,511,0.22879040241241455
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,float16,3,0.008420799672603608
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,fp8,511,0.12287039756774902
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,fp8,3,0.008225599676370621
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,fp8,7,0.00841279998421669
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,fp8,15,0.008352000266313553
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,float16,31,0.008393599838018417
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,fp8,31,0.008352000266313553
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,float16,63,0.008403199911117553
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,float16,127,0.008475200086832047
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,fp8,127,0.008355200290679932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,float16,255,0.00843999981880188
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,fp8,255,0.007374399900436401
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,float16,255,0.06584159731864929
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,float16,511,0.009596800059080124
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,fp8,1023,0.12338080406188964
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,fp8,511,0.008446399867534638
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,float16,1023,0.012097600102424621
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,fp8,1023,0.010387200117111205
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,float16,2047,0.020708799362182617
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,fp8,2047,0.012460800260305405
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,float16,4095,0.03757759928703308
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,fp8,4095,0.018748800456523895
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,float16,8191,0.06553279757499694
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,fp8,8191,0.037319999933242795
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,float16,1,0.12536799907684326
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,fp8,1,0.10676000118255616
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,float16,3,0.12532800436019897
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,fp8,3,0.1060912013053894
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,float16,7,0.12621599435806274
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,fp8,7,0.10559680461883544
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,float16,15,0.12740800380706788
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,float16,31,0.1271888017654419
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,fp8,31,0.10676640272140503
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,float16,63,0.12728960514068605
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,fp8,63,0.10677759647369385
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,float16,127,0.1333423972129822
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,fp8,127,0.1077072024345398
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,fp8,255,0.12983200550079346
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,float16,255,0.23130879402160645
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,float16,7,0.008387199789285659
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,float16,15,0.008417599648237229
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,12,12,128,1,float16,float16,1,0.24243040084838868
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,12,12,128,1,float16,fp8,1,0.20505280494689943
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,fp8,63,0.008321599662303924
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,12,12,128,1,float16,float16,3,0.24395999908447266
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,12,12,128,1,float16,fp8,3,0.20499999523162843
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,12,12,128,1,float16,float16,7,0.24433600902557373
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,12,12,128,1,float16,fp8,7,0.204915189743042
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,12,12,128,1,float16,float16,15,0.24410879611968994
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,12,12,128,1,float16,fp8,15,0.20516319274902345
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,12,12,128,1,float16,fp8,31,0.20507519245147704
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,12,12,128,1,float16,float16,31,0.2453808069229126
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,12,12,128,1,float16,float16,63,0.24455199241638184
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,float16,1,0.010358399897813796
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,12,12,128,1,float16,fp8,63,0.20510239601135255
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,fp8,1,0.00838399976491928
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,float16,3,0.010446400195360184
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,12,12,128,1,float16,float16,127,0.2560096025466919
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,fp8,3,0.008432000130414962
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,12,12,128,1,float16,fp8,127,0.20711040496826172
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,fp8,1,0.05698879957199097
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,float16,15,0.010451199859380722
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,fp8,15,0.008401600271463394
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,float16,31,0.01037440001964569
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,fp8,31,0.0083856001496315
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,fp8,63,0.008404800295829773
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,float16,127,0.010419200360774993
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,fp8,127,0.008392000198364257
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,float16,255,0.010360000282526016
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,fp8,255,0.00833119973540306
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,float16,511,0.012462399899959564
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,fp8,511,0.010435199737548828
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,float16,1023,0.021492800116539
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,float16,63,0.010232000052928925
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,fp8,1023,0.01257600039243698
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,float16,2047,0.04126079976558685
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,float16,4095,0.07173439860343933
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,fp8,4095,0.042428800463676454
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,fp8,8191,0.07502880096435546
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,fp8,15,0.10655839443206787
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,float16,8191,0.13237119913101197
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,fp8,2047,0.02048480063676834
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,float16,1,0.01247360035777092
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,float16,3,0.012441600114107132
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,float16,7,0.012508800625801087
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,float16,15,0.012478400021791458
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,fp8,1,0.010520000010728836
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,float16,31,0.012481600046157837
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,fp8,7,0.011430399864912033
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,fp8,31,0.010539200156927109
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,float16,63,0.012518399953842163
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,fp8,63,0.010463999956846238
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,float16,127,0.012457600235939026
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,fp8,127,0.010452800244092942
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,float16,255,0.01247360035777092
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,fp8,255,0.01043360009789467
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,float16,511,0.027163198590278624
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,fp8,511,0.014523200690746307
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,float16,127,0.010443200170993806
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,float16,1023,0.045928001403808594
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,fp8,2047,0.047219198942184445
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,float16,4095,0.1567792057991028
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,float16,2047,0.08350239992141724
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,fp8,4095,0.08635839819908142
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,float16,8191,0.3081023931503296
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,fp8,8191,0.1642240047454834
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,float16,7,0.01037440001964569
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,fp8,7,0.008382400125265121
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,float16,1,0.008408000320196151
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,float16,3,0.008416000008583068
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,fp8,16383,0.3240000009536743
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,float16,16383,0.6070511817932129
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,float16,7,0.008433599770069123
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,float16,15,0.008444800227880477
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,fp8,3,0.010500799864530563
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,float16,31,0.008414400368928909
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,fp8,15,0.01053759977221489
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,float16,63,0.008488000184297562
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,fp8,1023,0.029183998703956604
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,float16,127,0.009111999720335006
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,fp8,1,0.008452799916267396
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,float16,255,0.008523199707269669
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,fp8,3,0.008416000008583068
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,float16,511,0.009380800276994705
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,fp8,511,0.008436799794435502
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,float16,1023,0.009702400118112565
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,fp8,1023,0.008423999696969987
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,float16,2047,0.010473600029945374
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,fp8,2047,0.010388799756765366
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,float16,4095,0.01048479974269867
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,fp8,4095,0.01048479974269867
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,float16,8191,0.012492799758911132
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,fp8,8191,0.012425599992275238
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,fp8,16383,0.01404000073671341
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,fp8,15,0.00844319984316826
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,float16,1,0.008446399867534638
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,fp8,7,0.008398400247097015
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,fp8,1,0.00843520015478134
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,float16,16383,0.01565600037574768
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,float16,3,0.008417599648237229
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,float16,7,0.008507200330495835
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,fp8,3,0.008367999643087386
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,float16,15,0.008417599648237229
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,fp8,15,0.008406399935483932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,float16,31,0.008398400247097015
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,fp8,31,0.008420799672603608
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,float16,63,0.00841279998421669
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,fp8,63,0.008403199911117553
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,float16,127,0.008721599727869034
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,fp8,127,0.008436799794435502
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,fp8,7,0.008448000252246856
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,float16,255,0.008422400057315826
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,fp8,255,0.008420799672603608
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,float16,511,0.008430399745702744
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,fp8,31,0.008403199911117553
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,float16,1023,0.008497600257396699
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,fp8,511,0.008393599838018417
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,float16,2047,0.010436800122261048
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,fp8,2047,0.010406400263309478
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,float16,4095,0.012467200309038163
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,fp8,63,0.008408000320196151
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,fp8,4095,0.010417599976062775
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,fp8,8191,0.01247519999742508
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,float16,16383,0.029198399186134337
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,float16,1,0.010424000024795533
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,fp8,1,0.008499199897050858
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,float16,3,0.010367999970912933
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,fp8,3,0.008516799658536911
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,float16,7,0.010390400141477584
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,fp8,7,0.008414400368928909
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,float16,15,0.010438399761915207
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,fp8,15,0.008416000008583068
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,float16,31,0.01040479987859726
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,fp8,31,0.008473599702119828
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,float16,63,0.010412800312042236
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,fp8,63,0.009033600240945816
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,float16,127,0.010398399829864503
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,fp8,127,0.008579199761152267
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,float16,255,0.010414399951696397
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,fp8,255,0.00860000029206276
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,float16,511,0.012535999715328216
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,float16,1023,0.026947200298309326
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,fp8,1023,0.01430400013923645
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,float16,2047,0.04530239999294281
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,fp8,2047,0.028382399678230287
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,float16,4095,0.08216959834098816
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,fp8,255,0.008433599770069123
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,fp8,4095,0.04759199917316437
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,fp8,8191,0.08841599822044373
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,float16,16383,0.3058255910873413
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,fp8,16383,0.17235200405120848
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,fp8,1023,0.00844319984316826
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,float16,1,0.008371199667453765
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,fp8,1,0.008324799686670303
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,float16,8191,0.014483200013637542
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,float16,3,0.008345600217580795
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,fp8,3,0.008401600271463394
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,float16,7,0.008417599648237229
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,fp8,7,0.008367999643087386
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,fp8,16383,0.014579200744628906
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,fp8,15,0.008327999711036682
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,float16,31,0.008376000076532364
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,fp8,31,0.008291199803352356
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,float16,63,0.008436799794435502
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,fp8,63,0.008321599662303924
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,fp8,127,0.008416000008583068
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,fp8,127,0.008342400193214417
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,fp8,255,0.008377599716186523
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,float16,511,0.008428800106048583
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,fp8,511,0.00844319984316826
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,float16,1023,0.009257599711418152
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,fp8,1023,0.008396799862384795
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,float16,2047,0.010382399708032609
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,fp8,2047,0.009638399630784989
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,float16,4095,0.012435200065374375
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,fp8,4095,0.010436800122261048
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,float16,8191,0.027347201108932497
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,fp8,8191,0.012880000472068786
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,float16,16383,0.04549280107021332
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,fp8,16383,0.02885279953479767
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,float16,1,0.016651199758052827
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,fp8,1,0.014531199634075165
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,float16,3,0.016649599373340606
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,fp8,3,0.014552000164985656
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,float16,7,0.016574400663375854
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,fp8,7,0.014534400403499603
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,float16,15,0.01660960018634796
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,fp8,15,0.014601600170135499
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,float16,31,0.017057600617408752
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,fp8,31,0.014542399346828461
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,fp8,63,0.015335999429225922
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,float16,127,0.017825600504875184
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,float16,8191,0.1570912003517151
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,float16,255,0.02693760097026825
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,fp8,255,0.014891199767589569
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,fp8,127,0.015455999970436096
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,fp8,511,0.029003199934959412
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,float16,1023,0.08210880160331727
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,fp8,1023,0.04738720059394837
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,float16,2047,0.15436320304870604
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,float16,15,0.008318399637937545
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,fp8,2047,0.08621439933776856
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,float16,255,0.00838399976491928
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,float16,4095,0.29947519302368164
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,fp8,4095,0.16423200368881224
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,float16,1,0.027459201216697694
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,fp8,511,0.010496000200510025
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,fp8,1,0.022939200699329376
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,float16,3,0.02682879865169525
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,fp8,3,0.02281759977340698
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,float16,8191,0.5941535949707031
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,fp8,7,0.023316800594329834
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,fp8,8191,0.3240799903869629
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,float16,31,0.027275198698043825
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,fp8,31,0.022785599529743194
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,float16,63,0.017008000612258913
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,fp8,15,0.023841600120067596
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,float16,63,0.02755360007286072
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,fp8,63,0.023387199640274046
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,float16,127,0.030814400315284728
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,float16,255,0.046260800957679746
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,fp8,255,0.029811200499534608
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,fp8,511,0.04748480021953583
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,float16,511,0.08234400153160096
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,fp8,1023,0.08435680270195008
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,float16,1023,0.1547376036643982
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,float16,511,0.04538559913635254
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,float16,1,0.047358399629592894
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,fp8,2047,0.16129280328750611
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,float16,2047,0.299614405632019
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,fp8,1,0.03957119882106781
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,float16,3,0.04765279889106751
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,float16,7,0.04737119972705841
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,fp8,3,0.03983519971370697
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,float16,15,0.04737119972705841
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,float16,31,0.048020800948143004
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,fp8,31,0.040863999724388124
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,fp8,7,0.03996799886226654
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,float16,63,0.04938080012798309
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,fp8,63,0.03968800008296967
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,float16,127,0.008451200276613235
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,fp8,127,0.04116480052471161
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,float16,255,0.08329280018806458
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,fp8,255,0.04925920069217682
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,fp8,511,0.08449919819831848
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,float16,7,0.02683520019054413
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,float16,15,0.027134400606155396
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,float16,1023,0.29886560440063475
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,float16,1,0.00838399976491928
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,float16,3,0.008403199911117553
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,fp8,1,0.00843520015478134
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,fp8,1023,0.15908639430999755
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,fp8,3,0.00840959995985031
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,float16,7,0.00841120034456253
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,fp8,7,0.008427199721336365
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,fp8,15,0.008339200168848038
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,float16,31,0.008457600325345992
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,fp8,31,0.008422400057315826
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,float16,63,0.008403199911117553
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,fp8,63,0.008382400125265121
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,float16,127,0.00838399976491928
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,fp8,127,0.008406399935483932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,float16,255,0.008444800227880477
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,fp8,255,0.008348800241947174
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,float16,511,0.008580800145864487
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,fp8,511,0.00833439975976944
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,float16,1023,0.010451199859380722
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,fp8,1023,0.008454400300979614
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,float16,2047,0.012460800260305405
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,fp8,2047,0.010398399829864503
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,float16,4095,0.027235201001167296
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,fp8,4095,0.01340160071849823
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,float16,8191,0.045259198546409606
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,fp8,8191,0.028476798534393312
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,float16,16383,0.08200799822807311
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,fp8,16383,0.04690560102462769
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,float16,1,0.08642240166664124
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,fp8,1,0.07354720234870911
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,float16,3,0.0866208016872406
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,fp8,3,0.07331839799880982
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,fp8,15,0.03984479904174805
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,float16,7,0.08661119937896729
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,float16,15,0.08695359826087952
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,fp8,15,0.07302719950675965
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,float16,31,0.08827679753303527
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,fp8,31,0.07309759855270385
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,float16,127,0.051399999856948854
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,float16,63,0.0884607970714569
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,fp8,63,0.07393919825553893
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,fp8,127,0.0755631983280182
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,float16,127,0.09237920045852661
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,float16,255,0.157041597366333
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,fp8,255,0.09167519807815552
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,float16,511,0.15535520315170287
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,fp8,1,0.13790559768676758
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,float16,1,0.1648416042327881
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,fp8,511,0.15944639444351197
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,float16,511,0.30021440982818604
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,float16,3,0.165393602848053
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,float16,7,0.16661759614944457
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,fp8,7,0.13820960521697997
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,fp8,15,0.13952800035476684
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,float16,15,0.16712479591369628
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,float16,31,0.1664191961288452
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,fp8,31,0.13935519456863404
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,fp8,127,0.024241599440574645
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,float16,63,0.16671199798583985
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,float16,15,0.008441600203514098
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,float16,1,0.008355200290679932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,float16,127,0.1746896028518677
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,fp8,127,0.14132959842681886
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,fp8,1,0.008420799672603608
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,float16,3,0.008352000266313553
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,float16,7,0.008377599716186523
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,fp8,7,0.008393599838018417
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,fp8,255,0.17518240213394165
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,fp8,15,0.008348800241947174
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,float16,255,0.3047136068344116
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,fp8,31,0.008387199789285659
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,float16,63,0.00836160033941269
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,fp8,63,0.008340799808502197
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,float16,127,0.008436799794435502
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,fp8,127,0.008396799862384795
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,float16,255,0.008363199979066848
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,fp8,255,0.008367999643087386
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,float16,511,0.00982080027461052
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,fp8,511,0.008444800227880477
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,fp8,7,0.07315199971199035
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,float16,1023,0.01061599999666214
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,fp8,1023,0.010377600044012069
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,float16,2047,0.026840001344680786
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,fp8,2047,0.012452799826860428
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,float16,4095,0.04537599980831146
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,fp8,4095,0.026878398656845093
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,fp8,8191,0.04564320147037506
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,float16,16383,0.1534991979598999
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,fp8,16383,0.08586879968643188
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,fp8,3,0.13757920265197754
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,float16,1,0.010382399708032609
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,float16,3,0.010436800122261048
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,fp8,1,0.008390399813652038
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,fp8,63,0.13934240341186524
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,float16,7,0.010355199873447418
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,float16,15,0.010449600219726563
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,fp8,7,0.008430399745702744
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,float16,31,0.010380800068378448
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,float16,15,0.008379200100898742
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,float16,63,0.010441599786281586
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,float16,31,0.008376000076532364
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,float16,127,0.010447999835014344
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,fp8,63,0.008441600203514098
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,float16,255,0.010412800312042236
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,fp8,255,0.008963199704885483
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,float16,511,0.012511999905109405
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,fp8,511,0.010494399815797806
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,float16,1023,0.027463999390602113
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,fp8,1023,0.014103999733924866
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,float16,2047,0.04642240107059479
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,fp8,2047,0.028764799237251282
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,float16,4095,0.08273280262947083
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,float16,8191,0.08084800243377685
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,fp8,4095,0.047328001260757445
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,float16,8191,0.15455360412597657
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,float16,16383,0.3017008066177368
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,fp8,3,0.00843840017914772
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,fp8,15,0.00843520015478134
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,fp8,3,0.008390399813652038
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,fp8,31,0.008364800363779068
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,fp8,32767,0.30903360843658445
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,float16,32767,0.5915056228637695
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,float16,3,0.00902559980750084
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,float16,7,0.008430399745702744
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,float16,15,0.008423999696969987
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,fp8,127,0.008364800363779068
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,float16,31,0.008472000062465668
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,fp8,8191,0.0844223976135254
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,float16,63,0.008401600271463394
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,fp8,7,0.008404800295829773
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,float16,127,0.009372799843549728
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,fp8,3,0.008392000198364257
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,fp8,127,0.008416000008583068
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,float16,255,0.008441600203514098
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,fp8,255,0.008401600271463394
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,float16,511,0.008428800106048583
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,fp8,511,0.008433599770069123
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,float16,1023,0.008428800106048583
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,fp8,1023,0.00843840017914772
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,fp8,16383,0.15930880308151246
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,fp8,31,0.008382400125265121
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,fp8,4095,0.01048159971833229
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,fp8,15,0.008406399935483932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,float16,8191,0.01432960033416748
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,fp8,8191,0.012486399710178375
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,fp8,16383,0.014555199444293976
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,float16,32767,0.018139199912548067
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,fp8,32767,0.01650400012731552
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,float16,1,0.008392000198364257
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,fp8,1,0.00835999995470047
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,float16,3,0.008444800227880477
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,fp8,3,0.008401600271463394
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,float16,7,0.008419200032949447
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,fp8,7,0.00841279998421669
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,float16,15,0.008404800295829773
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,fp8,15,0.008404800295829773
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,float16,31,0.008459199965000153
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,fp8,31,0.008430399745702744
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,float16,63,0.008748800307512284
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,fp8,63,0.008448000252246856
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,float16,127,0.008513599634170532
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,fp8,127,0.008427199721336365
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,float16,255,0.008399999886751174
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,fp8,255,0.00843999981880188
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,float16,511,0.008374399691820144
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,fp8,511,0.008428800106048583
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,float16,1023,0.008392000198364257
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,fp8,1023,0.008398400247097015
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,float16,2047,0.010384000092744827
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,fp8,1,0.008430399745702744
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,fp8,2047,0.010360000282526016
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,float16,4095,0.010873600095510482
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,fp8,4095,0.010465600341558457
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,float16,8191,0.01276639997959137
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,fp8,8191,0.012468799948692322
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,float16,16383,0.014572800695896148
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,float16,2047,0.010409600287675857
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,fp8,16383,0.013284799456596375
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,float16,32767,0.031174400448799135
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,fp8,32767,0.016579200327396394
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,float16,1,0.008366400003433227
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,fp8,1,0.008470399677753449
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,float16,1,0.008878400176763534
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,float16,3,0.00835840031504631
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,fp8,3,0.00841279998421669
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,float16,16383,0.016497600078582763
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,float16,7,0.008416000008583068
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,fp8,7,0.008376000076532364
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,float16,15,0.008419200032949447
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,fp8,15,0.008393599838018417
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,float16,31,0.008420799672603608
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,fp8,31,0.008380799740552902
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,float16,63,0.008392000198364257
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,fp8,63,0.008312000334262848
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,fp8,127,0.008315200358629227
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,float16,127,0.008428800106048583
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,float16,255,0.008379200100898742
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,fp8,255,0.008343999832868576
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,float16,511,0.008991999924182892
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,float16,1023,0.010451199859380722
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,float16,2047,0.027060800790786745
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,fp8,2047,0.012484800070524216
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,float16,4095,0.045219200849533084
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,float16,8191,0.08167359828948975
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,fp8,8191,0.04531520009040833
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,fp8,63,0.008441600203514098
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,float16,16383,0.15343519449234008
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,fp8,16383,0.08282080292701721
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,fp8,32767,0.15854719877243043
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,float16,32767,0.297105598449707
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,float16,1,0.00888959988951683
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,float16,3,0.008856000006198883
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,fp8,1,0.008451200276613235
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,fp8,3,0.008416000008583068
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,float16,7,0.008452799916267396
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,fp8,7,0.008377599716186523
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,float16,15,0.008423999696969987
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,fp8,15,0.008423999696969987
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,float16,31,0.008449599891901017
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,fp8,31,0.00835999995470047
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,float16,63,0.00846880003809929
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,fp8,63,0.008392000198364257
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,float16,127,0.00864799991250038
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,fp8,127,0.008395200222730636
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,float16,255,0.008433599770069123
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,fp8,255,0.008355200290679932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,fp8,511,0.00841120034456253
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,float16,1023,0.010355199873447418
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,fp8,2047,0.008364800363779068
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,fp8,1023,0.00856959968805313
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,float16,2047,0.010393600165843963
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,fp8,2047,0.010444799810647965
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,float16,4095,0.012511999905109405
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,fp8,4095,0.011151999980211259
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,float16,8191,0.014548799395561219
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,float16,16383,0.02900480031967163
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,fp8,16383,0.015505599975585937
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,float16,32767,0.04903360009193421
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,fp8,1023,0.010382399708032609
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,fp8,32767,0.0308896005153656
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,float16,1,0.01241919994354248
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,fp8,4095,0.026974400877952574
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,float16,3,0.01242239996790886
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,fp8,3,0.010463999956846238
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,float16,7,0.012521600723266602
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,fp8,7,0.01056160032749176
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,float16,15,0.01241919994354248
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,fp8,15,0.010983999818563461
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,float16,31,0.01249760016798973
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,fp8,31,0.010452800244092942
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,float16,63,0.012507200241088867
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,fp8,63,0.010454399883747101
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,float16,127,0.012510399520397186
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,fp8,127,0.01138719990849495
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,float16,255,0.012518399953842163
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,fp8,255,0.010883200168609618
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,float16,511,0.02778559923171997
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,fp8,1023,0.028880000114440918
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,float16,1023,0.04724799990653992
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,fp8,2047,0.0473471999168396
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,float16,2047,0.08345119953155518
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,fp8,4095,0.08417760133743286
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,float16,4095,0.15780479907989503
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,fp8,8191,0.15862239599227906
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,float16,8191,0.3048111915588379
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,float16,511,0.008675199747085572
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,float16,1,0.016603200137615202
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,fp8,1,0.01459839940071106
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,float16,4095,0.010449600219726563
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,fp8,16383,0.30412960052490234
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,fp8,8191,0.012505599856376648
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,float16,16383,0.6097296237945556
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,float16,7,0.016638399660587312
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,float16,15,0.016646400094032288
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,fp8,15,0.014688000082969666
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,float16,31,0.017052799463272095
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,fp8,31,0.014575999975204468
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,fp8,1,0.011271999776363372
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,float16,63,0.017150400578975676
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,float16,127,0.01664319932460785
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,fp8,127,0.014590400457382201
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,float16,255,0.02714720070362091
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,fp8,255,0.014575999975204468
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,float16,511,0.04535680115222931
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,fp8,511,0.028297600150108338
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,fp8,511,0.014535999298095703
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,float16,1023,0.08235999941825867
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,fp8,1023,0.04745280146598816
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,fp8,2047,0.08512160181999207
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,float16,2047,0.15524319410324097
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,fp8,4095,0.15922880172729492
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,float16,4095,0.3010639905929565
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,float16,1,0.027505600452423097
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,fp8,1,0.02446880042552948
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,fp8,3,0.024600000679492952
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,float16,7,0.02767519950866699
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,fp8,7,0.0228752002120018
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,float16,15,0.026825600862503053
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,float16,3,0.0166143998503685
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,fp8,15,0.023046399652957916
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,fp8,3,0.014532800018787383
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,float16,31,0.02690880000591278
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,fp8,31,0.023296000063419343
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,float16,63,0.027302399277687073
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,float16,127,0.030881598591804504
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,fp8,63,0.02462719976902008
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,fp8,127,0.023958399891853333
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,float16,255,0.045598399639129636
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,fp8,255,0.02919520139694214
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,float16,511,0.08210560083389282
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,fp8,511,0.04740639925003052
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,fp8,7,0.014635199308395385
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,float16,1023,0.155840003490448
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,float16,1,0.008451200276613235
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,fp8,63,0.014575999975204468
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,fp8,2047,0.1567855954170227
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,float16,2047,0.29986560344696045
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,fp8,1,0.00835840031504631
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,float16,3,0.008329600095748901
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,fp8,3,0.008366400003433227
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,fp8,7,0.008323200047016144
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,float16,7,0.008392000198364257
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,fp8,15,0.00835999995470047
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,float16,31,0.008371199667453765
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,fp8,31,0.008403199911117553
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,float16,63,0.008382400125265121
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,fp8,63,0.008350399881601333
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,float16,127,0.008428800106048583
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,fp8,127,0.008404800295829773
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,float16,255,0.00843999981880188
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,fp8,255,0.008419200032949447
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,float16,511,0.008425600081682205
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,fp8,511,0.008353599905967712
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,float16,1023,0.008395200222730636
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,fp8,1023,0.00836160033941269
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,fp8,2047,0.008697599917650223
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,float16,4095,0.012444800138473511
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,fp8,4095,0.010473600029945374
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,fp8,8191,0.014319999516010285
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,float16,16383,0.046076801419258115
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,fp8,16383,0.028911998867988585
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,float16,32767,0.08215199708938599
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,fp8,32767,0.04729920029640198
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,float16,1,0.04743039906024933
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,float16,3,0.026958400011062623
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,fp8,1,0.04118559956550598
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,float16,3,0.04731999933719635
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,fp8,3,0.041176000237464906
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,fp8,7,0.04122720062732697
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,float16,7,0.04734239876270294
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,float16,15,0.04728319942951202
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,fp8,15,0.04111199975013733
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,float16,31,0.04732159972190857
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,fp8,31,0.0410863995552063
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,float16,63,0.049369600415229795
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,fp8,63,0.04107840061187744
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,float16,127,0.05144000053405762
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,fp8,127,0.0429280012845993
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,float16,255,0.0824288010597229
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,fp8,255,0.050195199251174924
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,fp8,511,0.008403199911117553
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,fp8,511,0.08481280207633972
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,float16,511,0.15531519651412964
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,fp8,1023,0.08417760133743286
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,float16,1,0.08672800064086914
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,fp8,1,0.07401760220527649
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,fp8,1023,0.15769280195236207
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,float16,7,0.08684639930725098
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,fp8,3,0.07394880056381226
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,float16,15,0.00833280012011528
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,float16,15,0.08825280070304871
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,float16,31,0.08852319717407227
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,fp8,31,0.07396960258483887
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,float16,63,0.0884447991847992
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,float16,2047,0.01040000021457672
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,fp8,63,0.07571679949760438
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,float16,8191,0.027939200401306152
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,float16,127,0.09241920113563537
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,fp8,127,0.07598559856414795
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,fp8,255,0.09129279851913452
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,float16,255,0.15637919902801514
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,float16,1,0.008387199789285659
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,fp8,1,0.008455999940633774
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,fp8,511,0.1594383955001831
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,float16,3,0.008441600203514098
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,fp8,3,0.008377599716186523
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,float16,7,0.00843840017914772
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,float16,15,0.008363199979066848
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,fp8,7,0.00843520015478134
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,fp8,15,0.008329600095748901
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,float16,31,0.00846880003809929
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,fp8,31,0.008350399881601333
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,float16,63,0.008366400003433227
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,fp8,63,0.008425600081682205
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,float16,127,0.008372800052165985
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,fp8,127,0.008452799916267396
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,fp8,255,0.008379200100898742
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,float16,511,0.008771199733018875
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,fp8,511,0.00843999981880188
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,float16,1023,0.010388799756765366
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,fp8,1023,0.009160000085830688
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,float16,2047,0.012470400333404541
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,fp8,2047,0.010388799756765366
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,float16,4095,0.02675839960575104
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,fp8,4095,0.013177600502967835
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,float16,3,0.0865776002407074
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,float16,8191,0.04588960111141205
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,fp8,8191,0.02887519896030426
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,fp8,7,0.07420480251312256
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,fp8,15,0.07399359941482545
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,fp8,16383,0.04705120027065277
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,1,0.008404800295829773
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,fp8,32767,0.08337600231170654
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,3,0.008345600217580795
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,1,0.008404800295829773
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,7,0.008423999696969987
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,7,0.008396799862384795
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,15,0.008367999643087386
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,3,0.00838399976491928
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,15,0.008372800052165985
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,31,0.008342400193214417
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,31,0.008390399813652038
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,127,0.008398400247097015
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,127,0.008390399813652038
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,63,0.008366400003433227
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,255,0.008387199789285659
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,255,0.008363199979066848
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,float16,511,0.300932788848877
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,511,0.00835840031504631
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,511,0.010422399640083313
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,1023,0.010401599854230881
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,2047,0.02751680016517639
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,1023,0.01157120019197464
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,2047,0.012467200309038163
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,4095,0.045289599895477296
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,8191,0.08123679757118225
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,4095,0.027356800436973572
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,16383,0.15328320264816284
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,16383,0.08253279924392701
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,float16,255,0.008654399961233138
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,32767,0.15632640123367308
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,32767,0.29620800018310545
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,1,0.008342400193214417
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,65535,0.581544017791748
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,float16,1023,0.30201919078826905
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,65535,0.3031872034072876
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,3,0.008457600325345992
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,7,0.008446399867534638
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,float16,16383,0.08211680054664612
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,15,0.008422400057315826
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,31,0.008398400247097015
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,float16,32767,0.15400639772415162
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,63,0.008638399839401244
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,63,0.008449599891901017
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,127,0.008502399921417237
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,1,0.008345600217580795
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,255,0.00841120034456253
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,8191,0.04615040123462677
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,255,0.008326400071382523
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,511,0.008403199911117553
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,511,0.008428800106048583
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,1023,0.008460800349712371
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,1023,0.008423999696969987
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,2047,0.010412800312042236
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,2047,0.008470399677753449
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,4095,0.010425599664449692
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,4095,0.010380800068378448
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,8191,0.012563200294971466
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,8191,0.012521600723266602
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,16383,0.01666560024023056
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,3,0.008380799740552902
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,15,0.008387199789285659
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,32767,0.018646399676799773
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,31,0.008422400057315826
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,32767,0.016571199893951415
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,65535,0.020703999698162077
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,65535,0.018667200207710268
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,1,0.008795200288295746
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,1,0.008392000198364257
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,3,0.008423999696969987
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,3,0.00835679993033409
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,7,0.009188800305128097
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,7,0.008416000008583068
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,63,0.009487999975681305
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,15,0.008423999696969987
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,15,0.008462399989366532
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,31,0.008377599716186523
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,63,0.008425600081682205
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,31,0.00846719965338707
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,127,0.008507200330495835
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,127,0.008422400057315826
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,63,0.008476799726486206
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,255,0.00841279998421669
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,255,0.008414400368928909
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,511,0.008763200044631958
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,127,0.008404800295829773
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,1023,0.008779200166463852
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,511,0.008454400300979614
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,2047,0.010411199927330018
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,2047,0.008403199911117553
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,4095,0.010518400371074677
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,4095,0.010476800054311753
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,8191,0.012489599734544754
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,16383,0.015390400588512421
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,16383,0.014588800072669984
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,7,0.008416000008583068
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,32767,0.016590400040149687
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,65535,0.03467040061950684
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,1,0.008420799672603608
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,65535,0.01836639940738678
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,3,0.008403199911117553
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,3,0.00870719999074936
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,7,0.008423999696969987
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,15,0.008444800227880477
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,15,0.008353599905967712
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,31,0.008379200100898742
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,31,0.008455999940633774
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,63,0.008460800349712371
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,63,0.008423999696969987
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,127,0.008448000252246856
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,127,0.00841120034456253
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,255,0.008355200290679932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,255,0.00835999995470047
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,511,0.008452799916267396
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,511,0.008369600027799606
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,1023,0.010427200049161912
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,1023,0.00939679965376854
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,2047,0.012414400279521943
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,2047,0.010473600029945374
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,4095,0.027695998549461365
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,4095,0.01327199935913086
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,8191,0.04522719979286194
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,8191,0.02871200144290924
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,16383,0.08124480247497559
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,16383,0.04619680047035217
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,1023,0.008423999696969987
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,8191,0.013887999951839447
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,32767,0.1531440019607544
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,32767,0.08268160223960877
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,1,0.008398400247097015
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,1,0.008372800052165985
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,1,0.008422400057315826
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,65535,0.15640480518341066
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,7,0.008446399867534638
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,3,0.00844319984316826
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,65535,0.2976320028305054
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,7,0.008460800349712371
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,15,0.00844319984316826
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,31,0.008457600325345992
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,31,0.008427199721336365
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,63,0.00846880003809929
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,15,0.00843999981880188
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,63,0.008403199911117553
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,127,0.008433599770069123
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,127,0.008350399881601333
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,255,0.008489599823951722
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,255,0.00843999981880188
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,511,0.00843840017914772
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,511,0.008401600271463394
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,1023,0.008495999872684479
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,1023,0.00876000002026558
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,2047,0.01045600026845932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,2047,0.010411199927330018
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,4095,0.012254399806261062
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,8191,0.012863999605178833
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,4095,0.010478399693965912
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,8191,0.012464000284671784
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,16383,0.016070400178432465
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,16383,0.014353600144386292
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,32767,0.03107680082321167
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,32767,0.016601599752902985
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,65535,0.049511998891830444
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,float16,1,0.01000479981303215
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,65535,0.03261600136756897
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,fp8,1,0.00844319984316826
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,float16,3,0.010471999645233154
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,fp8,3,0.008446399867534638
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,float16,7,0.010422399640083313
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,fp8,7,0.0083856001496315
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,float16,15,0.010406400263309478
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,fp8,15,0.008422400057315826
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,float16,31,0.01043360009789467
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,fp8,31,0.008436799794435502
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,float16,63,0.010414399951696397
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,fp8,63,0.008355200290679932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,float16,127,0.01040479987859726
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,16383,0.016625599563121797
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,fp8,127,0.008980800211429597
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,fp8,255,0.009065599739551544
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,float16,511,0.012535999715328216
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,fp8,511,0.010496000200510025
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,float16,1023,0.026848000288009644
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,fp8,1023,0.013019199669361114
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,float16,2047,0.04536640048027039
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,fp8,2047,0.028809601068496705
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,float16,4095,0.08223519921302795
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,fp8,4095,0.04724160134792328
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,fp8,8191,0.08431360125541687
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,float16,8191,0.15445120334625245
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,32767,0.018142400681972502
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,fp8,16383,0.1589967966079712
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,3,0.008433599770069123
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,7,0.008470399677753449
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,fp8,1,0.010480000078678131
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,float16,1,0.01244639977812767
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,fp8,3,0.01048159971833229
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,float16,7,0.012483199685811996
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,float16,3,0.012545600533485413
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,fp8,7,0.010444799810647965
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,float16,15,0.012476799637079239
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,fp8,32767,0.3096031904220581
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,fp8,15,0.01048159971833229
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,float16,32767,0.589414405822754
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,float16,31,0.012494400143623352
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,fp8,63,0.01048159971833229
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,float16,255,0.012510399520397186
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,float16,63,0.01247360035777092
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,float16,127,0.01252799928188324
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,fp8,255,0.010523200035095215
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,float16,511,0.027728000283241273
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,fp8,511,0.014571200311183929
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,float16,1023,0.0457616001367569
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,fp8,1023,0.029065600037574767
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,float16,2047,0.0824512004852295
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,float16,4095,0.15750240087509154
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,fp8,4095,0.08429759740829468
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,fp8,8191,0.15854239463806152
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,float16,8191,0.3043231964111328
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,float16,255,0.010440000146627427
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,float16,1,0.016646400094032288
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,float16,3,0.016991999745368958
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,fp8,3,0.014574399590492249
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,float16,7,0.0171984001994133
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,fp8,7,0.014868800342082978
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,float16,15,0.01770080029964447
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,float16,31,0.017467199265956877
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,fp8,31,0.015905599296092986
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,float16,63,0.017505599558353423
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,fp8,63,0.014643199741840363
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,float16,127,0.01777600049972534
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,fp8,127,0.01454399973154068
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,float16,255,0.027107200026512145
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,float16,16383,0.2995471954345703
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,fp8,255,0.014612799882888794
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,float16,511,0.045371198654174806
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,float16,1023,0.08166880011558533
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,fp8,511,0.028492799401283263
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,fp8,1023,0.04717440009117126
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,fp8,31,0.012214399874210358
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,float16,2047,0.15394400358200072
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,fp8,2047,0.08491680026054382
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,fp8,127,0.010678400099277497
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,float16,4095,0.2985471963882446
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,3,0.008417599648237229
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,7,0.008419200032949447
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,3,0.008451200276613235
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,fp8,2047,0.0475488007068634
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,15,0.008388800173997879
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,15,0.008463999629020691
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,31,0.008444800227880477
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,31,0.00843840017914772
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,63,0.008428800106048583
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,127,0.008511999994516373
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,127,0.00849440023303032
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,255,0.008819200098514557
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,255,0.008396799862384795
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,511,0.009436800330877303
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,511,0.00843840017914772
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,fp8,1,0.014628799259662628
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,1023,0.008923199772834779
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,1023,0.008404800295829773
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,2047,0.010465600341558457
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,fp8,15,0.015097600221633912
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,2047,0.010406400263309478
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,4095,0.01244800016283989
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,4095,0.01045600026845932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,8191,0.014508800208568573
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,8191,0.012544000148773193
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,16383,0.030580800771713258
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,16383,0.014596800506114959
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,32767,0.030980798602104186
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,32767,0.04766719937324524
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,65535,0.08419679999351501
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,65535,0.0493120014667511
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,float16,1,0.027060800790786745
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,float16,3,0.026984000205993654
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,fp8,3,0.024774399399757386
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,float16,7,0.02686559855937958
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,fp8,7,0.02478879988193512
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,float16,15,0.026817598938941957
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,fp8,15,0.0247871994972229
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,float16,31,0.027374398708343507
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,1,0.00841120034456253
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,fp8,31,0.024804799258708952
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,1,0.008433599770069123
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,fp8,4095,0.1603008031845093
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,fp8,63,0.02478879988193512
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,float16,127,0.029897600412368774
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,7,0.00841120034456253
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,fp8,127,0.024742400646209715
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,float16,255,0.046235200762748715
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,fp8,511,0.047860801219940186
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,float16,511,0.08225600123405456
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,float16,1023,0.1539232015609741
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,63,0.008436799794435502
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,float16,1,0.04737119972705841
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,float16,2047,0.29891200065612794
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,fp8,2047,0.15787999629974364
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,float16,3,0.047819200158119204
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,fp8,3,0.04139840006828308
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,float16,7,0.047537600994110106
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,fp8,7,0.041294398903846743
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,float16,15,0.04763680100440979
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,fp8,15,0.041231998801231386
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,float16,31,0.04825119972229004
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,fp8,31,0.04118399918079376
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,float16,63,0.04938560128211975
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,fp8,63,0.04121760129928589
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,float16,127,0.05102400183677673
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,fp8,127,0.04335519969463349
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,float16,255,0.08258079886436462
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,fp8,255,0.05133439898490906
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,fp8,1,0.024736000597476958
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,fp8,511,0.08584480285644532
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,float16,511,0.15419039726257325
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,1,0.008404800295829773
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,1,0.008420799672603608
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,float16,1023,0.2974560022354126
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,fp8,1023,0.15821440219879152
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,3,0.008430399745702744
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,7,0.008419200032949447
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,3,0.008340799808502197
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,7,0.008395200222730636
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,15,0.008371199667453765
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,31,0.00840959995985031
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,31,0.008408000320196151
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,float16,63,0.027398398518562316
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,63,0.008342400193214417
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,127,0.00836160033941269
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,127,0.008416000008583068
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,255,0.00841279998421669
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,255,0.008347199857234954
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,fp8,255,0.030321601033210754
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,511,0.008367999643087386
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,511,0.00841279998421669
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,1023,0.00894559994339943
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,2047,0.01040479987859726
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,1023,0.008689600229263305
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,2047,0.0091279998421669
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,4095,0.012427199631929398
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,4095,0.010579200088977813
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,8191,0.027518400549888612
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,fp8,1023,0.08433759808540345
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,16383,0.04529440104961395
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,32767,0.0816703975200653
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,32767,0.0473008006811142
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,fp8,1,0.041150400042533876
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,65535,0.15339200496673583
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,3,0.008448000252246856
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,15,0.008423999696969987
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,7,0.008416000008583068
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,63,0.008404800295829773
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,15,0.008420799672603608
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,8191,0.013710400462150574
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,16383,0.028867200016975403
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,31,0.008403199911117553
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,63,0.008408000320196151
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,1,0.008446399867534638
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,65535,0.08298879861831665
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,127,0.008459199965000153
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,255,0.008355200290679932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,1,0.00846560001373291
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,511,0.008363199979066848
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,511,0.00843840017914772
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,1023,0.010411199927330018
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,7,0.00830719992518425
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,1023,0.00843520015478134
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,2047,0.012503999471664428
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,2047,0.010473600029945374
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,15,0.00835999995470047
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,4095,0.027344000339508057
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,4095,0.013672000169754029
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,8191,0.028366398811340333
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,8191,0.04582720100879669
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,16383,0.08188959956169128
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,16383,0.04629279971122742
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,31,0.008446399867534638
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,32767,0.08281279802322387
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,63,0.00835999995470047
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,65535,0.2964623928070068
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,3,0.008379200100898742
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,1,0.008399999886751174
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,3,0.00838399976491928
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,131071,0.5828879833221435
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,131071,0.30089919567108153
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,7,0.00835999995470047
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,15,0.008355200290679932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,255,0.008376000076532364
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,31,0.008416000008583068
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,1,0.008425600081682205
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,63,0.00844319984316826
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,3,0.008422400057315826
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,63,0.008417599648237229
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,127,0.00840959995985031
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,127,0.008401600271463394
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,255,0.008430399745702744
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,255,0.008388800173997879
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,511,0.00846880003809929
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,511,0.008353599905967712
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,1023,0.00843520015478134
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,7,0.008420799672603608
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,2047,0.008420799672603608
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,2047,0.008766400068998337
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,4095,0.010521599650382995
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,32767,0.15380640029907228
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,4095,0.010462400317192078
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,8191,0.01250240057706833
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,16383,0.0166703999042511
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,16383,0.016596800088882445
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,8191,0.012534399330615998
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,15,0.008366400003433227
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,32767,0.018651199340820313
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,32767,0.016604800522327424
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,65535,0.020751999318599702
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,65535,0.01656000018119812
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,1,0.00870240032672882
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,1,0.008355200290679932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,3,0.008414400368928909
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,31,0.008452799916267396
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,7,0.008425600081682205
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,15,0.008404800295829773
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,15,0.00843999981880188
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,31,0.008540800213813782
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,31,0.008444800227880477
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,63,0.00844319984316826
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,63,0.008390399813652038
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,127,0.008507200330495835
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,127,0.008432000130414962
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,255,0.008425600081682205
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,255,0.008433599770069123
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,511,0.008473599702119828
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,511,0.008376000076532364
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,1023,0.008511999994516373
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,1023,0.008452799916267396
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,2047,0.010473600029945374
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,2047,0.010487999767065048
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,4095,0.010489600151777268
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,4095,0.010502400249242783
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,8191,0.012534399330615998
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,8191,0.012505599856376648
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,16383,0.01660960018634796
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,16383,0.016601599752902985
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,32767,0.018673600256443025
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,1023,0.008419200032949447
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,32767,0.016732800006866454
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,65535,0.020692799985408784
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,65535,0.018700799345970152
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,131071,0.03762399852275848
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,131071,0.02070080041885376
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,1,0.008348800241947174
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,1,0.008449599891901017
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,3,0.008433599770069123
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,127,0.00841279998421669
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,7,0.008345600217580795
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,65535,0.15458719730377196
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,131071,0.022758400440216063
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,31,0.008379200100898742
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,15,0.008392000198364257
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,3,0.008460800349712371
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,63,0.008369600027799606
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,63,0.008369600027799606
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,7,0.008379200100898742
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,127,0.008408000320196151
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,127,0.008419200032949447
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,255,0.008376000076532364
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,255,0.00841279998421669
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,511,0.008372800052165985
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,1023,0.00846719965338707
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,1023,0.008392000198364257
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,2047,0.010462400317192078
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,2047,0.009017600119113922
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,4095,0.012455999851226807
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,4095,0.01043360009789467
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,8191,0.027316799759864806
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,8191,0.013407999277114868
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,16383,0.04573439955711365
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,16383,0.02885279953479767
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,32767,0.08204320073127747
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,32767,0.047435200214385985
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,65535,0.15356639623641968
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,65535,0.08533759713172913
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,3,0.008382400125265121
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,7,0.008316799998283386
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,131071,0.020716799795627593
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,1,0.008587200194597244
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,1,0.008488000184297562
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,3,0.008403199911117553
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,3,0.008449599891901017
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,31,0.008379200100898742
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,131071,0.2982736110687256
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,7,0.008428800106048583
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,15,0.008444800227880477
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,31,0.009113600105047226
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,15,0.008430399745702744
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,31,0.008379200100898742
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,63,0.00841120034456253
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,63,0.008417599648237229
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,127,0.00840959995985031
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,127,0.008380799740552902
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,255,0.00844319984316826
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,255,0.008392000198364257
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,511,0.008495999872684479
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,511,0.008417599648237229
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,1023,0.00840959995985031
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,1023,0.00840959995985031
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,2047,0.010273600369691849
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,2047,0.008432000130414962
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,4095,0.010465600341558457
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,4095,0.010388799756765366
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,8191,0.01252640038728714
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,16383,0.014584000408649444
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,16383,0.014558400213718414
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,32767,0.016625599563121797
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,32767,0.01661760061979294
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,65535,0.03451040089130401
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,131071,0.053487998247146604
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,131071,0.035452800989151004
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,1,0.008336000144481659
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,1,0.00835679993033409
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,3,0.00833280012011528
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,3,0.008320000022649765
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,7,0.008347199857234954
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,7,0.008343999832868576
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,15,0.008423999696969987
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,15,0.008347199857234954
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,31,0.008345600217580795
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,31,0.00833439975976944
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,63,0.008347199857234954
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,63,0.008366400003433227
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,127,0.008430399745702744
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,255,0.008408000320196151
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,255,0.008326400071382523
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,511,0.009891200065612792
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,15,0.008369600027799606
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,511,0.008374399691820144
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,1023,0.011515200138092041
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,2047,0.026811200380325317
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,7,0.009043200314044953
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,2047,0.012537600100040435
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,4095,0.045177599787712096
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,511,0.008377599716186523
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,8191,0.08060960173606872
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,8191,0.04557600021362305
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,16383,0.15232959985733033
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,16383,0.08234239816665649
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,8191,0.013128000497817992
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,65535,0.01822720021009445
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,32767,0.2965440034866333
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,float16,1,0.010385599732398988
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,fp8,1,0.008844800293445587
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,float16,3,0.010502400249242783
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,fp8,3,0.008950400352478027
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,float16,7,0.010395199805498124
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,65535,0.29789440631866454
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,65535,0.5832367897033691
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,float16,15,0.0104592002928257
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,fp8,15,0.008721599727869034
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,float16,31,0.010443200170993806
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,fp8,31,0.008433599770069123
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,float16,63,0.010476800054311753
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,fp8,63,0.008392000198364257
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,float16,127,0.010385599732398988
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,fp8,127,0.008425600081682205
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,float16,255,0.010390400141477584
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,127,0.008376000076532364
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,float16,511,0.012481600046157837
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,fp8,511,0.010520000010728836
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,1023,0.010356800258159637
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,float16,1023,0.027327999472618103
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,fp8,255,0.00844319984316826
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,fp8,1023,0.013257600367069244
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,131071,0.1559983968734741
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,fp8,2047,0.02868480086326599
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,float16,4095,0.08214719891548157
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,fp8,4095,0.04752320051193237
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,4095,0.02744640111923218
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,float16,8191,0.15391360521316527
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,fp8,8191,0.08331519961357117
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,float16,1,0.012483199685811996
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,fp8,1,0.01130559965968132
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,float16,3,0.012492799758911132
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,float16,16383,0.2986176013946533
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,float16,7,0.012481600046157837
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,float16,15,0.012558400630950928
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,fp8,7,0.01249919980764389
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,fp8,15,0.011488000303506852
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,float16,31,0.012542399764060973
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,fp8,31,0.011510399729013443
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,float16,63,0.012542399764060973
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,32767,0.15428160429000853
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,fp8,63,0.011268799751996994
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,float16,127,0.012513600289821625
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,float16,255,0.01255200058221817
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,fp8,255,0.011561600118875503
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,fp8,127,0.011483199894428253
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,float16,511,0.027556800842285158
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,fp8,511,0.014558400213718414
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,float16,1023,0.04644480049610138
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,fp8,1023,0.028891199827194215
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,float16,2047,0.08267679810523987
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,fp8,2047,0.047276800870895384
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,float16,4095,0.15645760297775269
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,fp8,7,0.008396799862384795
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,fp8,4095,0.08406559824943542
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,1,0.008320000022649765
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,3,0.008427199721336365
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,float16,8191,0.3027888059616089
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,7,0.008486399799585343
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,7,0.008388800173997879
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,15,0.008488000184297562
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,15,0.008387199789285659
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,31,0.008462399989366532
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,31,0.008374399691820144
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,63,0.009415999799966813
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,63,0.00844319984316826
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,127,0.008524800091981888
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,float16,2047,0.04572800099849701
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,127,0.00838399976491928
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,255,0.008436799794435502
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,255,0.008401600271463394
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,511,0.008369600027799606
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,511,0.008401600271463394
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,1023,0.008451200276613235
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,1023,0.008444800227880477
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,2047,0.010452800244092942
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,2047,0.010486400127410889
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,4095,0.0112527996301651
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,8191,0.01284639984369278
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,fp8,16383,0.15810400247573853
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,8191,0.012478400021791458
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,16383,0.016518400609493257
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,16383,0.014499199390411378
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,32767,0.03150720000267029
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,32767,0.016540800034999848
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,65535,0.05136479735374451
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,65535,0.031384000182151796
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,float16,1,0.016607999801635742
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,131071,0.08648800253868102
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,131071,0.05027840137481689
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,fp8,1,0.016548800468444824
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,float16,3,0.016625599563121797
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,float16,7,0.016657599806785585
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,fp8,7,0.016598400473594666
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,float16,15,0.01684959977865219
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,fp8,15,0.016521599888801575
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,float16,31,0.016673600673675536
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,fp8,31,0.016524800658226015
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,float16,63,0.016646400094032288
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,fp8,63,0.01652960032224655
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,float16,127,0.01863519996404648
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,fp8,127,0.016572800278663636
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,float16,255,0.027401599287986755
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,fp8,8191,0.15621119737625122
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,float16,511,0.04603680074214935
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,fp8,511,0.02889919877052307
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,float16,1023,0.0822767972946167
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,fp8,1023,0.04833120107650757
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,float16,2047,0.1543503999710083
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,fp8,2047,0.08594880104064942
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,fp8,4095,0.15975680351257324
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,float16,4095,0.30076639652252196
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,float16,1,0.02715040147304535
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,fp8,1,0.024799999594688416
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,float16,3,0.026969599723815917
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,4095,0.010451199859380722
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,float16,7,0.027438399195671082
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,fp8,3,0.012425599992275238
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,fp8,7,0.024779200553894043
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,float16,15,0.028302401304244995
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,fp8,15,0.024915200471878052
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,float16,31,0.028336000442504884
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,fp8,31,0.024804799258708952
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,float16,63,0.02841919958591461
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,fp8,63,0.024784000217914583
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,float16,127,0.030851200222969055
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,fp8,127,0.02481279969215393
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,float16,255,0.04662080109119415
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,fp8,3,0.016598400473594666
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,float16,511,0.08249920010566711
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,fp8,511,0.049060800671577455
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,float16,1023,0.15417759418487548
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,1,0.008404800295829773
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,fp8,1023,0.08528800010681152
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,fp8,255,0.01656319946050644
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,1,0.00841120034456253
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,3,0.008387199789285659
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,1,0.00849440023303032
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,3,0.00846719965338707
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,3,0.008462399989366532
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,7,0.00843520015478134
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,fp8,2047,0.15758719444274902
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,float16,2047,0.29941439628601074
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,15,0.008455999940633774
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,63,0.008417599648237229
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,31,0.00841120034456253
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,31,0.008423999696969987
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,63,0.008423999696969987
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,127,0.008454400300979614
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,127,0.00843999981880188
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,511,0.008419200032949447
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,511,0.008524800091981888
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,1023,0.010417599976062775
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,1023,0.008532799780368805
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,255,0.008484800159931184
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,2047,0.010414399951696397
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,2047,0.010503999888896942
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,4095,0.012481600046157837
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,4095,0.010892800241708755
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,8191,0.01448799967765808
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,8191,0.012505599856376648
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,16383,0.0292959988117218
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,16383,0.014883199334144592
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,32767,0.030825600028038025
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,65535,0.08419520258903504
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,65535,0.04936160147190094
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,131071,0.15604959726333617
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,131071,0.08596479892730713
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,float16,1,0.021244800090789794
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,fp8,1,0.020734399557113647
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,float16,3,0.022710399329662324
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,fp8,3,0.020656000077724456
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,float16,7,0.022771200537681578
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,fp8,255,0.030884799361228944
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,float16,15,0.022724799811840057
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,fp8,15,0.020745599269866945
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,float16,31,0.022735999524593355
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,fp8,31,0.021184000372886657
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,float16,63,0.02274080067873001
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,fp8,63,0.020916800200939178
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,float16,127,0.022755199670791627
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,fp8,127,0.022468799352645875
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,float16,255,0.021027199923992157
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,fp8,255,0.02067999988794327
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,float16,511,0.01865919977426529
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,fp8,511,0.016663999855518342
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,fp8,1023,0.018713599443435668
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,float16,2047,0.024753600358963013
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,fp8,2047,0.022675199806690215
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,7,0.00843999981880188
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,float16,1,0.010454399883747101
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,float16,1023,0.020744000375270844
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,fp8,1,0.009113600105047226
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,float16,3,0.010416000336408614
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,fp8,3,0.00844319984316826
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,fp8,7,0.008379200100898742
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,255,0.008462399989366532
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,fp8,15,0.008488000184297562
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,float16,15,0.01037440001964569
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,float16,31,0.010364799946546554
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,fp8,31,0.010016000270843506
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,float16,63,0.010443200170993806
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,fp8,63,0.009180799871683121
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,fp8,127,0.00841279998421669
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,float16,255,0.010372799634933472
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,fp8,255,0.008401600271463394
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,float16,511,0.011500799655914306
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,fp8,511,0.010492800176143647
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,float16,1023,0.014484800398349762
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,fp8,1023,0.012515200674533844
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,float16,2047,0.028921601176261903
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,fp8,2047,0.0186271995306015
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,float16,1,0.012539200484752655
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,fp8,1,0.012427199631929398
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,fp8,3,0.0248879998922348
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,float16,3,0.012503999471664428
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,float16,7,0.012486399710178375
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,fp8,3,0.012534399330615998
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,fp8,7,0.012481600046157837
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,float16,15,0.012622399628162384
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,fp8,15,0.012408000230789185
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,float16,31,0.012518399953842163
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,fp8,31,0.012531200051307678
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,float16,63,0.012515200674533844
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,fp8,63,0.0125231996178627
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,float16,127,0.012515200674533844
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,fp8,127,0.01241919994354248
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,float16,255,0.012494400143623352
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,fp8,255,0.012468799948692322
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,float16,511,0.01659200042486191
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,fp8,511,0.014563199877738953
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,float16,1023,0.030427199602127076
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,fp8,7,0.020751999318599702
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,fp8,1023,0.018688000738620758
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,float16,2047,0.049353599548339844
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,float16,1,0.014572800695896148
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,fp8,2047,0.03291679918766022
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,float16,3,0.014571200311183929
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,fp8,1,0.012518399953842163
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,float16,7,0.014574399590492249
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,fp8,3,0.012492799758911132
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,fp8,7,0.012479999661445617
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,fp8,15,0.012518399953842163
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,fp8,31,0.012451200187206269
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,float16,31,0.01454399973154068
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,fp8,63,0.012464000284671784
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,float16,63,0.014483200013637542
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,float16,127,0.014535999298095703
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,fp8,127,0.01247519999742508
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,float16,255,0.014569599926471711
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,fp8,255,0.01252480000257492
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,float16,511,0.028921601176261903
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,fp8,511,0.016620799899101257
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,15,0.008369600027799606
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,fp8,1023,0.03094559907913208
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,fp8,2047,0.05059199929237366
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,float16,2047,0.08435360193252564
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,float16,7,0.010470400005578995
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,float16,1,0.020640000700950623
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,fp8,1,0.020612800121307374
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,float16,3,0.020732800662517547
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,float16,7,0.020715199410915375
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,fp8,7,0.018692800402641298
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,float16,127,0.010441599786281586
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,float16,15,0.020707200467586517
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,float16,31,0.020667199790477753
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,fp8,15,0.018695999681949616
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,fp8,31,0.02065120041370392
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,float16,63,0.020692799985408784
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,float16,127,0.02070239931344986
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,float16,255,0.020609599351882935
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,fp8,127,0.02070080041885376
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,fp8,255,0.01865600049495697
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,fp8,511,0.016729600727558136
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,float16,1023,0.014524799585342408
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,32767,0.048419201374053956
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,fp8,1023,0.014507199823856353
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,fp8,2047,0.013516800105571746
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,float16,2047,0.01449279934167862
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,float16,1,0.014467200636863709
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,fp8,1,0.014526399970054626
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,float16,3,0.014537599682807923
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,fp8,3,0.01451680064201355
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,float16,7,0.014483200013637542
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,fp8,7,0.01449279934167862
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,float16,15,0.014537599682807923
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,fp8,15,0.014500799775123595
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,float16,31,0.014523200690746307
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,fp8,31,0.01454080045223236
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,float16,63,0.014580799639225006
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,fp8,63,0.01449279934167862
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,fp8,127,0.014528000354766845
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,float16,255,0.014553600549697876
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,fp8,255,0.014535999298095703
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,float16,511,0.012583999335765839
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,float16,15,0.014521600306034088
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,fp8,511,0.012544000148773193
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,float16,1023,0.012537600100040435
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,fp8,1023,0.010907199978828431
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,float16,2047,0.01244800016283989
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,fp8,2047,0.010735999792814255
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,float16,1,0.011383999884128571
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,float16,3,0.01143840029835701
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,fp8,1,0.010500799864530563
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,float16,7,0.010492800176143647
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,fp8,7,0.010516799986362457
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,float16,1023,0.04760960042476654
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,float16,15,0.010604800283908844
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,fp8,15,0.01053759977221489
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,float16,31,0.010623999685049058
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,fp8,31,0.010452800244092942
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,float16,63,0.010710400342941285
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,fp8,63,0.010486400127410889
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,float16,127,0.012462399899959564
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,fp8,3,0.020660799741744996
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,fp8,127,0.010576000064611435
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,float16,255,0.011422400176525117
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,float16,511,0.010683199763298035
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,fp8,63,0.018643200397491455
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,fp8,511,0.010451199859380722
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,float16,1023,0.010425599664449692
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,fp8,1023,0.010369600355625152
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,float16,511,0.017025600373744964
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,fp8,2047,0.010462400317192078
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,float16,1,0.010416000336408614
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,float16,3,0.010363200306892395
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,fp8,1,0.01048159971833229
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,float16,7,0.010414399951696397
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,fp8,3,0.010436800122261048
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,fp8,7,0.010473600029945374
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,float16,15,0.010407999902963639
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,fp8,15,0.009521599858999252
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,fp8,31,0.010345599800348281
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,float16,63,0.010407999902963639
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,fp8,63,0.010417599976062775
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,float16,127,0.010440000146627427
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,fp8,127,0.0104032002389431
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,float16,255,0.010447999835014344
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,fp8,255,0.008916799724102021
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,float16,511,0.010424000024795533
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,fp8,511,0.009964799880981446
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,float16,1023,0.010422399640083313
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,fp8,1023,0.010440000146627427
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,float16,2047,0.010451199859380722
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,fp8,2047,0.010393600165843963
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,float16,1,0.020692799985408784
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,float16,127,0.01459839940071106
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,fp8,1,0.02065120041370392
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,float16,3,0.020681600272655486
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,fp8,3,0.02061759978532791
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,float16,7,0.020707200467586517
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,float16,15,0.020692799985408784
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,fp8,15,0.020654399693012238
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,float16,31,0.020703999698162077
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,fp8,31,0.020603199303150178
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,float16,63,0.020707200467586517
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,fp8,63,0.02062080055475235
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,float16,127,0.020598399639129638
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,fp8,127,0.020585599541664123
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,float16,255,0.020611199736595153
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,fp8,255,0.018692800402641298
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,float16,511,0.01687680035829544
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,fp8,511,0.016516800224781036
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,float16,1023,0.014575999975204468
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,fp8,1023,0.014472000300884247
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,float16,2047,0.014561599493026734
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,fp8,2047,0.014475199580192565
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,float16,1,0.014569599926471711
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,fp8,1,0.014483200013637542
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,float16,3,0.014545600116252898
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,fp8,3,0.014494399726390838
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,float16,7,0.014575999975204468
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,fp8,7,0.014486399292945863
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,fp8,3,0.01048159971833229
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,fp8,15,0.014580799639225006
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,float16,31,0.014611199498176575
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,fp8,31,0.014502400159835815
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,float16,63,0.01451520025730133
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,fp8,63,0.014523200690746307
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,float16,127,0.01451359987258911
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,fp8,127,0.014571200311183929
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,fp8,255,0.010452800244092942
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,float16,255,0.014662399888038635
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,float16,511,0.012518399953842163
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,fp8,511,0.01252640038728714
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,float16,1023,0.012464000284671784
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,float16,2047,0.011844799667596818
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,fp8,2047,0.010603199899196624
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,fp8,255,0.014511999487876893
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,float16,1,0.011748799681663513
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,float16,2047,0.010447999835014344
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,float16,3,0.011299200356006622
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,fp8,1,0.010489600151777268
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,float16,7,0.011644800007343293
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,fp8,3,0.01048159971833229
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,fp8,7,0.01048479974269867
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,fp8,15,0.010491199791431427
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,float16,31,0.01053600013256073
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,float16,31,0.010451199859380722
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,fp8,31,0.010416000336408614
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,float16,63,0.010500799864530563
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,fp8,63,0.010518400371074677
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,fp8,127,0.01085119992494583
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,float16,255,0.010603199899196624
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,float16,127,0.011615999788045884
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,fp8,255,0.01043040007352829
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,float16,511,0.010526400059461594
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,float16,1023,0.010500799864530563
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,fp8,1023,0.010444799810647965
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,float16,2047,0.012534399330615998
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,fp8,2047,0.010700800269842149
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,float16,1,0.010489600151777268
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,float16,3,0.010500799864530563
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,fp8,3,0.01005759984254837
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,float16,7,0.010515200346708298
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,fp8,7,0.00974240005016327
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,float16,15,0.010473600029945374
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,fp8,15,0.010489600151777268
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,float16,31,0.010449600219726563
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,fp8,31,0.010460799932479859
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,float16,63,0.010460799932479859
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,fp8,63,0.010334400087594986
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,float16,127,0.01048159971833229
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,float16,15,0.014550399780273438
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,fp8,127,0.009974399954080582
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,float16,255,0.010476800054311753
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,fp8,255,0.010371199995279311
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,float16,511,0.010444799810647965
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,fp8,511,0.010356800258159637
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,float16,1023,0.010465600341558457
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,fp8,1023,0.010369600355625152
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,float16,2047,0.011364799737930299
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,fp8,2047,0.010454399883747101
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,float16,1,0.020689600706100465
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,fp8,1,0.020686399936676026
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,fp8,1023,0.012484800070524216
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,fp8,3,0.020729599893093108
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,float16,7,0.02110079973936081
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,fp8,7,0.020819200575351714
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,float16,15,0.02115039974451065
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,fp8,15,0.02069759964942932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,float16,31,0.020737600326538087
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,fp8,31,0.020654399693012238
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,float16,63,0.020707200467586517
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,float16,15,0.011932799965143204
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,float16,127,0.020755200088024138
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,fp8,127,0.020744000375270844
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,float16,255,0.020737600326538087
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,fp8,255,0.0207056000828743
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,float16,511,0.018456000089645385
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,float16,1023,0.01542080044746399
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,fp8,511,0.010542400181293488
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,fp8,1023,0.014638400077819825
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,float16,2047,0.018571199476718904
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,fp8,2047,0.01664479970932007
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,fp8,1,0.010302399843931198
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,float16,1,0.014496000111103058
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,float16,3,0.014703999459743499
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,fp8,3,0.014528000354766845
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,fp8,7,0.020659199357032774
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,float16,7,0.014596800506114959
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,fp8,7,0.014502400159835815
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,float16,15,0.016492800414562227
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,fp8,15,0.014603200554847717
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,float16,31,0.015014399588108063
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,fp8,31,0.014574399590492249
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,float16,63,0.014588800072669984
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,float16,127,0.014686399698257446
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,fp8,127,0.01451520025730133
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,float16,255,0.014571200311183929
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,fp8,255,0.014524799585342408
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,float16,511,0.01454399973154068
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,fp8,511,0.012518399953842163
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,float16,1023,0.0150736004114151
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,fp8,1023,0.01451680064201355
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,float16,1,0.010463999956846238
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,fp8,2047,0.01663520038127899
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,fp8,1,0.010463999956846238
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,float16,3,0.01043040007352829
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,fp8,3,0.009523200243711472
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,float16,2047,0.018662400543689728
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,float16,7,0.010487999767065048
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,fp8,7,0.009712000191211701
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,fp8,15,0.010407999902963639
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,float16,31,0.010424000024795533
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,fp8,31,0.0104032002389431
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,float16,63,0.010491199791431427
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,fp8,63,0.010416000336408614
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,float16,127,0.010470400005578995
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,fp8,127,0.01040479987859726
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,float16,255,0.010471999645233154
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,float16,15,0.010452800244092942
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,fp8,255,0.010390400141477584
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,float16,511,0.010846400260925293
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,fp8,511,0.010440000146627427
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,float16,1023,0.014467200636863709
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,float16,3,0.020931200683116914
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,fp8,1023,0.012503999471664428
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,fp8,2047,0.017342400550842286
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,float16,1,0.010567999631166457
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,fp8,1,0.010441599786281586
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,float16,3,0.010412800312042236
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,fp8,3,0.010444799810647965
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,float16,7,0.010422399640083313
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,fp8,7,0.010385599732398988
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,float16,15,0.01058880016207695
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,fp8,15,0.010411199927330018
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,float16,31,0.010497599840164185
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,float16,63,0.010524799674749374
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,fp8,63,0.02078080028295517
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,fp8,31,0.010467199981212616
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,float16,127,0.010468800365924836
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,fp8,127,0.010491199791431427
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,float16,255,0.010492800176143647
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,fp8,255,0.010463999956846238
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,float16,511,0.014547200500965118
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,fp8,511,0.012476799637079239
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,float16,1023,0.0288239985704422
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,fp8,1023,0.014767999947071075
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,float16,2047,0.046910399198532106
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,fp8,1,0.014635199308395385
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,fp8,511,0.01666560024023056
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,fp8,2047,0.029707199335098265
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,float16,1,0.020691199600696562
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,fp8,1,0.020681600272655486
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,float16,3,0.020632000267505647
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,float16,7,0.02072319984436035
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,fp8,3,0.020707200467586517
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,fp8,7,0.020662400126457214
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,float16,15,0.020654399693012238
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,fp8,15,0.020713600516319274
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,float16,31,0.020688000321388244
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,fp8,31,0.02067999988794327
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,float16,63,0.020641599595546723
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,fp8,63,0.02069920003414154
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,float16,127,0.02078399956226349
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,float16,255,0.02062239944934845
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,fp8,127,0.02070239931344986
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,fp8,255,0.018680000305175783
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,float16,511,0.01663520038127899
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,fp8,511,0.016596800088882445
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,float16,1023,0.014628799259662628
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,fp8,1023,0.014569599926471711
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,float16,2047,0.014529600739479065
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,fp8,2047,0.014467200636863709
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,float16,1,0.014609600603580474
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,float16,3,0.014521600306034088
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,fp8,3,0.014591999351978302
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,float16,7,0.01462399959564209
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,fp8,7,0.014601600170135499
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,fp8,15,0.014535999298095703
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,float16,31,0.01462559998035431
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,fp8,31,0.014567999541759491
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,float16,63,0.014560000598430633
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,fp8,63,0.014567999541759491
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,float16,127,0.014550399780273438
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,fp8,127,0.014558400213718414
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,fp8,255,0.014547200500965118
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,float16,255,0.014564800262451171
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,float16,511,0.012505599856376648
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,fp8,511,0.012515200674533844
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,float16,1023,0.012529599666595458
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,float16,2047,0.01252799928188324
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,fp8,2047,0.011473599821329117
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,float16,1,0.010449600219726563
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,fp8,1,0.010576000064611435
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,float16,3,0.010491199791431427
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,fp8,3,0.010516799986362457
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,float16,2047,0.029078400135040282
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,float16,7,0.01053759977221489
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,float16,15,0.010486400127410889
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,fp8,15,0.01043360009789467
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,float16,31,0.012508800625801087
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,fp8,31,0.010609599947929382
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,float16,63,0.012494400143623352
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,fp8,63,0.010369600355625152
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,fp8,63,0.010592000186443329
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,float16,127,0.010507199913263321
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,float16,255,0.010929600149393082
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,fp8,255,0.010489600151777268
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,float16,511,0.010622400045394897
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,fp8,511,0.010412800312042236
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,float16,1023,0.010500799864530563
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,fp8,1023,0.010380800068378448
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,float16,2047,0.012435200065374375
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,fp8,2047,0.01109120026230812
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,fp8,63,0.014575999975204468
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,float16,1,0.008966399729251862
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,fp8,1,0.008401600271463394
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,float16,3,0.009700799733400345
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,fp8,3,0.008470399677753449
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,fp8,1,0.014577600359916686
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,float16,7,0.008483199775218964
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,fp8,7,0.008843199908733368
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,float16,15,0.00844319984316826
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,float16,15,0.014563199877738953
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,float16,31,0.008507200330495835
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,fp8,31,0.008441600203514098
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,float16,63,0.010344000160694122
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,float16,127,0.010438399761915207
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,fp8,63,0.008399999886751174
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,float16,255,0.00955360010266304
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,fp8,255,0.008523199707269669
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,float16,511,0.008396799862384795
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,fp8,127,0.008636800199747085
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,fp8,511,0.008433599770069123
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,float16,1023,0.010440000146627427
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,fp8,1023,0.008489599823951722
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,float16,2047,0.010659199953079224
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,fp8,2047,0.010409600287675857
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,fp8,1,0.010513599961996078
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,float16,3,0.012432000041007996
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,fp8,3,0.010513599961996078
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,float16,7,0.012441600114107132
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,fp8,1023,0.01045600026845932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,float16,15,0.012492799758911132
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,fp8,7,0.010542400181293488
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,fp8,15,0.01048479974269867
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,fp8,31,0.010807999968528747
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,float16,63,0.012476799637079239
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,fp8,63,0.010606399923563003
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,float16,127,0.012544000148773193
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,float16,31,0.012478400021791458
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,float16,255,0.01249760016798973
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,float16,511,0.014558400213718414
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,fp8,7,0.010955200344324113
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,fp8,511,0.012515200674533844
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,float16,1023,0.01871519982814789
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,fp8,1023,0.016569599509239197
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,float16,1,0.012534399330615998
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,fp8,255,0.010425599664449692
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,float16,3,0.012468799948692322
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,float16,7,0.01247360035777092
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,fp8,7,0.01093439981341362
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,fp8,127,0.010449600219726563
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,float16,15,0.012561599910259246
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,fp8,15,0.01231200024485588
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,float16,31,0.012483199685811996
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,fp8,31,0.012190400063991547
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,float16,63,0.01249919980764389
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,fp8,63,0.012115199863910676
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,float16,127,0.012457600235939026
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,fp8,127,0.01072160005569458
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,float16,255,0.01252640038728714
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,fp8,255,0.010497599840164185
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,float16,511,0.01658879965543747
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,fp8,511,0.014468799531459808
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,float16,1023,0.030963200330734252
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,fp8,1023,0.01868640035390854
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,fp8,1,0.016625599563121797
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,float16,3,0.01865919977426529
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,fp8,3,0.01658080071210861
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,float16,1,0.018665599822998046
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,float16,7,0.018651199340820313
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,fp8,7,0.017211200296878816
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,float16,15,0.018641600012779237
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,fp8,15,0.008551999926567078
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,fp8,15,0.016840000450611115
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,float16,31,0.018628799915313722
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,fp8,31,0.017019200325012206
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,float16,63,0.01874080002307892
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,fp8,63,0.017020800709724428
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,float16,255,0.020380799472332
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,float16,127,0.018588800728321076
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,fp8,255,0.0166143998503685
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,float16,511,0.03286559879779816
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,float16,1,0.012417600303888322
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,float16,1023,0.051635199785232545
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,fp8,1023,0.03299039900302887
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,float16,1,0.022745600342750548
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,fp8,1,0.018323199450969697
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,float16,3,0.022631999850273133
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,fp8,3,0.018254399299621582
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,float16,7,0.02271520048379898
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,fp8,7,0.018035200238227845
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,float16,15,0.02263360023498535
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,fp8,127,0.010502400249242783
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,fp8,15,0.018563200533390046
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,fp8,31,0.01810719966888428
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,float16,63,0.022759999334812164
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,fp8,63,0.018172800540924072
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,float16,127,0.0227183997631073
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,fp8,127,0.01842080056667328
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,float16,255,0.029476800560951234
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,fp8,255,0.01841920018196106
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,float16,511,0.048337599635124205
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,fp8,1,0.012518399953842163
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,fp8,3,0.012142399698495865
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,float16,1,0.016663999855518342
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,float16,1023,0.08597279787063598
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,fp8,1023,0.0512224018573761
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,fp8,1,0.014572800695896148
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,float16,3,0.016577599942684172
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,float16,7,0.016569599509239197
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,float16,15,0.016521599888801575
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,fp8,7,0.014574399590492249
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,fp8,15,0.014582400023937226
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,fp8,31,0.014577600359916686
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,float16,63,0.016545599699020384
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,fp8,63,0.014528000354766845
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,float16,127,0.016607999801635742
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,fp8,127,0.014561599493026734
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,float16,255,0.01653759926557541
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,fp8,255,0.014603200554847717
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,fp8,1,0.0166143998503685
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,float16,3,0.018593600392341612
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,fp8,3,0.016542400419712066
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,fp8,127,0.016612799465656282
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,float16,7,0.01775680035352707
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,fp8,7,0.016575999557971954
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,float16,15,0.017948800325393678
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,fp8,511,0.023083199560642243
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,fp8,15,0.016595199704170227
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,fp8,31,0.01658719927072525
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,float16,63,0.018174399435520173
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,fp8,63,0.01611199975013733
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,float16,127,0.018136000633239745
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,fp8,127,0.01607840061187744
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,float16,255,0.0186831995844841
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,fp8,255,0.01462559998035431
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,fp8,1,0.027209600806236266
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,float16,1,0.031009599566459656
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,float16,3,0.030921599268913268
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,float16,31,0.02266719937324524
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,fp8,3,0.027777600288391113
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,float16,7,0.030880001187324525
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,fp8,7,0.026958400011062623
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,float16,15,0.03097119927406311
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,fp8,15,0.028044798970222475
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,float16,31,0.030939200520515443
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,float16,63,0.03100000023841858
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,fp8,63,0.02727839946746826
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,fp8,127,0.027478399872779845
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,float16,255,0.035227200388908385
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,fp8,511,0.030771198868751525
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,fp8,255,0.028867200016975403
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,float16,127,0.03102560043334961
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,fp8,3,0.014552000164985656
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,float16,3,0.03704319894313812
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,fp8,3,0.028772801160812378
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,float16,7,0.03702239990234375
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,float16,31,0.0165583997964859
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,fp8,7,0.02871040105819702
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,float16,15,0.037124800682067874
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,fp8,15,0.027758398652076723
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,float16,31,0.03749440014362335
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,fp8,31,0.02815360128879547
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,float16,63,0.03723999857902527
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,float16,127,0.039131200313568114
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,fp8,127,0.028884801268577575
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,fp8,63,0.028865599632263185
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,float16,255,0.04932000041007996
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,fp8,255,0.032390400767326355
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,float16,1,0.01817920058965683
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,1,128,1,float16,float16,3,0.024784000217914583
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,1,128,1,float16,fp8,1,0.02211039960384369
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,1,128,1,float16,float16,7,0.024902400374412537
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,1,128,1,float16,fp8,7,0.0226160004734993
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,1,128,1,float16,float16,15,0.024769599735736846
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,1,128,1,float16,fp8,15,0.022726400196552275
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,1,128,1,float16,float16,31,0.024784000217914583
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,1,128,1,float16,fp8,31,0.02271520048379898
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,1,128,1,float16,float16,63,0.024801599979400634
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,1,128,1,float16,float16,127,0.024796800315380098
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,1,128,1,float16,fp8,127,0.022232000529766083
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,2,128,1,float16,float16,1,0.02683840095996857
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,2,128,1,float16,fp8,1,0.022811199724674224
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,2,128,1,float16,float16,3,0.026833599805831908
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,2,128,1,float16,fp8,3,0.022761599719524385
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,2,128,1,float16,float16,7,0.026855999231338502
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,fp8,31,0.027875199913978577
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,2,128,1,float16,fp8,7,0.02414879947900772
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,2,128,1,float16,float16,15,0.026859200000762938
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,2,128,1,float16,fp8,15,0.024366399645805357
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,2,128,1,float16,float16,31,0.02688640058040619
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,2,128,1,float16,fp8,31,0.024516800045967103
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,2,128,1,float16,float16,63,0.02685439884662628
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,2,128,1,float16,fp8,63,0.023507200181484222
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,2,128,1,float16,float16,127,0.028867200016975403
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,2,128,1,float16,fp8,127,0.02407840043306351
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,float16,1,0.03701759874820709
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,4,128,1,float16,fp8,1,0.04842559993267059
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,4,128,1,float16,float16,3,0.053523200750350955
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,4,128,1,float16,fp8,3,0.048747199773788455
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,4,128,1,float16,float16,7,0.054097598791122435
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,4,128,1,float16,fp8,7,0.04796159863471985
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,4,128,1,float16,float16,15,0.054225599765777587
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,4,128,1,float16,fp8,15,0.04772480130195618
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,4,128,1,float16,float16,31,0.05480480194091797
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,4,128,1,float16,fp8,31,0.04731520116329193
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,4,128,1,float16,float16,63,0.05727519989013672
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,1,128,1,float16,float16,1,0.024851199984550477
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,4,128,1,float16,float16,127,0.05975199937820434
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,4,128,1,float16,fp8,127,0.05123360157012939
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,8,128,1,float16,float16,1,0.06782720088958741
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,8,128,1,float16,fp8,1,0.0483487993478775
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,8,128,1,float16,float16,3,0.06778079867362977
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,8,128,1,float16,fp8,3,0.04771519899368286
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,8,128,1,float16,float16,7,0.06786239743232728
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,1,128,1,float16,fp8,3,0.022787199914455415
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,8,128,1,float16,fp8,7,0.04773919880390167
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,8,128,1,float16,float16,15,0.06803200244903565
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,8,128,1,float16,fp8,15,0.04734080135822296
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,8,128,1,float16,float16,31,0.0699567973613739
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,float16,31,0.01780160069465637
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,8,128,1,float16,fp8,31,0.04775039851665497
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,8,128,1,float16,fp8,63,0.04936000108718872
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,8,128,1,float16,float16,127,0.06886079907417297
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,8,128,1,float16,fp8,127,0.04940159916877747
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,float16,1,0.02070239931344986
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,fp8,1,0.02073120027780533
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,float16,3,0.02073120027780533
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,fp8,3,0.02072319984436035
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,float16,7,0.020740799605846405
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,fp8,7,0.020689600706100465
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,float16,15,0.020718400180339814
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,fp8,15,0.020735999941825865
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,float16,31,0.020734399557113647
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,fp8,31,0.02072319984436035
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,float16,63,0.020662400126457214
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,fp8,63,0.020747199654579163
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,float16,127,0.020686399936676026
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,fp8,127,0.02070080041885376
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,float16,255,0.020611199736595153
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,fp8,255,0.020744000375270844
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,float16,511,0.016672000288963318
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,float16,1023,0.014500799775123595
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,fp8,511,0.0165583997964859
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,fp8,1023,0.014500799775123595
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,float16,2047,0.014585599303245544
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,fp8,2047,0.014483200013637542
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,float16,1,0.014552000164985656
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,4,128,1,float16,float16,1,0.05347520112991333
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,fp8,1,0.014588800072669984
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,fp8,3,0.014577600359916686
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,float16,7,0.01454080045223236
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,float16,15,0.014577600359916686
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,fp8,15,0.014572800695896148
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,float16,31,0.014603200554847717
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,fp8,31,0.014585599303245544
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,float16,63,0.014633600413799287
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,fp8,63,0.01448799967765808
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,float16,127,0.01459839940071106
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,4,128,1,float16,fp8,63,0.04796639978885651
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,fp8,127,0.014494399726390838
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,float16,255,0.014532800018787383
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,fp8,255,0.014473600685596466
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,float16,511,0.012484800070524216
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,fp8,511,0.012529599666595458
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,float16,1023,0.012495999783277511
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,fp8,1023,0.011819200217723846
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,float16,2047,0.012532800436019897
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,fp8,2047,0.012443199753761292
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,float16,1,0.010908800363540649
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,fp8,1,0.0104032002389431
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,float16,3,0.010449600219726563
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,fp8,3,0.010385599732398988
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,float16,7,0.01072319969534874
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,fp8,7,0.010411199927330018
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,float16,15,0.010947199910879135
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,fp8,15,0.010502400249242783
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,float16,31,0.010419200360774993
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,fp8,31,0.010515200346708298
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,float16,63,0.010440000146627427
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,8,128,1,float16,float16,63,0.0687120020389557
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,fp8,63,0.010500799864530563
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,float16,127,0.010463999956846238
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,128,1,128,1,float16,fp8,63,0.02274399995803833
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,float16,255,0.010417599976062775
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,float16,511,0.010473600029945374
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,fp8,255,0.010473600029945374
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,fp8,1023,0.01048159971833229
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,float16,2047,0.012516799569129943
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,fp8,2047,0.011249600350856781
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,float16,1,0.010467199981212616
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,fp8,1,0.010435199737548828
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,fp8,3,0.010294400155544281
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,float16,7,0.01043199971318245
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,fp8,7,0.010406400263309478
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,float16,15,0.010096000134944915
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,fp8,15,0.008984000235795975
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,float16,31,0.010259199887514114
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,float16,3,0.014585599303245544
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,fp8,31,0.008846399933099746
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,float16,63,0.010375999659299851
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,fp8,63,0.010372799634933472
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,float16,127,0.010452800244092942
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,fp8,127,0.010380800068378448
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,float16,255,0.010782399773597717
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,fp8,255,0.010406400263309478
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,float16,511,0.010441599786281586
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,fp8,511,0.008436799794435502
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,float16,1023,0.010454399883747101
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,fp8,1023,0.010388799756765366
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,float16,2047,0.012529599666595458
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,fp8,2047,0.011486399918794632
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,1,128,1,float16,float16,1,0.039124798774719236
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,1,128,1,float16,fp8,1,0.03304960131645203
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,1,128,1,float16,float16,3,0.03916319906711578
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,1,128,1,float16,fp8,3,0.03311040103435516
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,1,128,1,float16,float16,7,0.03912320137023926
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,1,128,1,float16,fp8,7,0.03300639986991882
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,1,128,1,float16,float16,15,0.03913280069828033
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,1,128,1,float16,fp8,15,0.03298560082912445
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,1,128,1,float16,float16,31,0.03913280069828033
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,1,128,1,float16,fp8,31,0.033032000064849854
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,1,128,1,float16,fp8,63,0.033032000064849854
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,1,128,1,float16,float16,63,0.04046080112457275
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,2,128,1,float16,float16,1,0.04562079906463623
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,2,128,1,float16,fp8,1,0.03962079882621765
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,2,128,1,float16,float16,3,0.04553279876708984
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,2,128,1,float16,fp8,3,0.03961920142173767
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,2,128,1,float16,float16,7,0.045659199357032776
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,fp8,511,0.010289599746465683
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,2,128,1,float16,fp8,7,0.040092799067497256
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,float16,1023,0.010471999645233154
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,2,128,1,float16,float16,15,0.045840001106262206
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,2,128,1,float16,fp8,15,0.039480000734329224
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,2,128,1,float16,float16,31,0.04715999960899353
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,2,128,1,float16,float16,63,0.047651201486587524
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,2,128,1,float16,fp8,63,0.040227198600769044
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,float16,3,0.010487999767065048
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,4,128,1,float16,float16,1,0.1010591983795166
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,4,128,1,float16,float16,3,0.10073280334472656
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,4,128,1,float16,fp8,1,0.08828160166740417
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,4,128,1,float16,fp8,3,0.08853759765625
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,4,128,1,float16,float16,7,0.10188800096511841
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,4,128,1,float16,fp8,7,0.0882416009902954
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,4,128,1,float16,float16,15,0.10318880081176758
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,4,128,1,float16,fp8,15,0.08840799927711487
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,fp8,1,0.02858240008354187
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,4,128,1,float16,float16,31,0.10491839647293091
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,4,128,1,float16,fp8,31,0.09057279825210571
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,4,128,1,float16,float16,63,0.1064784049987793
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,fp8,7,0.014523200690746307
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,4,128,1,float16,fp8,63,0.0912447988986969
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,8,128,1,float16,fp8,1,0.08727679848670959
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,8,128,1,float16,float16,1,0.12736799716949462
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,8,128,1,float16,float16,3,0.12747360467910768
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,8,128,1,float16,float16,7,0.12795519828796387
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,8,128,1,float16,fp8,7,0.08710880279541015
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,8,128,1,float16,float16,15,0.12813440561294556
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,8,128,1,float16,fp8,15,0.08783519864082337
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,8,128,1,float16,float16,31,0.12884800434112548
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,8,128,1,float16,fp8,31,0.08863199949264526
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,8,128,1,float16,float16,63,0.12859840393066407
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,8,128,1,float16,fp8,63,0.08835520148277283
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,1,128,1,float16,float16,1,0.07005760073661804
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,1,128,1,float16,fp8,1,0.05961920022964477
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,1,128,1,float16,float16,3,0.07045120000839233
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,1,128,1,float16,float16,7,0.07018719911575318
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,1,128,1,float16,fp8,7,0.05953760147094726
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,1,128,1,float16,float16,15,0.07022079825401306
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,1,128,1,float16,fp8,15,0.05949119925498962
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,1,128,1,float16,float16,31,0.07112799882888794
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,1,128,1,float16,fp8,31,0.059787201881408694
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,2,128,1,float16,float16,1,0.0837440013885498
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,2,128,1,float16,fp8,1,0.0724511981010437
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,2,128,1,float16,float16,3,0.08464959859848023
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,2,128,1,float16,fp8,3,0.07199040055274963
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,2,128,1,float16,float16,7,0.08453119993209839
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,2,128,1,float16,fp8,7,0.07298880219459533
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,2,128,1,float16,float16,15,0.08451359868049621
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,2,128,1,float16,fp8,15,0.07315679788589477
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,2,128,1,float16,fp8,31,0.07274879813194275
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,4,128,1,float16,float16,1,0.19650399684906006
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,4,128,1,float16,fp8,1,0.1696720004081726
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,fp8,127,0.010390400141477584
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,4,128,1,float16,float16,3,0.1968224048614502
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,4,128,1,float16,fp8,3,0.16995359659194947
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,4,128,1,float16,float16,7,0.19729280471801758
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,2,128,1,float16,fp8,31,0.04010719954967499
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,4,128,1,float16,fp8,7,0.17080960273742676
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,4,128,1,float16,float16,15,0.19822239875793457
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,4,128,1,float16,fp8,15,0.17161600589752196
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,4,128,1,float16,float16,31,0.20118401050567628
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,8,128,1,float16,float16,1,0.2479680061340332
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,8,128,1,float16,fp8,1,0.16751199960708618
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,8,128,1,float16,fp8,3,0.16677119731903076
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,8,128,1,float16,float16,7,0.24839839935302735
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,8,128,1,float16,fp8,7,0.16628479957580566
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,8,128,1,float16,fp8,15,0.16615999937057496
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,128,8,128,1,float16,fp8,3,0.08664479851722717
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,8,128,1,float16,float16,31,0.24773919582366943
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,8,128,1,float16,fp8,31,0.1665824055671692
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,float16,1,0.020707200467586517
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,fp8,1,0.020684799551963805
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,fp8,3,0.020683200657367708
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,1,128,1,float16,fp8,3,0.05936639904975891
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,float16,7,0.02072480022907257
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,fp8,7,0.02069920003414154
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,fp8,15,0.02067680060863495
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,float16,31,0.020667199790477753
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,float16,15,0.02072480022907257
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,2,128,1,float16,float16,31,0.08509600162506104
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,float16,63,0.020776000618934632
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,fp8,63,0.020657600462436677
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,float16,127,0.020710399746894835
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,fp8,127,0.020716799795627593
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,float16,255,0.020670400559902193
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,fp8,255,0.020684799551963805
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,fp8,511,0.016612799465656282
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,float16,1023,0.014612799882888794
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,float16,511,0.01663679927587509
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,fp8,1023,0.014511999487876893
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,float16,2047,0.016599999368190767
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,fp8,2047,0.014609600603580474
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,fp8,1,0.014547200500965118
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,float16,3,0.014553600549697876
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,fp8,3,0.014571200311183929
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,float16,7,0.01451520025730133
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,float16,15,0.01488800048828125
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,4,128,1,float16,fp8,31,0.17268480062484742
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,fp8,15,0.014494399726390838
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,8,128,1,float16,float16,3,0.24983201026916504
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,float16,31,0.014593599736690522
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,fp8,31,0.014585599303245544
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,fp8,63,0.014537599682807923
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,float16,127,0.014608000218868256
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,128,8,128,1,float16,float16,15,0.249183988571167
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,fp8,127,0.014614400267601014
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,float16,255,0.014577600359916686
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,fp8,255,0.014585599303245544
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,fp8,511,0.012547199428081513
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,float16,511,0.013555200397968292
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,float16,1023,0.012467200309038163
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,fp8,1023,0.012451200187206269
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,float16,2047,0.014483200013637542
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,float16,3,0.02072640061378479
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,fp8,2047,0.012863999605178833
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,float16,3,0.010950399935245514
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,fp8,1,0.010438399761915207
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,fp8,3,0.010438399761915207
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,float16,7,0.010550399869680404
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,fp8,7,0.010435199737548828
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,fp8,15,0.010454399883747101
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,float16,31,0.010505600273609162
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,fp8,31,0.020692799985408784
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,fp8,31,0.010505600273609162
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,float16,63,0.010745599865913391
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,fp8,63,0.010494399815797806
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,float16,127,0.011142399907112122
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,fp8,127,0.01074879989027977
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,float16,255,0.010744000226259232
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,fp8,255,0.010502400249242783
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,float16,511,0.010462400317192078
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,float16,1023,0.0125231996178627
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,fp8,511,0.010465600341558457
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,fp8,1023,0.011369600147008895
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,float16,2047,0.014561599493026734
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,fp8,2047,0.014539200067520141
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,float16,1,0.008433599770069123
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,fp8,1,0.008414400368928909
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,float16,3,0.008425600081682205
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,fp8,3,0.008392000198364257
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,float16,7,0.008470399677753449
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,float16,1,0.014632000029087067
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,fp8,15,0.008377599716186523
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,float16,31,0.008473599702119828
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,fp8,7,0.014560000598430633
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,fp8,31,0.008388800173997879
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,float16,63,0.00841279998421669
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,fp8,63,0.008388800173997879
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,fp8,127,0.008372800052165985
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,float16,255,0.008401600271463394
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,fp8,255,0.008395200222730636
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,float16,511,0.010451199859380722
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,float16,63,0.014507199823856353
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,float16,1023,0.012535999715328216
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,fp8,1023,0.010473600029945374
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,fp8,2047,0.014494399726390838
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,float16,1,0.019326399266719817
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,fp8,1,0.018700799345970152
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,float16,3,0.019246399402618408
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,fp8,3,0.018699200451374055
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,float16,7,0.01867839992046356
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,float16,1,0.010523200035095215
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,fp8,7,0.0186831995844841
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,float16,15,0.018641600012779237
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,fp8,15,0.01860480010509491
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,float16,31,0.01870719939470291
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,fp8,31,0.01860480010509491
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,float16,15,0.010876800119876861
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,float16,63,0.018688000738620758
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,fp8,63,0.018964800238609313
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,float16,127,0.01905599981546402
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,float16,255,0.019308799505233766
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,fp8,255,0.01866399943828583
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,float16,511,0.01661120057106018
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,fp8,511,0.016540800034999848
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,float16,1023,0.018716800212860107
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,fp8,1023,0.017566399276256563
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,float16,2047,0.02268480062484741
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,fp8,2047,0.020608000457286835
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,float16,1,0.01040000021457672
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,fp8,1,0.008392000198364257
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,float16,3,0.010387200117111205
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,fp8,3,0.008396799862384795
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,float16,7,0.010372799634933472
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,fp8,7,0.008420799672603608
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,float16,15,0.010393600165843963
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,fp8,15,0.00844319984316826
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,float16,31,0.010422399640083313
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,fp8,31,0.008343999832868576
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,float16,63,0.010412800312042236
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,fp8,63,0.00833119973540306
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,float16,127,0.01040000021457672
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,fp8,7,0.008390399813652038
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,float16,255,0.010375999659299851
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,fp8,127,0.008564800024032593
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,float16,15,0.008342400193214417
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,fp8,255,0.008401600271463394
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,fp8,511,0.010460799932479859
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,float16,511,0.01249919980764389
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,fp8,1023,0.01250240057706833
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,float16,2047,0.02951200008392334
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,fp8,2047,0.017817600071430205
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,float16,1,0.012668800354003907
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,fp8,1,0.012503999471664428
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,float16,127,0.008463999629020691
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,float16,3,0.012590399384498597
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,float16,7,0.012484800070524216
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,fp8,7,0.01249919980764389
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,float16,15,0.012516799569129943
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,fp8,15,0.012492799758911132
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,float16,31,0.01250240057706833
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,fp8,511,0.009214399755001068
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,fp8,31,0.012449599802494049
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,float16,63,0.012547199428081513
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,fp8,3,0.012481600046157837
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,fp8,63,0.012464000284671784
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,float16,127,0.012534399330615998
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,fp8,127,0.012544000148773193
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,float16,255,0.012539200484752655
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,fp8,255,0.012494400143623352
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,float16,511,0.0165583997964859
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,float16,2047,0.027928000688552855
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,float16,1023,0.03070879876613617
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,fp8,511,0.01454080045223236
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,fp8,1023,0.01870879977941513
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,float16,1,0.014561599493026734
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,fp8,2047,0.03297759890556336
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,fp8,1,0.012481600046157837
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,float16,3,0.014523200690746307
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,fp8,3,0.012505599856376648
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,float16,7,0.014569599926471711
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,float16,15,0.014532800018787383
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,fp8,15,0.012520000338554382
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,float16,31,0.014558400213718414
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,fp8,31,0.01250240057706833
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,float16,63,0.01451359987258911
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,fp8,63,0.012491200119256973
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,fp8,127,0.012460800260305405
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,float16,127,0.01459999978542328
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,float16,255,0.014529600739479065
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,fp8,255,0.012465599924325943
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,float16,511,0.028870400786399842
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,float16,1023,0.04804159998893738
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,fp8,1023,0.030371201038360596
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,float16,2047,0.08479040265083312
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,fp8,127,0.018598400056362152
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,float16,1,0.01855839937925339
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,fp8,1,0.016616000235080718
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,float16,3,0.018535999953746794
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,fp8,3,0.016654400527477263
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,float16,7,0.01857759952545166
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,float16,15,0.018572799861431122
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,fp8,15,0.01661120057106018
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,float16,31,0.018534399569034576
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,fp8,31,0.01659200042486191
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,float16,63,0.01855839937925339
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,fp8,63,0.016603200137615202
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,float16,1023,0.01449279934167862
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,float16,127,0.018643200397491455
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,fp8,127,0.016633599996566772
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,fp8,255,0.016548800468444824
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,float16,511,0.014580799639225006
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,fp8,511,0.014545600116252898
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,float16,1023,0.012542399764060973
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,fp8,1023,0.012529599666595458
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,float16,2047,0.014185599982738495
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,fp8,2047,0.013078400492668152
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,fp8,1,0.01250080019235611
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,float16,3,0.012544000148773193
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,float16,2047,0.04907839894294739
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,fp8,3,0.012513600289821625
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,float16,7,0.01281919926404953
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,fp8,7,0.01250240057706833
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,fp8,7,0.01249919980764389
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,float16,15,0.012564800679683685
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,fp8,15,0.012529599666595458
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,float16,31,0.012540799379348756
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,fp8,31,0.012521600723266602
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,float16,63,0.012547199428081513
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,fp8,63,0.012505599856376648
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,fp8,127,0.012505599856376648
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,float16,255,0.012510399520397186
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,fp8,255,0.012544000148773193
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,fp8,511,0.016604800522327424
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,float16,511,0.012486399710178375
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,fp8,511,0.012503999471664428
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,float16,1023,0.012464000284671784
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,fp8,1023,0.010523200035095215
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,float16,2047,0.012438400089740754
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,float16,1,0.010411199927330018
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,fp8,1,0.010441599786281586
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,fp8,2047,0.010523200035095215
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,fp8,3,0.010486400127410889
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,float16,7,0.010425599664449692
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,float16,3,0.010449600219726563
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,fp8,7,0.010486400127410889
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,float16,15,0.010470400005578995
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,fp8,15,0.010460799932479859
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,float16,31,0.010491199791431427
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,fp8,7,0.016920000314712524
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,fp8,31,0.010435199737548828
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,float16,63,0.01045600026845932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,fp8,63,0.010419200360774993
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,fp8,127,0.010468800365924836
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,float16,255,0.01048159971833229
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,float16,511,0.010487999767065048
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,fp8,511,0.010391999781131745
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,fp8,255,0.010468800365924836
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,float16,1023,0.010502400249242783
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,fp8,1023,0.010454399883747101
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,float16,2047,0.010452800244092942
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,fp8,2047,0.0104592002928257
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,fp8,1,0.010385599732398988
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,float16,255,0.017788800597190856
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,fp8,3,0.010360000282526016
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,float16,7,0.010476800054311753
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,fp8,7,0.009329599887132644
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,float16,15,0.010380800068378448
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,fp8,15,0.009334400296211243
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,float16,31,0.01043360009789467
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,float16,1,0.012535999715328216
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,float16,63,0.010411199927330018
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,fp8,63,0.010416000336408614
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,float16,127,0.010380800068378448
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,fp8,127,0.010393600165843963
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,float16,255,0.01034879982471466
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,float16,127,0.012545600533485413
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,fp8,255,0.009712000191211701
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,float16,511,0.010369600355625152
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,fp8,511,0.009963200241327286
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,float16,1023,0.010407999902963639
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,fp8,1023,0.010382399708032609
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,float16,2047,0.01051200032234192
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,fp8,2047,0.010422399640083313
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,float16,1,0.01860000044107437
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,fp8,2047,0.04963839948177338
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,fp8,1,0.016649599373340606
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,float16,3,0.018580800294876097
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,fp8,3,0.018568000197410582
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,float16,7,0.018595199286937713
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,fp8,7,0.01677599996328354
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,float16,15,0.01863040030002594
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,fp8,15,0.018587200343608855
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,float16,31,0.01860480010509491
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,fp8,31,0.018654400110244752
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,float16,63,0.01865600049495697
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,fp8,63,0.016947199404239655
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,fp8,127,0.018561600148677825
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,float16,127,0.010569600015878677
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,float16,255,0.016641600430011748
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,fp8,255,0.0165583997964859
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,float16,511,0.014630399644374847
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,float16,1023,0.012622399628162384
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,fp8,1023,0.012520000338554382
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,float16,2047,0.014457599818706512
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,fp8,2047,0.012590399384498597
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,float16,1,0.012531200051307678
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,fp8,1,0.01250080019235611
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,float16,3,0.013214400410652161
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,fp8,3,0.01250240057706833
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,float16,7,0.013279999792575835
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,fp8,7,0.012510399520397186
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,float16,15,0.012630400061607362
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,float16,1,0.010406400263309478
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,fp8,15,0.01252480000257492
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,fp8,31,0.012531200051307678
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,float16,63,0.012585599720478059
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,float16,31,0.013158400356769562
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,float16,3,0.01040000021457672
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,fp8,63,0.012481600046157837
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,float16,127,0.012508800625801087
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,fp8,127,0.012545600533485413
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,float16,255,0.012582400441169738
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,fp8,255,0.012535999715328216
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,float16,511,0.012412799894809723
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,float16,1023,0.012438400089740754
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,fp8,1023,0.010521599650382995
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,fp8,2047,0.010454399883747101
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,fp8,1,0.010446400195360184
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,float16,3,0.010444799810647965
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,fp8,3,0.010438399761915207
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,float16,7,0.010446400195360184
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,fp8,31,0.010449600219726563
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,fp8,7,0.010435199737548828
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,float16,15,0.010425599664449692
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,fp8,15,0.010497599840164185
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,float16,63,0.010475199669599533
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,fp8,31,0.010411199927330018
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,float16,127,0.01043360009789467
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,float16,31,0.010478399693965912
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,fp8,127,0.010385599732398988
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,fp8,255,0.010478399693965912
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,fp8,63,0.010491199791431427
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,float16,511,0.010412800312042236
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,float16,1023,0.010446400195360184
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,float16,255,0.010444799810647965
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,fp8,1023,0.010454399883747101
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,float16,2047,0.01239520013332367
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,fp8,2047,0.010471999645233154
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,float16,1,0.010366400331258773
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,fp8,1,0.010344000160694122
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,float16,3,0.010388799756765366
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,float16,127,0.018615999817848207
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,fp8,3,0.009544000029563904
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,fp8,7,0.009656000137329101
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,fp8,511,0.01459999978542328
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,float16,15,0.010388799756765366
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,float16,31,0.010377600044012069
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,fp8,31,0.010353600233793258
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,float16,63,0.010454399883747101
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,fp8,63,0.010375999659299851
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,float16,127,0.010369600355625152
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,fp8,127,0.010267200320959092
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,float16,255,0.010375999659299851
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,fp8,255,0.009374400228261947
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,float16,511,0.010366400331258773
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,fp8,511,0.010342399775981902
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,float16,1023,0.010449600219726563
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,float16,2047,0.010475199669599533
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,fp8,1023,0.010441599786281586
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,float16,1,0.010462400317192078
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,float16,2047,0.01053600013256073
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,fp8,2047,0.01045600026845932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,float16,1,0.018668800592422485
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,float16,3,0.018636800348758698
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,fp8,3,0.01860159933567047
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,float16,7,0.018662400543689728
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,fp8,1,0.018670399487018586
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,fp8,7,0.01868640035390854
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,float16,15,0.018654400110244752
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,fp8,15,0.018590399622917177
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,fp8,31,0.018651199340820313
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,float16,31,0.01860480010509491
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,float16,63,0.018651199340820313
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,fp8,63,0.018587200343608855
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,float16,127,0.018676799535751343
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,float16,255,0.018688000738620758
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,fp8,127,0.018622399866580965
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,fp8,255,0.01849920004606247
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,float16,511,0.016897599399089813
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,fp8,511,0.016275200247764587
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,float16,1023,0.014496000111103058
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,fp8,1023,0.014539200067520141
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,float16,1,0.014523200690746307
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,fp8,2047,0.015014399588108063
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,fp8,1,0.012598399817943574
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,float16,3,0.014534400403499603
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,fp8,3,0.012516799569129943
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,float16,7,0.014460800588130951
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,fp8,7,0.012513600289821625
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,float16,15,0.014499199390411378
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,fp8,511,0.01045759990811348
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,fp8,15,0.012831999361515046
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,float16,31,0.014500799775123595
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,float16,63,0.014496000111103058
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,fp8,31,0.01390880048274994
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,fp8,63,0.014449599385261535
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,float16,127,0.014545600116252898
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,fp8,127,0.014276799559593201
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,float16,255,0.014561599493026734
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,fp8,255,0.014246399700641631
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,float16,511,0.012529599666595458
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,float16,1023,0.014567999541759491
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,float16,2047,0.018668800592422485
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,fp8,1023,0.014000000059604644
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,fp8,2047,0.016631999611854555
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,float16,1,0.01045759990811348
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,float16,3,0.010391999781131745
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,fp8,3,0.010417599976062775
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,float16,7,0.01043360009789467
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,fp8,7,0.010428799688816071
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,fp8,15,0.010427200049161912
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,fp8,15,0.010364799946546554
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,float16,31,0.010425599664449692
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,fp8,31,0.01037919968366623
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,float16,63,0.010411199927330018
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,fp8,63,0.01040479987859726
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,float16,127,0.010398399829864503
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,fp8,127,0.010409600287675857
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,float16,255,0.010428799688816071
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,fp8,255,0.010371199995279311
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,float16,511,0.011660800129175187
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,fp8,511,0.010447999835014344
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,float16,1023,0.014500799775123595
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,fp8,1023,0.01252640038728714
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,fp8,511,0.012412799894809723
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,float16,2047,0.02881920039653778
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,fp8,2047,0.01659359931945801
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,fp8,1,0.01040479987859726
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,fp8,3,0.010387200117111205
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,float16,3,0.01051200032234192
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,float16,7,0.010424000024795533
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,fp8,7,0.010382399708032609
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,fp8,15,0.010366400331258773
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,float16,15,0.010425599664449692
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,float16,31,0.01045600026845932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,fp8,31,0.0104032002389431
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,float16,63,0.01080000028014183
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,fp8,63,0.010377600044012069
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,float16,127,0.010864000022411346
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,fp8,127,0.010499200224876404
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,float16,255,0.010518400371074677
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,float16,2047,0.016633599996566772
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,fp8,255,0.010449600219726563
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,float16,511,0.012494400143623352
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,fp8,511,0.012425599992275238
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,float16,1023,0.02847839891910553
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,fp8,1023,0.01467519998550415
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,float16,2047,0.04706720113754272
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,fp8,2047,0.030108800530433653
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,float16,1,0.018639999628067016
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,fp8,1,0.01858399957418442
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,float16,3,0.018638400733470915
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,fp8,3,0.01855040043592453
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,fp8,511,0.01252640038728714
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,float16,7,0.018611200153827667
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,float16,15,0.018603199720382692
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,fp8,15,0.018651199340820313
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,float16,31,0.01855199933052063
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,fp8,1,0.009459199756383896
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,fp8,31,0.018643200397491455
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,float16,63,0.018651199340820313
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,float16,7,0.010371199995279311
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,fp8,63,0.01674560010433197
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,fp8,127,0.01855680048465729
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,float16,255,0.01855199933052063
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,fp8,255,0.016575999557971954
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,float16,511,0.014612799882888794
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,fp8,511,0.014587199687957764
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,float16,1023,0.014580799639225006
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,fp8,1023,0.012569600343704223
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,float16,2047,0.014523200690746307
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,fp8,2047,0.014590400457382201
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,float16,1,0.01252640038728714
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,fp8,1,0.012518399953842163
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,float16,3,0.013649600744247436
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,fp8,3,0.01257600039243698
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,float16,7,0.013670399785041809
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,fp8,7,0.012510399520397186
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,float16,15,0.0125231996178627
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,fp8,15,0.012540799379348756
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,float16,31,0.01358560025691986
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,fp8,31,0.01249919980764389
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,float16,63,0.01422400027513504
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,float16,127,0.012516799569129943
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,fp8,63,0.012516799569129943
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,fp8,127,0.012611199915409089
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,float16,255,0.012534399330615998
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,fp8,255,0.012534399330615998
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,float16,511,0.012488000094890594
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,fp8,511,0.012460800260305405
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,fp8,1023,0.010999999940395355
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,float16,2047,0.012468799948692322
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,float16,1023,0.012515200674533844
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,fp8,2047,0.010460799932479859
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,float16,1,0.01045600026845932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,float16,3,0.010471999645233154
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,fp8,3,0.010486400127410889
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,float16,7,0.010478399693965912
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,fp8,7,0.010492800176143647
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,float16,15,0.010444799810647965
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,fp8,15,0.010492800176143647
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,float16,31,0.01043199971318245
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,float16,63,0.01043360009789467
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,fp8,31,0.01048159971833229
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,fp8,63,0.010460799932479859
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,float16,127,0.01050880029797554
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,fp8,127,0.010486400127410889
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,float16,255,0.010476800054311753
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,fp8,255,0.010449600219726563
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,float16,511,0.010503999888896942
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,fp8,511,0.01045600026845932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,float16,1023,0.01050880029797554
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,fp8,1023,0.010489600151777268
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,float16,2047,0.012438400089740754
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,fp8,2047,0.012409599870443344
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,float16,1,0.009867200255393982
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,float16,3,0.009929600358009338
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,fp8,3,0.009372799843549728
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,float16,7,0.009880000352859497
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,fp8,7,0.008448000252246856
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,float16,15,0.008516799658536911
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,fp8,15,0.0084927998483181
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,float16,31,0.008774399757385254
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,fp8,31,0.008459199965000153
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,float16,63,0.008420799672603608
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,fp8,63,0.008979199826717377
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,fp8,7,0.01857919991016388
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,float16,127,0.010385599732398988
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,fp8,127,0.00873119980096817
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,float16,255,0.01027199998497963
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,fp8,255,0.008611200004816055
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,float16,511,0.008539199829101562
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,float16,127,0.0186256006360054
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,float16,1023,0.01043199971318245
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,float16,15,0.010436800122261048
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,float16,2047,0.010846400260925293
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,fp8,2047,0.010388799756765366
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,float16,1,0.01247360035777092
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,fp8,1,0.010507199913263321
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,float16,3,0.012433599680662155
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,fp8,3,0.010868799686431885
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,float16,7,0.01162400022149086
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,fp8,7,0.010502400249242783
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,float16,15,0.012374400347471236
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,fp8,15,0.010505600273609162
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,float16,1,0.01061279997229576
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,fp8,1,0.010409600287675857
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,float16,63,0.01223680004477501
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,fp8,63,0.010427200049161912
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,float16,127,0.01284160017967224
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,fp8,127,0.010475199669599533
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,float16,255,0.010486400127410889
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,fp8,255,0.010480000078678131
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,float16,511,0.012505599856376648
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,fp8,511,0.012483199685811996
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,fp8,1,0.00838399976491928
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,float16,1023,0.01844799965620041
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,float16,1,0.01252799928188324
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,fp8,1,0.0108815997838974
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,float16,3,0.01252480000257492
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,fp8,3,0.010513599961996078
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,float16,7,0.012486399710178375
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,fp8,7,0.010496000200510025
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,float16,15,0.01250240057706833
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,fp8,15,0.0110944002866745
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,float16,31,0.012481600046157837
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,fp8,511,0.00843520015478134
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,fp8,1023,0.008955200016498566
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,float16,63,0.012532800436019897
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,fp8,63,0.010471999645233154
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,float16,127,0.01252480000257492
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,fp8,127,0.010478399693965912
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,float16,255,0.012438400089740754
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,fp8,255,0.01048479974269867
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,float16,511,0.016603200137615202
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,fp8,511,0.014271999895572662
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,float16,1023,0.0296640008687973
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,float16,1,0.018668800592422485
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,fp8,1,0.016564799845218657
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,float16,3,0.01860480010509491
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,fp8,3,0.016607999801635742
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,float16,31,0.01244800016283989
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,float16,7,0.018651199340820313
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,fp8,7,0.017099200189113616
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,float16,15,0.018624000251293182
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,fp8,15,0.016628800332546233
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,float16,31,0.01865759938955307
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,fp8,31,0.017099200189113616
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,float16,63,0.018699200451374055
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,fp8,63,0.017183999717235564
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,float16,127,0.018729600310325622
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,fp8,127,0.0166703999042511
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,float16,255,0.018676799535751343
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,float16,511,0.03201279938220978
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,fp8,511,0.022686399519443512
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,fp8,255,0.01833759993314743
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,float16,1,0.02276480048894882
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,float16,1023,0.05114719867706299
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,fp8,1023,0.03291040062904358
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,fp8,1,0.018639999628067016
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,float16,3,0.02276480048894882
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,float16,7,0.02272319942712784
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,fp8,7,0.018611200153827667
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,float16,15,0.02274879962205887
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,fp8,1023,0.0165120005607605
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,fp8,15,0.018639999628067016
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,float16,31,0.02276639938354492
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,fp8,31,0.018598400056362152
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,fp8,63,0.018350400030612946
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,fp8,127,0.0186271995306015
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,float16,255,0.02889440059661865
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,fp8,255,0.018561600148677825
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,float16,511,0.04753600060939789
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,fp8,511,0.030100798606872557
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,float16,1023,0.08444160223007202
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,float16,1,0.016579200327396394
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,fp8,1,0.014535999298095703
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,fp8,31,0.010790400207042694
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,float16,3,0.016519999504089354
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,float16,7,0.016513599455356597
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,fp8,3,0.014571200311183929
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,fp8,7,0.014571200311183929
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,float16,15,0.01658719927072525
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,fp8,15,0.014459200203418732
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,float16,31,0.01653279960155487
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,float16,63,0.016648000478744505
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,fp8,1023,0.018675200641155243
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,fp8,63,0.014560000598430633
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,float16,127,0.016548800468444824
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,fp8,127,0.014532800018787383
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,float16,255,0.014923200011253357
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,fp8,255,0.014446400105953217
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,float16,1,0.017158399522304534
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,fp8,1,0.014558400213718414
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,float16,3,0.01839199960231781
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,float16,7,0.01791200041770935
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,fp8,3,0.014646400511264802
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,fp8,31,0.010995200276374817
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,float16,15,0.017161600291728973
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,fp8,15,0.014596800506114959
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,float16,31,0.016944000124931337
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,fp8,31,0.014596800506114959
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,float16,63,0.018188799917697906
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,fp8,63,0.015761600434780122
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,float16,127,0.018489600718021394
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,fp8,3,0.017497600615024568
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,fp8,127,0.015702399611473083
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,float16,255,0.018481600284576415
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,fp8,255,0.014579200744628906
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,float16,1,0.030856001377105712
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,fp8,1,0.02678399980068207
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,float16,3,0.030899199843406677
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,float16,63,0.02279199957847595
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,float16,127,0.022711999714374542
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,float16,7,0.03089280128479004
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,fp8,3,0.026841598749160766
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,fp8,7,0.0268095999956131
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,float16,31,0.030820798873901368
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,float16,15,0.031009599566459656
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,fp8,31,0.026870399713516235
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,float16,63,0.030873599648475646
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,fp8,63,0.026929599046707154
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,fp8,127,0.02688640058040619
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,fp8,1023,0.05087839961051941
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,float16,127,0.030947199463844298
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,float16,255,0.03349120020866394
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,fp8,255,0.028411200642585753
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,float16,1,0.037115201354026794
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,float16,3,0.0371071994304657
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,fp8,3,0.027153599262237548
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,float16,7,0.03733119964599609
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,fp8,7,0.027132800221443175
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,float16,15,0.0377375990152359
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,fp8,15,0.02746880054473877
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,float16,31,0.037143999338150026
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,fp8,31,0.028300800919532777
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,float16,63,0.03769600093364715
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,fp8,63,0.027051201462745665
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,float16,127,0.03935999870300293
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,fp8,127,0.02880159914493561
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,float16,255,0.04904319941997528
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,fp8,255,0.03203679919242859
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,1,128,1,float16,float16,1,0.022734400629997254
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,fp8,31,0.014484800398349762
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,1,128,1,float16,fp8,1,0.020712000131607056
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,1,128,1,float16,float16,3,0.023680000007152556
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,1,128,1,float16,fp8,3,0.02069759964942932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,1,128,1,float16,fp8,7,0.020878399908542632
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,1,128,1,float16,float16,15,0.0237184002995491
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,1,128,1,float16,fp8,15,0.02072799950838089
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,1,128,1,float16,fp8,31,0.020707200467586517
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,1,128,1,float16,float16,63,0.022859199345111846
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,1,128,1,float16,fp8,63,0.020716799795627593
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,1,128,1,float16,float16,127,0.023175999522209167
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,1,128,1,float16,fp8,127,0.02133920043706894
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,2,128,1,float16,float16,1,0.026739200949668883
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,2,128,1,float16,fp8,1,0.02274879962205887
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,2,128,1,float16,float16,3,0.026782399415969847
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,2,128,1,float16,fp8,3,0.02279199957847595
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,fp8,7,0.01456640064716339
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,2,128,1,float16,fp8,7,0.022785599529743194
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,2,128,1,float16,float16,15,0.02682879865169525
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,2,128,1,float16,fp8,15,0.02276639938354492
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,2,128,1,float16,float16,31,0.026824000477790832
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,2,128,1,float16,fp8,31,0.022734400629997254
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,2,128,1,float16,float16,63,0.026814401149749756
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,2,128,1,float16,fp8,63,0.022761599719524385
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,2,128,1,float16,float16,127,0.027263998985290527
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,2,128,1,float16,fp8,127,0.022759999334812164
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,4,128,1,float16,float16,1,0.05349439978599548
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,4,128,1,float16,fp8,1,0.047275200486183167
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,4,128,1,float16,float16,3,0.05345119833946228
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,4,128,1,float16,fp8,3,0.04736480116844177
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,4,128,1,float16,float16,7,0.0534496009349823
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,4,128,1,float16,fp8,7,0.04735200107097626
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,4,128,1,float16,float16,15,0.053495997190475465
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,4,128,1,float16,fp8,15,0.04734080135822296
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,4,128,1,float16,float16,31,0.053686398267745974
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,4,128,1,float16,fp8,31,0.047332799434661864
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,4,128,1,float16,float16,63,0.055936002731323244
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,fp8,15,0.027100801467895508
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,4,128,1,float16,fp8,63,0.047363200783729555
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,8,128,1,float16,float16,1,0.0681551992893219
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,fp8,1,0.027108800411224366
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,8,128,1,float16,float16,3,0.06797119975090027
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,8,128,1,float16,fp8,3,0.0473008006811142
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,8,128,1,float16,float16,7,0.06798239946365356
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,8,128,1,float16,fp8,7,0.047328001260757445
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,1,128,1,float16,float16,7,0.023972800374031066
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,8,128,1,float16,float16,15,0.06841760277748107
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,8,128,1,float16,fp8,15,0.047409600019454955
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,1,128,1,float16,float16,31,0.024489599466323852
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,8,128,1,float16,float16,31,0.06985920071601867
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,8,128,1,float16,fp8,31,0.04732959866523743
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,8,128,1,float16,float16,63,0.07011359930038452
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,8,128,1,float16,fp8,63,0.04750399887561798
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,float16,1,0.01860159933567047
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,8,128,1,float16,fp8,127,0.04933600127696991
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,fp8,1,0.01865759938955307
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,float16,3,0.018622399866580965
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,fp8,3,0.018646399676799773
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,float16,7,0.018595199286937713
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,fp8,7,0.01857759952545166
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,float16,15,0.018651199340820313
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,fp8,15,0.018641600012779237
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,float16,31,0.018628799915313722
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,fp8,31,0.018588800728321076
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,float16,63,0.01863519996404648
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,fp8,63,0.0186271995306015
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,float16,127,0.018624000251293182
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,fp8,127,0.01857600063085556
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,float16,255,0.018248000741004945
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,fp8,255,0.01706400066614151
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,fp8,511,0.014558400213718414
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,float16,1023,0.014455999433994293
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,fp8,1023,0.012807999551296235
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,float16,2047,0.014545600116252898
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,fp8,2047,0.014563199877738953
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,float16,1,0.012913599610328674
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,fp8,1,0.01250080019235611
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,float16,3,0.013179199397563934
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,fp8,3,0.012484800070524216
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,float16,7,0.012503999471664428
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,4,128,1,float16,float16,127,0.05779520273208618
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,fp8,7,0.012508800625801087
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,float16,15,0.012521600723266602
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,fp8,15,0.01252640038728714
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,4,128,1,float16,fp8,127,0.04920639991760254
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,float16,31,0.012583999335765839
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,fp8,31,0.013369600474834441
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,float16,63,0.012558400630950928
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,float16,127,0.014539200067520141
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,fp8,127,0.012555199861526489
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,float16,255,0.012603199481964112
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,fp8,255,0.012544000148773193
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,float16,511,0.012532800436019897
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,fp8,511,0.012476799637079239
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,float16,1023,0.012432000041007996
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,fp8,1023,0.01071999967098236
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,float16,2047,0.01276959925889969
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,fp8,2047,0.01162400022149086
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,float16,1,0.010391999781131745
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,float16,3,0.010435199737548828
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,fp8,3,0.010380800068378448
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,float16,7,0.01037919968366623
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,fp8,7,0.010449600219726563
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,float16,15,0.010460799932479859
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,8,128,1,float16,float16,127,0.07008640170097351
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,float16,31,0.010447999835014344
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,fp8,31,0.010446400195360184
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,2,128,1,float16,float16,7,0.02672159969806671
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,float16,63,0.010398399829864503
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,fp8,63,0.010454399883747101
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,float16,127,0.010476800054311753
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,fp8,127,0.010468800365924836
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,float16,255,0.01040000021457672
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,fp8,255,0.010411199927330018
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,fp8,511,0.010360000282526016
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,float16,1023,0.01043199971318245
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,fp8,1023,0.010462400317192078
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,float16,511,0.01661120057106018
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,float16,2047,0.012467200309038163
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,fp8,2047,0.011219199746847153
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,float16,3,0.010390400141477584
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,fp8,1,0.010344000160694122
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,fp8,3,0.010467199981212616
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,float16,7,0.010463999956846238
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,fp8,7,0.008664000034332275
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,float16,15,0.010407999902963639
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,fp8,15,0.008627200126647949
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,float16,31,0.010425599664449692
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,fp8,31,0.00865119993686676
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,float16,63,0.010454399883747101
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,fp8,63,0.008687999844551087
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,float16,127,0.010361599922180175
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,fp8,127,0.008956799656152726
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,float16,255,0.010425599664449692
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,fp8,255,0.008508799970149994
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,float16,511,0.009558399766683578
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,fp8,511,0.008388800173997879
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,float16,1023,0.010435199737548828
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,fp8,1023,0.010380800068378448
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,fp8,2047,0.012361600250005721
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,float16,2047,0.0125231996178627
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,96,8,128,1,float16,fp8,1,0.04736160039901734
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,1,128,1,float16,float16,1,0.037591999769210814
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,fp8,63,0.012491200119256973
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,1,128,1,float16,float16,3,0.03803519904613495
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,1,128,1,float16,float16,7,0.037833601236343384
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,1,128,1,float16,fp8,7,0.03294720053672791
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,1,128,1,float16,float16,15,0.037345600128173825
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,1,128,1,float16,fp8,15,0.03300159871578216
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,1,128,1,float16,float16,31,0.03804000020027161
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,1,128,1,float16,fp8,31,0.032979199290275575
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,fp8,1,0.010420800000429154
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,1,128,1,float16,float16,63,0.03846240043640137
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,1,128,1,float16,fp8,63,0.03300319910049439
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,2,128,1,float16,float16,1,0.045347198843955994
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,2,128,1,float16,fp8,1,0.03904480040073395
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,2,128,1,float16,float16,3,0.045296001434326175
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,2,128,1,float16,fp8,3,0.0390608012676239
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,2,128,1,float16,float16,7,0.04530400037765503
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,2,128,1,float16,fp8,7,0.0390751987695694
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,fp8,15,0.01045600026845932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,2,128,1,float16,float16,15,0.04540480077266693
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,2,128,1,float16,fp8,15,0.0389488011598587
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,2,128,1,float16,float16,31,0.04575519859790802
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,2,128,1,float16,fp8,31,0.03909600079059601
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,2,128,1,float16,float16,63,0.0474128007888794
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,2,128,1,float16,fp8,63,0.03914720118045807
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,4,128,1,float16,float16,1,0.0997983992099762
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,4,128,1,float16,fp8,1,0.08811360001564025
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,4,128,1,float16,float16,3,0.10091520547866821
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,4,128,1,float16,fp8,3,0.08829759955406188
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,4,128,1,float16,float16,7,0.10063199996948242
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,4,128,1,float16,fp8,7,0.08843039870262145
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,float16,511,0.010388799756765366
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,4,128,1,float16,fp8,15,0.088372802734375
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,4,128,1,float16,float16,31,0.10413600206375122
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,4,128,1,float16,float16,15,0.10130239725112915
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,float16,1,0.010422399640083313
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,4,128,1,float16,float16,63,0.10368959903717041
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,4,128,1,float16,fp8,63,0.09062560200691223
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,8,128,1,float16,float16,1,0.12927520275115967
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,8,128,1,float16,fp8,1,0.08652960062026978
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,8,128,1,float16,fp8,3,0.08712000250816346
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,8,128,1,float16,float16,3,0.12963680028915406
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,8,128,1,float16,float16,7,0.12986079454421998
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,8,128,1,float16,fp8,7,0.08651040196418762
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,8,128,1,float16,fp8,15,0.08740000128746032
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,8,128,1,float16,float16,15,0.13151999711990356
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,8,128,1,float16,float16,31,0.12983360290527343
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,8,128,1,float16,fp8,31,0.08841279745101929
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,8,128,1,float16,float16,63,0.1300112009048462
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,8,128,1,float16,fp8,63,0.08855360150337219
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,1,128,1,float16,float16,1,0.06688640117645264
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,1,128,1,float16,fp8,1,0.056196802854537965
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,1,128,1,float16,float16,7,0.06693440079689025
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,1,128,1,float16,float16,3,0.0669920027256012
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,1,128,1,float16,float16,15,0.0677456021308899
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,1,128,1,float16,fp8,15,0.056524801254272464
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,1,128,1,float16,fp8,1,0.032953599095344545
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,1,128,1,float16,float16,31,0.06836159825325012
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,1,128,1,float16,fp8,3,0.03287039995193482
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,2,128,1,float16,float16,1,0.08085600137710572
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,2,128,1,float16,float16,3,0.08173760175704955
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,2,128,1,float16,fp8,3,0.06953279972076416
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,2,128,1,float16,float16,7,0.08113120198249817
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,2,128,1,float16,fp8,7,0.06971359848976136
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,2,128,1,float16,float16,15,0.08221279978752136
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,2,128,1,float16,fp8,15,0.06993920207023621
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,2,128,1,float16,float16,31,0.08185279965400696
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,2,128,1,float16,fp8,31,0.07044000029563904
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,4,128,1,float16,float16,1,0.1927904009819031
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,4,128,1,float16,fp8,1,0.16668319702148438
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,4,128,1,float16,float16,3,0.19334720373153685
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,4,128,1,float16,fp8,3,0.1668015956878662
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,4,128,1,float16,float16,7,0.19334559440612792
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,4,128,1,float16,fp8,7,0.16716320514678956
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,4,128,1,float16,float16,15,0.193612802028656
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,4,128,1,float16,fp8,15,0.1677791953086853
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,96,4,128,1,float16,fp8,31,0.08829119801521301
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,4,128,1,float16,float16,31,0.19497439861297608
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,4,128,1,float16,fp8,31,0.16799039840698243
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,8,128,1,float16,fp8,1,0.1647279977798462
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,8,128,1,float16,float16,1,0.2522576093673706
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,8,128,1,float16,float16,3,0.2518320083618164
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,8,128,1,float16,fp8,3,0.16609920263290406
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,8,128,1,float16,float16,7,0.2521888017654419
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,8,128,1,float16,fp8,7,0.16608480215072632
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,8,128,1,float16,float16,15,0.25152480602264404
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,1,128,1,float16,fp8,3,0.05622400045394897
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,1,128,1,float16,fp8,7,0.0557856023311615
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,8,128,1,float16,fp8,15,0.16640959978103637
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,8,128,1,float16,float16,31,0.25167360305786135
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,8,128,1,float16,fp8,31,0.16596959829330443
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,fp8,1,0.018638400733470915
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,float16,3,0.018671999871730804
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,float16,7,0.0185248002409935
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,fp8,7,0.018632000684738158
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,float16,15,0.018676799535751343
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,1,128,1,float16,fp8,31,0.05591679811477661
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,fp8,3,0.018628799915313722
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,float16,31,0.018592000007629395
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,float16,63,0.018571199476718904
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,fp8,63,0.018595199286937713
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,96,2,128,1,float16,fp8,1,0.06906399726867676
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,float16,127,0.018648000061511995
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,fp8,127,0.018619200587272643
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,float16,255,0.01863519996404648
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,fp8,255,0.016659200191497803
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,fp8,511,0.014603200554847717
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,float16,1023,0.014532800018787383
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,float16,2047,0.014596800506114959
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,fp8,1023,0.012540799379348756
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,fp8,2047,0.014555199444293976
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,float16,1,0.014446400105953217
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,fp8,1,0.012567999958992004
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,float16,3,0.014511999487876893
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,fp8,3,0.01247519999742508
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,float16,7,0.014470399916172027
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,fp8,7,0.012561599910259246
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,float16,15,0.0144896000623703
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,fp8,15,0.012574400007724761
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,float16,31,0.014505599439144135
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,float16,63,0.014484800398349762
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,fp8,31,0.012556800246238708
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,fp8,63,0.012532800436019897
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,float16,127,0.014511999487876893
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,fp8,127,0.012558400630950928
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,float16,255,0.013230399787425995
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,fp8,255,0.012639999389648438
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,float16,511,0.012535999715328216
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,fp8,511,0.012481600046157837
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,float16,1023,0.012438400089740754
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,fp8,1023,0.012438400089740754
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,float16,2047,0.014499199390411378
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,float16,1,0.010513599961996078
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,fp8,1,0.01043199971318245
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,float16,3,0.010487999767065048
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,fp8,3,0.010425599664449692
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,float16,7,0.010503999888896942
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,fp8,7,0.01048159971833229
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,float16,15,0.010467199981212616
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,fp8,15,0.010440000146627427
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,float16,31,0.01048159971833229
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,fp8,2047,0.012516799569129943
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,fp8,31,0.010814400017261505
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,float16,63,0.010419200360774993
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,fp8,63,0.010475199669599533
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,fp8,127,0.010416000336408614
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,float16,255,0.01045759990811348
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,float16,127,0.010507199913263321
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,fp8,255,0.010395199805498124
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,float16,511,0.010391999781131745
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,fp8,511,0.01045759990811348
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,float16,1023,0.012457600235939026
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,fp8,1023,0.01048479974269867
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,fp8,2047,0.01305759996175766
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,float16,1,0.008366400003433227
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,float16,2047,0.014497600495815277
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,float16,3,0.008403199911117553
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,fp8,3,0.008401600271463394
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,float16,7,0.008446399867534638
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,fp8,7,0.00833119973540306
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,float16,15,0.008430399745702744
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,fp8,15,0.00833280012011528
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,float16,31,0.008363199979066848
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,fp8,31,0.008343999832868576
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,fp8,63,0.008347199857234954
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,float16,1,0.018638400733470915
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,float16,127,0.008425600081682205
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,fp8,127,0.008419200032949447
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,float16,255,0.008404800295829773
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,fp8,255,0.008395200222730636
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,fp8,15,0.01855680048465729
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,fp8,511,0.008406399935483932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,fp8,31,0.018585599958896637
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,float16,1023,0.012505599856376648
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,fp8,1023,0.010443200170993806
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,float16,2047,0.027671998739242552
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,fp8,2047,0.014433600008487701
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,float16,511,0.015326400101184846
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,float16,3,0.014564800262451171
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,fp8,3,0.014528000354766845
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,float16,7,0.014691199362277984
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,fp8,7,0.014579200744628906
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,float16,15,0.014905600249767304
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,fp8,15,0.014524799585342408
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,float16,31,0.014764800667762756
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,float16,63,0.014632000029087067
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,fp8,31,0.014574399590492249
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,fp8,63,0.014502400159835815
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,float16,127,0.015143999457359314
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,fp8,127,0.014585599303245544
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,float16,255,0.015268799662590028
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,fp8,255,0.014500799775123595
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,float16,511,0.014579200744628906
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,fp8,511,0.012494400143623352
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,float16,1023,0.01658719927072525
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,fp8,1023,0.014531199634075165
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,float16,2047,0.01863519996404648
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,fp8,2047,0.016599999368190767
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,fp8,4095,0.021116800606250763
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,float16,4095,0.033190399408340454
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,float16,1,0.010390400141477584
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,fp8,1,0.010356800258159637
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,float16,3,0.010438399761915207
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,fp8,3,0.010254400223493576
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,float16,7,0.010420800000429154
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,fp8,7,0.01032480001449585
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,float16,15,0.010387200117111205
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,fp8,15,0.01048159971833229
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,fp8,31,0.010468800365924836
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,float16,63,0.010422399640083313
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,fp8,63,0.010427200049161912
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,float16,127,0.010449600219726563
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,fp8,127,0.010366400331258773
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,float16,255,0.010449600219726563
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,fp8,255,0.010417599976062775
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,fp8,1,0.00835999995470047
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,float16,511,0.011268799751996994
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,fp8,511,0.01040000021457672
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,float16,1023,0.014508800208568573
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,fp8,1023,0.01247360035777092
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,float16,2047,0.028969600796699524
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,float16,63,0.008416000008583068
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,fp8,2047,0.017075200378894807
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,float16,4095,0.04778079986572266
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,fp8,4095,0.030454400181770324
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,float16,1,0.010707200318574906
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,fp8,1,0.010489600151777268
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,float16,3,0.010465600341558457
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,float16,511,0.010364799946546554
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,fp8,7,0.010382399708032609
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,float16,15,0.010515200346708298
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,fp8,15,0.01037440001964569
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,float16,31,0.010440000146627427
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,float16,1,0.01478240042924881
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,float16,63,0.010454399883747101
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,fp8,63,0.0104592002928257
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,fp8,31,0.010460799932479859
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,float16,127,0.010486400127410889
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,fp8,127,0.010452800244092942
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,float16,255,0.010766399651765823
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,fp8,255,0.010375999659299851
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,float16,511,0.014452800154685974
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,fp8,511,0.012428800016641617
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,float16,1023,0.028867200016975403
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,float16,2047,0.0472927987575531
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,fp8,2047,0.02958880066871643
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,float16,4095,0.08386080265045166
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,fp8,4095,0.04920479953289032
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,float16,1,0.012489599734544754
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,fp8,1,0.010446400195360184
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,float16,3,0.01252640038728714
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,fp8,3,0.01048320010304451
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,float16,7,0.012444800138473511
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,fp8,7,0.010516799986362457
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,float16,15,0.012455999851226807
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,fp8,15,0.010521599650382995
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,float16,31,0.012518399953842163
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,fp8,31,0.01146719977259636
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,float16,63,0.01242239996790886
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,fp8,63,0.011423999816179276
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,float16,127,0.012465599924325943
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,float16,31,0.010369600355625152
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,fp8,127,0.011585599929094314
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,fp8,255,0.010460799932479859
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,float16,511,0.02889440059661865
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,fp8,511,0.014524799585342408
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,float16,1023,0.04734880030155182
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,fp8,1023,0.02957119941711426
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,float16,2047,0.08408799767494202
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,fp8,2047,0.04862399995326996
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,float16,1,0.014552000164985656
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,float16,4095,0.15985440015792846
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,fp8,4095,0.08790400028228759
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,fp8,1,0.014574399590492249
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,float16,3,0.014564800262451171
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,float16,7,0.014529600739479065
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,fp8,3,0.014547200500965118
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,fp8,7,0.014539200067520141
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,float16,15,0.014500799775123595
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,fp8,15,0.01451680064201355
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,float16,31,0.014542399346828461
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,fp8,31,0.014548799395561219
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,float16,63,0.014596800506114959
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,fp8,63,0.014577600359916686
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,float16,127,0.014547200500965118
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,fp8,127,0.014582400023937226
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,float16,255,0.014575999975204468
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,fp8,255,0.014553600549697876
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,float16,511,0.01255200058221817
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,fp8,3,0.01043360009789467
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,float16,1023,0.012425599992275238
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,fp8,1023,0.010539200156927109
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,float16,7,0.010552000254392624
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,fp8,2047,0.010460799932479859
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,float16,4095,0.012511999905109405
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,float16,1,0.010502400249242783
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,fp8,1,0.010480000078678131
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,fp8,4095,0.01255040019750595
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,float16,3,0.010478399693965912
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,fp8,3,0.010478399693965912
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,fp8,1,0.014569599926471711
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,float16,15,0.010486400127410889
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,float16,31,0.01167839989066124
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,fp8,31,0.01141280010342598
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,float16,63,0.01202080026268959
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,fp8,63,0.010470400005578995
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,fp8,1023,0.014601600170135499
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,float16,127,0.01241919994354248
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,fp8,127,0.010447999835014344
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,float16,255,0.011367999762296677
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,fp8,255,0.010465600341558457
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,float16,511,0.010583999752998351
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,fp8,511,0.01090560033917427
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,float16,1023,0.010398399829864503
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,fp8,1023,0.010447999835014344
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,float16,2047,0.01050880029797554
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,fp8,2047,0.010417599976062775
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,float16,4095,0.012559999525547028
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,fp8,4095,0.012409599870443344
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,float16,1,0.01037440001964569
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,float16,255,0.012409599870443344
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,fp8,1,0.010451199859380722
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,float16,3,0.010393600165843963
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,fp8,3,0.009379199892282485
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,float16,7,0.010390400141477584
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,fp8,7,0.009372799843549728
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,fp8,15,0.009243199974298478
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,float16,31,0.010414399951696397
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,fp8,31,0.00939520001411438
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,float16,63,0.01040000021457672
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,float16,15,0.010344000160694122
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,fp8,63,0.009403199702501298
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,fp8,127,0.00942239984869957
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,float16,255,0.010449600219726563
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,fp8,255,0.010387200117111205
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,float16,511,0.010380800068378448
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,float16,127,0.010390400141477584
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,fp8,511,0.012495999783277511
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,float16,1023,0.010406400263309478
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,float16,2047,0.01048320010304451
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,float16,2047,0.011952000111341477
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,fp8,2047,0.010468800365924836
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,float16,4095,0.011321599781513213
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,fp8,4095,0.01242400035262108
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,float16,1,0.008363199979066848
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,fp8,1,0.008454400300979614
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,fp8,7,0.010480000078678131
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,float16,3,0.008369600027799606
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,fp8,3,0.00843520015478134
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,float16,7,0.008422400057315826
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,fp8,15,0.010515200346708298
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,float16,15,0.008449599891901017
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,fp8,15,0.008377599716186523
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,float16,31,0.00844319984316826
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,fp8,31,0.00835840031504631
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,float16,63,0.008419200032949447
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,float16,127,0.008611200004816055
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,fp8,127,0.008312000334262848
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,fp8,63,0.008457600325345992
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,float16,255,0.00844319984316826
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,float16,511,0.008460800349712371
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,fp8,511,0.008432000130414962
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,float16,1023,0.008432000130414962
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,fp8,1023,0.008457600325345992
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,float16,2047,0.010444799810647965
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,fp8,2047,0.010377600044012069
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,fp8,4095,0.010502400249242783
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,float16,1,0.014577600359916686
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,fp8,1,0.014526399970054626
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,float16,3,0.01451520025730133
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,fp8,3,0.014481599628925323
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,fp8,7,0.014472000300884247
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,float16,15,0.014563199877738953
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,fp8,15,0.014475199580192565
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,float16,31,0.014537599682807923
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,fp8,31,0.014547200500965118
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,float16,63,0.01454399973154068
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,fp8,63,0.014511999487876893
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,float16,127,0.014561599493026734
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,fp8,127,0.014440000057220459
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,float16,255,0.014440000057220459
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,fp8,511,0.010465600341558457
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,fp8,1023,0.009406399726867676
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,float16,511,0.012511999905109405
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,float16,1023,0.012432000041007996
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,fp8,1023,0.012443199753761292
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,float16,2047,0.012438400089740754
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,float16,7,0.010452800244092942
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,fp8,2047,0.010475199669599533
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,float16,4095,0.012600000202655792
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,fp8,4095,0.012520000338554382
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,float16,1,0.010467199981212616
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,fp8,1,0.010471999645233154
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,float16,3,0.010473600029945374
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,fp8,3,0.010486400127410889
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,float16,7,0.010556799918413162
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,fp8,255,0.008448000252246856
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,fp8,7,0.01045759990811348
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,float16,15,0.010465600341558457
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,fp8,15,0.01045759990811348
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,fp8,31,0.010447999835014344
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,float16,63,0.012403199821710587
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,fp8,63,0.01058719977736473
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,float16,127,0.012383999675512314
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,fp8,127,0.010556799918413162
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,float16,4095,0.011334399878978729
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,fp8,255,0.010444799810647965
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,float16,511,0.010507199913263321
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,fp8,511,0.010883200168609618
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,float16,1023,0.010396800190210342
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,fp8,1023,0.01053600013256073
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,float16,2047,0.010476800054311753
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,fp8,2047,0.010505600273609162
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,float16,4095,0.012521600723266602
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,fp8,4095,0.012451200187206269
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,float16,1,0.010388799756765366
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,fp8,1,0.009363199770450591
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,float16,7,0.01446239948272705
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,fp8,3,0.008408000320196151
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,float16,7,0.010353600233793258
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,fp8,7,0.0090768001973629
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,float16,15,0.010395199805498124
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,fp8,15,0.00939679965376854
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,float16,31,0.010406400263309478
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,fp8,31,0.010409600287675857
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,float16,63,0.010425599664449692
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,fp8,63,0.010384000092744827
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,float16,127,0.010363200306892395
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,fp8,127,0.010097599774599075
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,fp8,255,0.01435679942369461
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,fp8,255,0.0103472001850605
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,float16,511,0.010377600044012069
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,fp8,511,0.009412799775600434
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,float16,1023,0.010470400005578995
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,fp8,7,0.008404800295829773
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,fp8,1023,0.010369600355625152
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,float16,2047,0.010476800054311753
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,fp8,2047,0.010475199669599533
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,float16,4095,0.012556800246238708
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,fp8,4095,0.012412799894809723
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,float16,1,0.008486399799585343
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,fp8,1,0.008408000320196151
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,float16,3,0.008476799726486206
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,float16,31,0.010470400005578995
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,fp8,3,0.008444800227880477
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,float16,7,0.00843999981880188
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,fp8,7,0.008324799686670303
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,float16,15,0.008708799630403519
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,float16,255,0.012414400279521943
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,fp8,15,0.008436799794435502
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,float16,63,0.008609600365161896
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,fp8,63,0.00835840031504631
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,float16,127,0.008444800227880477
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,fp8,127,0.008422400057315826
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,float16,255,0.008752000331878663
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,fp8,255,0.008423999696969987
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,fp8,511,0.008491200208663941
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,float16,1023,0.008416000008583068
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,fp8,1023,0.008432000130414962
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,float16,2047,0.010441599786281586
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,fp8,2047,0.010385599732398988
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,float16,4095,0.012492799758911132
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,fp8,4095,0.010471999645233154
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,float16,1,0.014500799775123595
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,float16,3,0.010358399897813796
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,fp8,1,0.01451359987258911
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,float16,3,0.014585599303245544
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,fp8,3,0.014574399590492249
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,float16,7,0.014584000408649444
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,fp8,7,0.014571200311183929
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,float16,15,0.014606399834156037
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,float16,31,0.014567999541759491
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,fp8,31,0.014588800072669984
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,float16,63,0.014548799395561219
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,float16,127,0.014620800316333771
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,fp8,127,0.014547200500965118
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,float16,255,0.014664000272750855
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,fp8,255,0.01462399959564209
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,float16,511,0.013007999956607818
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,fp8,511,0.012532800436019897
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,float16,1023,0.012476799637079239
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,fp8,511,0.012478400021791458
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,float16,2047,0.014569599926471711
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,fp8,2047,0.01329279989004135
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,float16,4095,0.017231999337673186
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,fp8,4095,0.01661760061979294
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,float16,1,0.010691200196743012
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,fp8,1,0.010555200278759003
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,float16,3,0.010683199763298035
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,fp8,3,0.010422399640083313
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,float16,7,0.010476800054311753
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,float16,31,0.00843520015478134
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,fp8,7,0.01053439974784851
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,float16,15,0.011417599767446518
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,fp8,15,0.010435199737548828
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,fp8,31,0.010523200035095215
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,float16,31,0.010694400221109391
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,float16,63,0.010436800122261048
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,float16,511,0.00838399976491928
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,float16,127,0.010470400005578995
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,fp8,127,0.010471999645233154
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,float16,255,0.01043199971318245
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,fp8,255,0.010489600151777268
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,float16,511,0.010467199981212616
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,fp8,511,0.0104592002928257
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,float16,1023,0.01244800016283989
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,fp8,1023,0.011563199758529662
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,float16,2047,0.014571200311183929
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,fp8,2047,0.014555199444293976
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,float16,4095,0.028910401463508605
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,fp8,4095,0.018607999384403228
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,float16,1,0.008419200032949447
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,fp8,1,0.008452799916267396
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,fp8,15,0.014593599736690522
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,float16,3,0.008315200358629227
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,fp8,3,0.008423999696969987
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,float16,7,0.008396799862384795
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,fp8,7,0.008364800363779068
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,float16,15,0.008423999696969987
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,fp8,15,0.00846719965338707
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,float16,31,0.008393599838018417
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,float16,63,0.008396799862384795
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,fp8,31,0.008462399989366532
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,fp8,63,0.00835679993033409
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,fp8,127,0.008380799740552902
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,float16,255,0.00841279998421669
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,fp8,255,0.008430399745702744
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,float16,255,0.010337600111961364
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,fp8,1023,0.012427199631929398
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,fp8,511,0.008515200018882752
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,float16,1023,0.012503999471664428
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,fp8,1023,0.010716799646615982
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,float16,2047,0.027190399169921876
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,fp8,2047,0.014478400349617004
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,float16,4095,0.04521760046482086
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,fp8,4095,0.028830400109291075
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,float16,1,0.010121600329875946
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,fp8,1,0.008419200032949447
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,float16,3,0.010183999687433243
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,fp8,3,0.008462399989366532
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,float16,7,0.010451199859380722
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,fp8,7,0.008529599756002426
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,float16,15,0.010443200170993806
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,fp8,15,0.00846560001373291
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,float16,31,0.010407999902963639
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,fp8,31,0.00841120034456253
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,fp8,31,0.008475200086832047
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,float16,63,0.010427200049161912
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,fp8,63,0.008392000198364257
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,float16,127,0.010375999659299851
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,float16,255,0.010385599732398988
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,fp8,255,0.008694399893283845
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,fp8,63,0.010713600367307664
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,fp8,511,0.010451199859380722
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,float16,1023,0.027668800950050355
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,fp8,1023,0.013948799669742584
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,float16,2047,0.04556640088558197
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,fp8,2047,0.028911998867988585
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,float16,4095,0.08253600001335144
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,fp8,4095,0.04779999852180481
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,float16,1,0.014563199877738953
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,float16,3,0.014470399916172027
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,fp8,3,0.014484800398349762
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,float16,7,0.014577600359916686
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,fp8,7,0.014567999541759491
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,fp8,63,0.014585599303245544
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,float16,15,0.014574399590492249
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,float16,31,0.014593599736690522
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,fp8,31,0.014552000164985656
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,float16,63,0.014537599682807923
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,float16,127,0.008446399867534638
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,float16,127,0.014572800695896148
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,fp8,127,0.014484800398349762
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,float16,255,0.014577600359916686
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,fp8,255,0.014478400349617004
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,float16,511,0.010406400263309478
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,fp8,511,0.012433599680662155
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,float16,1023,0.012515200674533844
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,fp8,1023,0.010496000200510025
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,float16,2047,0.012449599802494049
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,fp8,2047,0.0105103999376297
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,float16,4095,0.012486399710178375
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,fp8,4095,0.012432000041007996
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,float16,1,0.011614400148391723
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,fp8,1,0.010471999645233154
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,float16,3,0.01186240017414093
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,fp8,3,0.010419200360774993
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,fp8,7,0.010491199791431427
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,float16,15,0.0105103999376297
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,fp8,15,0.010414399951696397
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,float16,31,0.010489600151777268
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,float16,7,0.011572799831628799
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,fp8,31,0.010499200224876404
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,float16,63,0.010420800000429154
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,fp8,63,0.010444799810647965
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,float16,127,0.010470400005578995
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,fp8,127,0.008406399935483932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,fp8,127,0.010425599664449692
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,fp8,255,0.010440000146627427
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,float16,511,0.012507200241088867
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,fp8,511,0.01048320010304451
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,float16,1023,0.010468800365924836
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,fp8,1023,0.01048479974269867
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,float16,2047,0.012464000284671784
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,fp8,2047,0.010422399640083313
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,float16,4095,0.012542399764060973
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,fp8,4095,0.012484800070524216
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,fp8,1,0.009523200243711472
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,float16,3,0.010447999835014344
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,fp8,1,0.014545600116252898
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,fp8,3,0.010422399640083313
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,float16,7,0.01048320010304451
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,fp8,7,0.01037440001964569
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,float16,15,0.010375999659299851
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,fp8,15,0.009635200351476669
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,float16,31,0.010377600044012069
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,fp8,31,0.009427200257778167
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,float16,63,0.010311999917030334
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,fp8,63,0.014499199390411378
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,fp8,63,0.009431999921798707
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,float16,127,0.010412800312042236
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,fp8,127,0.009136000275611877
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,float16,255,0.010471999645233154
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,fp8,255,0.010310400277376175
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,float16,511,0.01255200058221817
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,fp8,511,0.010353600233793258
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,float16,1023,0.010463999956846238
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,fp8,1023,0.010425599664449692
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,float16,2047,0.011454399675130844
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,float16,4095,0.012520000338554382
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,fp8,4095,0.012425599992275238
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,float16,1,0.008428800106048583
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,fp8,1,0.00840959995985031
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,float16,3,0.008416000008583068
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,fp8,3,0.008355200290679932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,float16,7,0.008376000076532364
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,fp8,7,0.00841120034456253
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,float16,15,0.008401600271463394
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,fp8,15,0.008316799998283386
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,float16,31,0.008382400125265121
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,float16,255,0.010463999956846238
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,fp8,31,0.008377599716186523
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,float16,63,0.0083856001496315
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,fp8,63,0.008318399637937545
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,float16,127,0.008404800295829773
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,fp8,127,0.008315200358629227
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,float16,255,0.008312000334262848
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,fp8,255,0.008390399813652038
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,float16,511,0.008382400125265121
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,fp8,511,0.008329600095748901
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,float16,1023,0.008404800295829773
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,fp8,1023,0.008463999629020691
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,float16,2047,0.010452800244092942
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,fp8,2047,0.010388799756765366
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,float16,4095,0.012432000041007996
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,fp8,4095,0.010441599786281586
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,float16,1,0.010372799634933472
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,fp8,1,0.008659200370311737
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,float16,3,0.010447999835014344
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,fp8,3,0.008401600271463394
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,float16,7,0.010364799946546554
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,fp8,7,0.008414400368928909
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,float16,15,0.010366400331258773
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,fp8,15,0.008396799862384795
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,float16,31,0.010428799688816071
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,fp8,31,0.008718399703502655
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,float16,63,0.010351999849081039
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,fp8,63,0.008377599716186523
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,float16,127,0.010441599786281586
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,fp8,127,0.008408000320196151
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,fp8,255,0.00840959995985031
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,fp8,15,0.014582400023937226
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,float16,511,0.012014400213956833
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,fp8,511,0.010443200170993806
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,float16,1023,0.014500799775123595
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,fp8,1023,0.012561599910259246
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,float16,2047,0.02892639935016632
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,float16,1,0.012515200674533844
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,fp8,1,0.012511999905109405
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,fp8,2047,0.017796799540519714
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,fp8,3,0.012452799826860428
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,float16,7,0.012521600723266602
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,float16,511,0.010494399815797806
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,float16,15,0.012444800138473511
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,fp8,15,0.012503999471664428
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,float16,31,0.012520000338554382
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,fp8,31,0.012507200241088867
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,float16,63,0.012544000148773193
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,fp8,63,0.012491200119256973
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,float16,127,0.012537600100040435
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,fp8,127,0.012470400333404541
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,float16,255,0.012545600533485413
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,fp8,255,0.012521600723266602
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,float16,511,0.016545599699020384
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,fp8,511,0.01454399973154068
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,float16,1023,0.030656000971794127
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,fp8,1023,0.018648000061511995
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,float16,2047,0.049272000789642334
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,float16,1,0.014526399970054626
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,fp8,1,0.0125231996178627
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,float16,3,0.014585599303245544
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,fp8,3,0.012511999905109405
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,float16,511,0.010476800054311753
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,float16,7,0.01448799967765808
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,fp8,7,0.0124719999730587
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,float16,15,0.01451520025730133
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,fp8,15,0.012544000148773193
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,float16,31,0.0144896000623703
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,fp8,31,0.012539200484752655
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,float16,63,0.014571200311183929
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,float16,1,0.01040000021457672
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,float16,127,0.014496000111103058
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,fp8,127,0.012464000284671784
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,float16,255,0.014574399590492249
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,fp8,255,0.012439999729394913
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,float16,511,0.028809601068496705
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,fp8,511,0.01660960018634796
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,float16,255,0.010363200306892395
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,float16,1023,0.0484688013792038
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,fp8,1023,0.03094080090522766
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,float16,2047,0.08539680242538453
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,float16,1,0.01714400053024292
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,float16,3,0.012510399520397186
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,fp8,1,0.014582400023937226
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,float16,3,0.017183999717235564
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,fp8,7,0.012459199875593185
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,fp8,7,0.014539200067520141
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,float16,7,0.017067199945449828
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,float16,15,0.01656000018119812
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,fp8,2047,0.010460799932479859
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,float16,31,0.016667200624942778
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,fp8,31,0.014767999947071075
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,float16,63,0.017350399494171144
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,fp8,63,0.014664000272750855
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,float16,127,0.017932799458503724
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,fp8,127,0.014902399480342865
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,float16,255,0.028856000304222106
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,fp8,255,0.015379199385643005
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,float16,511,0.047279998660087585
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,fp8,511,0.02951520085334778
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,fp8,2047,0.03311519920825958
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,float16,1023,0.08376320004463196
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,fp8,1023,0.04948799908161163
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,float16,1,0.012478400021791458
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,float16,2047,0.15697120428085326
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,fp8,2047,0.08901119828224183
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,fp8,1,0.010868799686431885
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,float16,3,0.012511999905109405
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,float16,7,0.01250080019235611
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,fp8,7,0.010531199723482132
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,fp8,15,0.010503999888896942
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,float16,31,0.012481600046157837
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,fp8,63,0.012537600100040435
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,fp8,31,0.010751999914646149
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,float16,63,0.012588800489902496
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,fp8,63,0.012470400333404541
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,float16,127,0.012583999335765839
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,fp8,127,0.012457600235939026
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,float16,255,0.012540799379348756
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,fp8,255,0.011644800007343293
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,float16,511,0.016627199947834015
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,float16,1,0.0186271995306015
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,fp8,511,0.01451680064201355
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,fp8,1,0.017129600048065186
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,float16,3,0.018838399648666383
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,fp8,3,0.017390400171279907
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,float16,7,0.018883199989795686
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,fp8,7,0.01709119975566864
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,float16,15,0.018673600256443025
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,fp8,15,0.017761600017547608
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,float16,31,0.018681600689888
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,fp8,31,0.01659200042486191
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,float16,63,0.018620799481868743
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,fp8,63,0.01659200042486191
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,fp8,127,0.01669600009918213
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,float16,127,0.018700799345970152
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,float16,255,0.018649600446224213
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,fp8,3,0.014577600359916686
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,fp8,255,0.017179200053215028
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,float16,511,0.032625600695610046
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,float16,1,0.0226623997092247
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,fp8,511,0.022668799757957457
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,fp8,1,0.018564799427986146
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,float16,3,0.022694399952888487
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,float16,7,0.022771200537681578
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,fp8,3,0.01796479970216751
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,fp8,15,0.014900800585746766
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,fp8,7,0.018305599689483643
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,float16,15,0.02276480048894882
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,float16,31,0.02271520048379898
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,float16,63,0.022708800435066224
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,fp8,31,0.018592000007629395
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,fp8,63,0.018566399812698364
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,float16,127,0.022728000581264497
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,fp8,127,0.018564799427986146
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,float16,255,0.0289792001247406
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,fp8,255,0.018571199476718904
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,float16,511,0.04830400049686432
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,float16,1,0.026927998661994933
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,float16,3,0.0268640011548996
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,fp8,3,0.022745600342750548
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,float16,7,0.02693600058555603
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,fp8,7,0.022915199398994446
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,float16,15,0.027799999713897704
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,fp8,15,0.024031999707221984
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,float16,31,0.02767519950866699
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,fp8,31,0.022779199481010436
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,fp8,3,0.01069599986076355
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,fp8,63,0.022785599529743194
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,float16,15,0.01252480000257492
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,float16,127,0.030928000807762146
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,fp8,127,0.024350400269031524
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,float16,255,0.047286400198936464
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,fp8,255,0.029713600873947144
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,float16,511,0.08467519879341126
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,fp8,511,0.04907679855823517
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,float16,1,0.018486399948596955
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,fp8,1,0.016513599455356597
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,float16,3,0.01839040070772171
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,fp8,3,0.016550399363040924
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,fp8,7,0.016516800224781036
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,float16,15,0.018595199286937713
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,fp8,15,0.016540800034999848
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,float16,31,0.018572799861431122
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,fp8,2047,0.050164800882339475
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,fp8,31,0.016548800468444824
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,float16,63,0.01852319985628128
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,fp8,63,0.016527999937534333
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,float16,127,0.018569600582122803
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,fp8,127,0.016551999747753142
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,float16,255,0.018535999953746794
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,fp8,255,0.016516800224781036
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,float16,1,0.030851200222969055
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,fp8,1,0.02889919877052307
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,float16,3,0.030947199463844298
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,fp8,3,0.02874400019645691
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,fp8,7,0.02860960066318512
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,fp8,15,0.018607999384403228
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,float16,15,0.030924800038337707
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,float16,31,0.030888000130653383
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,fp8,511,0.02927039861679077
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,fp8,31,0.028595200181007384
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,fp8,1,0.024247999489307403
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,float16,63,0.030865600705146788
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,fp8,63,0.028539198637008666
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,fp8,127,0.02821600139141083
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,float16,255,0.03483040034770966
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,fp8,255,0.029164800047874452
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,float16,1,0.037118399143218996
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,fp8,1,0.02815839946269989
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,float16,3,0.037108799815177916
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,fp8,3,0.028496000170707702
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,float16,7,0.037195199728012086
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,fp8,7,0.028303998708724975
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,float16,63,0.028560000658035278
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,float16,31,0.03707840144634247
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,fp8,15,0.028678399324417115
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,float16,15,0.037422400712966916
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,float16,63,0.03790079951286316
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,fp8,63,0.028630399703979494
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,float16,127,0.03922719955444336
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,fp8,127,0.028825598955154418
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,fp8,255,0.03145120143890381
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,float16,255,0.04936639964580536
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,float16,1,0.04776639938354492
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,fp8,1,0.040780800580978396
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,fp8,3,0.041171199083328246
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,float16,7,0.04749439954757691
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,float16,15,0.04742560088634491
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,float16,7,0.0186256006360054
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,fp8,15,0.04112319946289063
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,float16,31,0.04932000041007996
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,fp8,31,0.04111360013484955
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,float16,63,0.04949280023574829
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,fp8,63,0.04115999937057495
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,float16,127,0.05303840041160583
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,fp8,127,0.04303520023822784
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,float16,255,0.08628320097923278
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,fp8,255,0.05204799771308899
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,float16,1,0.014548799395561219
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,fp8,1,0.014611199498176575
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,float16,3,0.014572800695896148
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,fp8,3,0.014486399292945863
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,float16,7,0.014547200500965118
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,fp8,7,0.014571200311183929
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,float16,15,0.014550399780273438
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,fp8,15,0.014545600116252898
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,fp8,31,0.014511999487876893
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,float16,31,0.014569599926471711
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,fp8,15,0.02914080023765564
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,float16,63,0.014476799964904785
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,fp8,63,0.014550399780273438
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,float16,127,0.014564800262451171
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,fp8,127,0.014483200013637542
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,float16,255,0.01451359987258911
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,float16,511,0.012515200674533844
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,fp8,511,0.012438400089740754
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,float16,1023,0.012465599924325943
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,float16,127,0.031062400341033934
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,float16,2047,0.01250240057706833
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,fp8,2047,0.010523200035095215
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,float16,4095,0.013763199746608733
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,fp8,4095,0.012520000338554382
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,fp8,1,0.010494399815797806
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,float16,3,0.010505600273609162
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,fp8,3,0.010502400249242783
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,float16,7,0.010911999642848969
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,fp8,31,0.02887679934501648
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,float16,15,0.010476800054311753
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,fp8,15,0.010491199791431427
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,float16,31,0.010553599894046783
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,fp8,7,0.01051200032234192
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,float16,3,0.04786399900913239
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,fp8,31,0.010487999767065048
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,float16,63,0.012281599640846252
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,fp8,63,0.010527999699115753
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,float16,127,0.012428800016641617
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,fp8,127,0.010526400059461594
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,fp8,7,0.041150400042533876
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,float16,255,0.010428799688816071
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,fp8,255,0.01043360009789467
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,float16,511,0.010632000118494033
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,fp8,511,0.010462400317192078
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,float16,1023,0.010460799932479859
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,fp8,1023,0.010446400195360184
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,float16,2047,0.012412799894809723
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,fp8,2047,0.01112319976091385
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,float16,4095,0.01459999978542328
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,fp8,4095,0.012516799569129943
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,float16,1,0.008376000076532364
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,fp8,1,0.008414400368928909
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,float16,3,0.009667199850082398
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,fp8,3,0.008444800227880477
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,float16,7,0.008918400108814239
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,fp8,7,0.008428800106048583
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,float16,15,0.008460800349712371
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,float16,31,0.010372799634933472
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,float16,63,0.010358399897813796
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,fp8,63,0.008560000360012055
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,float16,127,0.010335999727249145
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,fp8,127,0.008550400286912918
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,float16,255,0.009731200337409974
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,fp8,255,0.008430399745702744
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,float16,511,0.008449599891901017
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,fp8,511,0.00835999995470047
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,float16,1023,0.010361599922180175
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,fp8,255,0.014508800208568573
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,fp8,1023,0.009140799939632415
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,float16,2047,0.010523200035095215
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,fp8,2047,0.010387200117111205
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,fp8,1023,0.012403199821710587
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,fp8,4095,0.012398400157690049
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,float16,1,0.008399999886751174
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,fp8,1,0.008414400368928909
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,float16,1,0.010470400005578995
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,fp8,3,0.008321599662303924
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,float16,7,0.008343999832868576
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,fp8,7,0.008340799808502197
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,float16,15,0.008451200276613235
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,fp8,15,0.008398400247097015
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,float16,31,0.008457600325345992
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,fp8,31,0.008327999711036682
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,float16,63,0.008436799794435502
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,fp8,63,0.008337599784135818
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,float16,127,0.008454400300979614
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,float16,255,0.008408000320196151
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,fp8,127,0.008340799808502197
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,fp8,255,0.008432000130414962
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,float16,511,0.008395200222730636
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,fp8,511,0.008364800363779068
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,float16,1023,0.010388799756765366
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,float16,2047,0.012430399656295776
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,fp8,1023,0.008427199721336365
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,fp8,2047,0.010463999956846238
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,float16,4095,0.027353599667549133
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,fp8,4095,0.013310399651527405
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,1,128,1,float16,float16,1,0.026915198564529418
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,1,128,1,float16,fp8,1,0.024726399779319765
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,1,128,1,float16,float16,3,0.026888000965118408
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,1,128,1,float16,fp8,3,0.024081599712371827
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,1,128,1,float16,float16,7,0.026851201057434083
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,1,128,1,float16,fp8,7,0.024742400646209715
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,1,128,1,float16,float16,15,0.026787200570106508
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,1,128,1,float16,fp8,15,0.024822400510311128
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,1,128,1,float16,float16,31,0.026868799328804018
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,1,128,1,float16,fp8,31,0.024307200312614442
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,1,128,1,float16,float16,63,0.027011200785636902
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,1,128,1,float16,fp8,63,0.024742400646209715
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,1,128,1,float16,float16,127,0.028788799047470094
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,1,128,1,float16,fp8,127,0.024772800505161285
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,2,128,1,float16,float16,1,0.05362399816513062
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,fp8,15,0.008486399799585343
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,2,128,1,float16,fp8,1,0.04941279888153076
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,2,128,1,float16,float16,3,0.053799998760223386
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,2,128,1,float16,fp8,3,0.04921599924564361
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,2,128,1,float16,float16,7,0.05351520180702209
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,2,128,1,float16,fp8,7,0.04938879907131195
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,2,128,1,float16,float16,15,0.053660798072814944
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,2,128,1,float16,fp8,15,0.04921759963035584
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,2,128,1,float16,float16,31,0.054788798093795776
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,2,128,1,float16,fp8,31,0.049300798773765565
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,2,128,1,float16,float16,63,0.05691360235214234
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,float16,7,0.030959999561309813
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,2,128,1,float16,fp8,63,0.04896959960460663
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,2,128,1,float16,float16,127,0.059617602825164796
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,2,128,1,float16,fp8,127,0.05026559829711914
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,4,128,1,float16,float16,1,0.0678063988685608
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,4,128,1,float16,float16,3,0.06786080002784729
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,4,128,1,float16,fp8,3,0.04835360050201416
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,4,128,1,float16,float16,7,0.06785920262336731
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,4,128,1,float16,fp8,7,0.048332801461219786
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,4,128,1,float16,float16,15,0.06784480214118957
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,4,128,1,float16,fp8,15,0.04771679937839508
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,4,128,1,float16,fp8,31,0.0483951985836029
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,4,128,1,float16,float16,31,0.06876479983329772
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,4,128,1,float16,float16,63,0.06818079948425293
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,4,128,1,float16,fp8,63,0.04899199903011322
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,4,128,1,float16,float16,127,0.06949120163917541
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,4,128,1,float16,fp8,127,0.04951840043067932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,float16,4095,0.013251200318336487
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,8,128,1,float16,fp8,1,0.0739408016204834
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,float16,3,0.008502399921417237
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,8,128,1,float16,float16,1,0.08822240233421326
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,8,128,1,float16,float16,7,0.08839840292930604
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,8,128,1,float16,float16,15,0.08912479877471924
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,8,128,1,float16,fp8,7,0.07400320172309875
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,8,128,1,float16,fp8,15,0.07396960258483887
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,8,128,1,float16,float16,31,0.08919360041618347
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,8,128,1,float16,fp8,31,0.07401279807090759
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,8,128,1,float16,float16,63,0.08860160112380981
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,8,128,1,float16,fp8,63,0.07570559978485107
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,8,128,1,float16,fp8,127,0.07624959945678711
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,1,128,1,float16,float16,1,0.04593600034713745
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,1,128,1,float16,fp8,1,0.04115999937057495
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,1,128,1,float16,fp8,3,0.04110080003738403
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,1,128,1,float16,float16,7,0.045281600952148435
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,1,128,1,float16,fp8,7,0.04100959897041321
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,1,128,1,float16,float16,15,0.04651840031147003
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,1,128,1,float16,fp8,15,0.04110879898071289
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,1,128,1,float16,float16,31,0.0475600004196167
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,1,128,1,float16,fp8,31,0.04108479917049408
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,1,128,1,float16,float16,63,0.048260799050331114
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,fp8,31,0.008377599716186523
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,1,128,1,float16,fp8,63,0.041196799278259276
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,2,128,1,float16,float16,1,0.10062079429626465
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,2,128,1,float16,fp8,1,0.08922079801559449
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,2,128,1,float16,float16,3,0.10084160566329955
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,2,128,1,float16,fp8,3,0.08956480026245117
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,4,128,1,float16,fp8,1,0.047628799080848695
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,2,128,1,float16,float16,7,0.10223040580749512
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,2,128,1,float16,float16,15,0.10315359830856323
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,2,128,1,float16,fp8,15,0.08861600160598755
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,2,128,1,float16,float16,31,0.10462239980697632
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,2,128,1,float16,float16,63,0.10567679405212402
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,2,128,1,float16,fp8,63,0.09264479875564575
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,4,128,1,float16,float16,1,0.12715840339660645
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,4,128,1,float16,fp8,1,0.08844159841537476
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,4,128,1,float16,float16,3,0.12717119455337525
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,8,128,1,float16,float16,3,0.08732159733772278
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,4,128,1,float16,fp8,3,0.08865280151367187
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,4,128,1,float16,float16,7,0.12778400182723998
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,4,128,1,float16,fp8,7,0.08832640051841736
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,4,128,1,float16,fp8,15,0.08889120221138
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,4,128,1,float16,float16,31,0.12796159982681274
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,4,128,1,float16,fp8,31,0.08950080275535584
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,4,128,1,float16,float16,63,0.12795039415359497
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,4,128,1,float16,fp8,63,0.09026399850845337
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,1,128,1,float16,float16,3,0.045931199193000795
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,8,128,1,float16,float16,1,0.16836960315704347
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,8,128,1,float16,float16,3,0.16851040124893188
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,8,128,1,float16,fp8,3,0.13981280326843262
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,8,128,1,float16,float16,7,0.16883679628372192
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,8,128,1,float16,fp8,7,0.1399440050125122
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,8,128,1,float16,float16,15,0.16832640171051025
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,8,128,1,float16,fp8,15,0.14035840034484864
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,8,128,1,float16,float16,31,0.16872639656066896
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,8,128,1,float16,fp8,31,0.14071040153503417
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,8,128,1,float16,float16,63,0.16876800060272218
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,8,128,1,float16,fp8,63,0.14148800373077391
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,float16,1,0.01456640064716339
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,fp8,1,0.014524799585342408
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,float16,3,0.014572800695896148
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,fp8,3,0.014545600116252898
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,float16,7,0.014547200500965118
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,fp8,7,0.014499199390411378
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,float16,15,0.014499199390411378
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,2,128,1,float16,fp8,7,0.08855999708175659
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,fp8,15,0.014532800018787383
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,float16,31,0.014601600170135499
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,float16,63,0.015004800260066986
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,2,128,1,float16,fp8,31,0.09014559984207153
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,float16,127,0.01451359987258911
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,fp8,127,0.014468799531459808
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,float16,255,0.01454399973154068
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,fp8,255,0.014494399726390838
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,float16,511,0.012598399817943574
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,fp8,511,0.012460800260305405
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,float16,1023,0.012454400211572647
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,fp8,1023,0.012403199821710587
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,float16,2047,0.012454400211572647
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,8,128,1,float16,fp8,3,0.07399359941482545
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,float16,4095,0.014528000354766845
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,float16,1,0.010507199913263321
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,fp8,4095,0.014472000300884247
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,fp8,1,0.010529600083827972
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,fp8,3,0.01053439974784851
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,float16,7,0.01043360009789467
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,fp8,7,0.0104592002928257
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,float16,15,0.010449600219726563
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,64,8,128,1,float16,float16,127,0.09556639790534974
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,fp8,15,0.01040479987859726
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,float16,31,0.010489600151777268
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,fp8,31,0.010395199805498124
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,float16,63,0.010480000078678131
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,fp8,63,0.010478399693965912
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,float16,127,0.010436800122261048
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,fp8,127,0.010441599786281586
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,float16,255,0.010441599786281586
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,fp8,255,0.010462400317192078
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,float16,511,0.010443200170993806
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,fp8,511,0.01043199971318245
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,float16,1023,0.01043199971318245
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,fp8,1023,0.010478399693965912
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,float16,2047,0.012415999919176102
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,fp8,2047,0.010497599840164185
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,float16,4095,0.014545600116252898
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,fp8,4095,0.013123199343681335
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,float16,1,0.009888000041246413
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,fp8,1,0.010382399708032609
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,float16,3,0.01034879982471466
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,fp8,31,0.014601600170135499
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,fp8,3,0.010385599732398988
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,fp8,63,0.014523200690746307
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,float16,7,0.00987360030412674
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,fp8,7,0.010377600044012069
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,float16,15,0.010401599854230881
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,fp8,15,0.008558399975299835
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,fp8,31,0.010422399640083313
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,float16,63,0.010443200170993806
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,fp8,63,0.010375999659299851
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,float16,127,0.010449600219726563
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,fp8,127,0.008745600283145905
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,float16,255,0.010444799810647965
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,fp8,255,0.009121599793434142
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,fp8,2047,0.012486399710178375
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,float16,511,0.010441599786281586
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,fp8,511,0.008448000252246856
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,float16,1023,0.010507199913263321
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,fp8,1023,0.010444799810647965
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,float16,2047,0.012492799758911132
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,fp8,2047,0.011707200109958649
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,float16,4095,0.02882240116596222
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,float16,1,0.00840959995985031
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,4,128,1,float16,float16,15,0.12906240224838256
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,float16,3,0.008355200290679932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,fp8,3,0.008433599770069123
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,float16,7,0.008340799808502197
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,fp8,7,0.008372800052165985
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,float16,15,0.008315200358629227
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,fp8,15,0.008342400193214417
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,float16,31,0.008321599662303924
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,fp8,31,0.008451200276613235
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,float16,63,0.00840959995985031
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,64,8,128,1,float16,fp8,1,0.1394991993904114
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,float16,127,0.008324799686670303
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,float16,255,0.008414400368928909
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,fp8,127,0.008377599716186523
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,fp8,255,0.008371199667453765
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,float16,511,0.01034879982471466
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,float16,1023,0.01223519966006279
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,fp8,1023,0.010345599800348281
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,float16,2047,0.027031999826431275
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,float16,4095,0.045182400941848756
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,fp8,4095,0.027881601452827455
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,float16,1,0.01085280030965805
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,fp8,1,0.010503999888896942
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,float16,3,0.01088000014424324
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,fp8,3,0.010454399883747101
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,float16,7,0.010465600341558457
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,fp8,7,0.010436800122261048
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,float16,15,0.010475199669599533
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,fp8,15,0.010496000200510025
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,float16,31,0.01045600026845932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,fp8,31,0.010475199669599533
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,float16,63,0.010494399815797806
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,fp8,63,0.010492800176143647
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,float16,127,0.01120000034570694
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,float16,31,0.010412800312042236
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,fp8,127,0.010660800337791442
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,float16,255,0.010465600341558457
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,fp8,255,0.01043360009789467
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,float16,511,0.010387200117111205
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,fp8,511,0.010476800054311753
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,float16,1023,0.012484800070524216
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,float16,2047,0.014564800262451171
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,float16,3,0.010416000336408614
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,fp8,2047,0.013407999277114868
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,fp8,4095,0.014523200690746307
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,fp8,1,0.00825600028038025
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,float16,4095,0.029179200530052185
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,fp8,4095,0.01855680048465729
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,float16,8191,0.04745439887046814
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,float16,1,0.008348800241947174
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,fp8,1,0.00840959995985031
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,fp8,8191,0.030880001187324525
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,fp8,3,0.008342400193214417
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,float16,7,0.008422400057315826
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,float16,15,0.008459199965000153
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,fp8,15,0.008399999886751174
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,float16,31,0.008395200222730636
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,fp8,31,0.008428800106048583
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,fp8,63,0.008377599716186523
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,float16,63,0.008425600081682205
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,float16,127,0.0084927998483181
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,fp8,511,0.008387199789285659
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,fp8,127,0.008430399745702744
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,float16,255,0.008399999886751174
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,float16,511,0.010444799810647965
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,fp8,511,0.009932799637317658
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,float16,1023,0.012513600289821625
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,fp8,1023,0.010675200074911118
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,float16,2047,0.02791520059108734
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,fp8,2047,0.014553600549697876
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,float16,4095,0.04530400037765503
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,fp8,4095,0.028841599822044373
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,float16,8191,0.08152959942817688
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,fp8,8191,0.047075200080871585
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,float16,1,0.009736000001430512
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,fp8,1,0.008366400003433227
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,float16,3,0.010328000038862228
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,fp8,3,0.008524800091981888
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,float16,7,0.010411199927330018
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,fp8,7,0.008451200276613235
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,fp8,1023,0.010497599840164185
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,float16,15,0.010438399761915207
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,fp8,15,0.008382400125265121
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,float16,31,0.010446400195360184
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,fp8,31,0.008419200032949447
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,fp8,63,0.008737599849700928
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,float16,127,0.010407999902963639
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,fp8,127,0.00905120000243187
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,float16,3,0.008454400300979614
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,float16,255,0.010391999781131745
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,fp8,255,0.008819200098514557
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,fp8,7,0.008345600217580795
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,fp8,511,0.010478399693965912
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,float16,1023,0.02764959931373596
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,float16,511,0.012432000041007996
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,fp8,1023,0.014535999298095703
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,fp8,63,0.008372800052165985
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,float16,2047,0.046993601322174075
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,fp8,2047,0.02879520058631897
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,float16,4095,0.08220800161361694
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,fp8,4095,0.04772480130195618
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,fp8,255,0.00846560001373291
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,fp8,8191,0.08502079844474793
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,float16,8191,0.15627360343933105
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,fp8,1,0.01141119971871376
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,float16,3,0.012441600114107132
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,float16,1,0.012435200065374375
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,fp8,3,0.010897599905729295
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,float16,15,0.01244639977812767
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,fp8,15,0.010497599840164185
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,fp8,7,0.011406400054693223
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,float16,7,0.012468799948692322
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,fp8,31,0.010496000200510025
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,float16,63,0.012452799826860428
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,fp8,63,0.011400000005960465
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,float16,127,0.012488000094890594
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,float16,255,0.012464000284671784
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,fp8,127,0.010475199669599533
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,float16,511,0.02871519923210144
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,fp8,511,0.01456640064716339
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,float16,1023,0.04711999893188477
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,fp8,1023,0.02881760001182556
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,float16,2047,0.08379679918289185
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,fp8,2047,0.04905279874801636
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,float16,4095,0.15859359502792358
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,fp8,4095,0.08638880252838135
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,float16,1,0.010468800365924836
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,fp8,1,0.010417599976062775
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,float16,8191,0.30956320762634276
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,float16,3,0.010468800365924836
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,fp8,8191,0.1658511996269226
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,float16,7,0.010516799986362457
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,fp8,3,0.010444799810647965
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,float16,15,0.010460799932479859
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,fp8,7,0.010470400005578995
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,fp8,15,0.010497599840164185
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,float16,31,0.010478399693965912
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,fp8,31,0.01053439974784851
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,float16,63,0.010470400005578995
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,fp8,63,0.010476800054311753
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,float16,127,0.010441599786281586
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,fp8,127,0.010449600219726563
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,float16,255,0.010494399815797806
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,fp8,255,0.010428799688816071
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,float16,511,0.010473600029945374
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,fp8,511,0.010398399829864503
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,float16,1023,0.010468800365924836
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,fp8,1023,0.01048479974269867
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,float16,2047,0.01048479974269867
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,float16,4095,0.012486399710178375
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,fp8,2047,0.01045600026845932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,float16,63,0.010427200049161912
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,fp8,8191,0.012489599734544754
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,float16,8191,0.014499199390411378
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,float16,1,0.010467199981212616
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,fp8,1,0.008473599702119828
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,float16,3,0.010384000092744827
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,float16,7,0.010505600273609162
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,float16,15,0.010422399640083313
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,fp8,15,0.008839999884366989
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,float16,31,0.010361599922180175
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,fp8,31,0.009059199690818786
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,float16,63,0.010395199805498124
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,fp8,63,0.009387200325727462
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,float16,127,0.010372799634933472
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,fp8,127,0.010358399897813796
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,fp8,2047,0.012478400021791458
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,float16,255,0.010398399829864503
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,fp8,255,0.010351999849081039
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,float16,511,0.010388799756765366
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,fp8,511,0.010363200306892395
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,float16,1023,0.010382399708032609
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,float16,31,0.01250080019235611
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,float16,2047,0.01043360009789467
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,fp8,2047,0.010468800365924836
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,fp8,4095,0.01048320010304451
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,fp8,255,0.010516799986362457
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,float16,8191,0.014499199390411378
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,float16,1,0.00836160033941269
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,fp8,1,0.008310399949550629
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,float16,3,0.008414400368928909
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,fp8,3,0.008404800295829773
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,fp8,8191,0.012544000148773193
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,float16,7,0.008420799672603608
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,fp8,7,0.00835679993033409
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,float16,15,0.008444800227880477
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,fp8,15,0.008367999643087386
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,fp8,31,0.008472000062465668
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,float16,63,0.008337599784135818
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,fp8,63,0.008377599716186523
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,float16,127,0.008444800227880477
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,fp8,127,0.008364800363779068
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,float16,255,0.008452799916267396
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,fp8,255,0.008423999696969987
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,float16,511,0.008452799916267396
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,fp8,511,0.008478400111198426
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,float16,1023,0.008369600027799606
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,fp8,1023,0.008406399935483932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,float16,2047,0.010416000336408614
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,fp8,2047,0.008500800281763077
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,float16,4095,0.010486400127410889
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,fp8,3,0.010080000013113022
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,fp8,4095,0.010435199737548828
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,float16,8191,0.013900800049304963
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,fp8,8191,0.012503999471664428
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,fp8,1,0.00836160033941269
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,float16,3,0.008795200288295746
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,float16,7,0.00841120034456253
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,fp8,3,0.008376000076532364
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,fp8,7,0.00841279998421669
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,fp8,15,0.008382400125265121
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,float16,31,0.008451200276613235
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,fp8,31,0.008379200100898742
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,float16,63,0.00841120034456253
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,fp8,63,0.008436799794435502
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,fp8,1023,0.010367999970912933
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,float16,127,0.00840959995985031
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,fp8,127,0.008342400193214417
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,float16,4095,0.011327999830245971
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,float16,255,0.008408000320196151
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,float16,511,0.00846560001373291
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,fp8,511,0.008387199789285659
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,float16,1023,0.009382399916648864
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,fp8,1023,0.008486399799585343
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,float16,2047,0.01035040020942688
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,fp8,2047,0.01043040007352829
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,float16,4095,0.011534400284290314
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,fp8,4095,0.010486400127410889
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,float16,8191,0.01305440068244934
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,float16,1,0.011691199988126755
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,fp8,1,0.010948800295591355
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,float16,3,0.011497599631547928
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,float16,31,0.008401600271463394
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,fp8,3,0.010543999820947647
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,float16,7,0.01242239996790886
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,float16,15,0.012129600346088409
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,fp8,15,0.01077279970049858
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,float16,31,0.01170400008559227
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,fp8,31,0.010449600219726563
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,float16,63,0.01043199971318245
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,fp8,4095,0.011112000048160552
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,float16,127,0.01048159971833229
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,fp8,127,0.010443200170993806
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,float16,255,0.01040000021457672
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,fp8,7,0.008390399813652038
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,fp8,255,0.010470400005578995
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,float16,511,0.010409600287675857
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,float16,1,0.008497600257396699
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,fp8,511,0.010384000092744827
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,float16,1023,0.010452800244092942
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,float16,2047,0.010679999738931656
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,fp8,2047,0.01045759990811348
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,float16,4095,0.012425599992275238
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,float16,8191,0.014486399292945863
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,float16,15,0.008448000252246856
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,fp8,8191,0.014313599467277527
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,float16,1,0.0104032002389431
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,fp8,1,0.009257599711418152
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,float16,3,0.010407999902963639
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,fp8,3,0.008558399975299835
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,float16,7,0.010406400263309478
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,fp8,7,0.008455999940633774
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,float16,15,0.010387200117111205
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,fp8,15,0.010369600355625152
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,float16,31,0.010406400263309478
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,fp8,255,0.00833280012011528
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,fp8,31,0.010284800082445145
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,float16,63,0.010452800244092942
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,fp8,63,0.008462399989366532
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,float16,127,0.010363200306892395
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,fp8,127,0.009387200325727462
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,float16,255,0.010454399883747101
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,fp8,255,0.00936800017952919
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,float16,511,0.010470400005578995
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,fp8,511,0.009609600156545639
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,float16,1023,0.010441599786281586
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,float16,2047,0.010449600219726563
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,fp8,1023,0.01037440001964569
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,fp8,8191,0.012532800436019897
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,fp8,2047,0.010471999645233154
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,float16,4095,0.012467200309038163
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,fp8,4095,0.010478399693965912
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,float16,8191,0.024648000299930573
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,fp8,8191,0.013976000249385834
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,float16,1,0.008436799794435502
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,fp8,1,0.00838399976491928
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,float16,3,0.008455999940633774
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,fp8,3,0.008339200168848038
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,fp8,7,0.008416000008583068
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,fp8,7,0.010496000200510025
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,float16,15,0.008404800295829773
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,fp8,15,0.00836160033941269
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,float16,31,0.00843520015478134
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,fp8,31,0.008392000198364257
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,float16,63,0.008369600027799606
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,fp8,63,0.008687999844551087
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,float16,127,0.008833599835634231
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,fp8,127,0.008448000252246856
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,float16,255,0.008371199667453765
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,fp8,255,0.008417599648237229
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,fp8,511,0.008433599770069123
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,float16,1023,0.00912960022687912
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,fp8,1023,0.008444800227880477
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,fp8,63,0.010465600341558457
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,fp8,2047,0.010388799756765366
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,float16,4095,0.010516799986362457
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,fp8,4095,0.010446400195360184
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,fp8,8191,0.012441600114107132
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,float16,1,0.008436799794435502
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,fp8,1,0.00843520015478134
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,fp8,1023,0.010414399951696397
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,float16,3,0.008430399745702744
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,fp8,3,0.008441600203514098
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,fp8,4095,0.012368000298738479
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,fp8,7,0.008406399935483932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,float16,15,0.00843999981880188
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,fp8,15,0.008377599716186523
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,float16,31,0.008393599838018417
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,fp8,31,0.008363199979066848
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,float16,63,0.008401600271463394
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,fp8,63,0.008350399881601333
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,float16,127,0.00843520015478134
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,fp8,127,0.008399999886751174
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,float16,255,0.008399999886751174
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,fp8,255,0.008427199721336365
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,fp8,511,0.008416000008583068
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,float16,1023,0.009436800330877303
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,fp8,1023,0.008395200222730636
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,float16,2047,0.010438399761915207
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,fp8,2047,0.010467199981212616
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,float16,4095,0.012464000284671784
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,fp8,4095,0.010831999778747558
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,float16,8191,0.014524799585342408
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,fp8,8191,0.012488000094890594
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,float16,1,0.010476800054311753
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,fp8,1,0.010471999645233154
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,float16,7,0.00844319984316826
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,fp8,3,0.01037919968366623
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,float16,7,0.010502400249242783
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,fp8,7,0.010355199873447418
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,float16,15,0.010449600219726563
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,fp8,15,0.01043040007352829
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,float16,31,0.010436800122261048
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,fp8,31,0.0104032002389431
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,float16,511,0.00835840031504631
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,fp8,63,0.010502400249242783
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,float16,127,0.01048479974269867
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,fp8,127,0.010478399693965912
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,float16,2047,0.010384000092744827
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,float16,255,0.010491199791431427
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,fp8,255,0.010471999645233154
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,float16,511,0.010391999781131745
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,fp8,511,0.01040479987859726
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,float16,8191,0.012726399302482604
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,float16,2047,0.012443199753761292
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,fp8,1023,0.010473600029945374
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,fp8,2047,0.01109120026230812
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,float16,4095,0.01454399973154068
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,fp8,4095,0.014467200636863709
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,float16,8191,0.028896000981330872
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,fp8,8191,0.018651199340820313
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,float16,1,0.010414399951696397
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,float16,3,0.010388799756765366
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,float16,7,0.008459199965000153
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,fp8,3,0.008724799752235413
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,float16,7,0.010395199805498124
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,fp8,7,0.009070400148630142
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,float16,15,0.0104032002389431
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,fp8,15,0.008926399797201157
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,float16,31,0.010385599732398988
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,fp8,31,0.009254399687051773
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,float16,63,0.010387200117111205
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,fp8,63,0.009252800047397614
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,float16,127,0.010471999645233154
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,float16,511,0.008851200342178345
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,fp8,127,0.009428799897432328
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,float16,255,0.010369600355625152
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,fp8,255,0.01037919968366623
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,float16,511,0.009656000137329101
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,float16,1023,0.010467199981212616
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,fp8,511,0.008449599891901017
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,fp8,1023,0.010353600233793258
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,float16,2047,0.012539200484752655
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,float16,4095,0.02881760001182556
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,fp8,4095,0.014563199877738953
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,float16,8191,0.04713599979877472
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,fp8,8191,0.028865599632263185
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,float16,1,0.008348800241947174
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,fp8,1,0.0083856001496315
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,float16,3,0.008312000334262848
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,fp8,3,0.008430399745702744
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,float16,7,0.008428800106048583
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,fp8,7,0.008369600027799606
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,float16,15,0.008329600095748901
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,fp8,15,0.008392000198364257
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,float16,31,0.00833119973540306
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,fp8,31,0.008329600095748901
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,float16,63,0.008339200168848038
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,fp8,63,0.008396799862384795
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,float16,127,0.00833439975976944
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,fp8,127,0.008432000130414962
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,float16,3,0.0104592002928257
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,float16,255,0.008395200222730636
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,fp8,255,0.008392000198364257
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,float16,511,0.010438399761915207
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,float16,1023,0.010532800108194351
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,fp8,1023,0.010411199927330018
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,float16,2047,0.027028799057006836
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,fp8,2047,0.012545600533485413
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,float16,63,0.01048479974269867
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,fp8,4095,0.02703840136528015
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,float16,8191,0.0816976010799408
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,fp8,8191,0.0463808000087738
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,float16,1023,0.010412800312042236
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,float16,3,0.010393600165843963
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,fp8,1,0.008403199911117553
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,fp8,3,0.008448000252246856
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,fp8,1,0.008846399933099746
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,fp8,7,0.008441600203514098
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,float16,15,0.010377600044012069
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,fp8,15,0.008428800106048583
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,float16,31,0.010371199995279311
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,fp8,31,0.008430399745702744
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,float16,63,0.010367999970912933
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,fp8,63,0.008448000252246856
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,float16,127,0.0103472001850605
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,fp8,127,0.008451200276613235
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,float16,255,0.010339199751615524
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,fp8,255,0.008423999696969987
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,float16,511,0.012427199631929398
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,fp8,511,0.010407999902963639
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,float16,1023,0.02743520140647888
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,fp8,1023,0.013711999356746673
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,float16,2047,0.04645760059356689
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,fp8,2047,0.02879199981689453
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,fp8,2047,0.012452799826860428
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,float16,4095,0.08328480124473572
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,fp8,4095,0.047767999768257144
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,float16,1,0.011393599957227708
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,float16,8191,0.157096004486084
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,fp8,8191,0.0877664029598236
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,fp8,1,0.010977599769830704
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,float16,3,0.011766400188207626
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,float16,7,0.011803200095891952
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,fp8,3,0.011062400043010711
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,float16,15,0.01127839982509613
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,float16,31,0.010476800054311753
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,fp8,31,0.01045759990811348
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,fp8,7,0.010529600083827972
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,float16,63,0.01048320010304451
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,float16,127,0.010476800054311753
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,fp8,127,0.010503999888896942
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,fp8,511,0.008342400193214417
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,float16,255,0.010499200224876404
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,fp8,255,0.010486400127410889
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,fp8,511,0.010422399640083313
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,float16,511,0.010463999956846238
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,fp8,1023,0.010371199995279311
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,float16,2047,0.01045600026845932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,float16,4095,0.04554559886455536
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,float16,4095,0.012465599924325943
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,fp8,2047,0.010449600219726563
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,fp8,4095,0.01250240057706833
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,fp8,8191,0.014521600306034088
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,float16,1,0.010387200117111205
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,fp8,1,0.010409600287675857
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,float16,3,0.010361599922180175
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,fp8,3,0.010353600233793258
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,float16,1,0.01037919968366623
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,float16,7,0.010384000092744827
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,float16,7,0.01045600026845932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,float16,15,0.01043040007352829
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,float16,31,0.010361599922180175
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,fp8,31,0.009363199770450591
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,float16,63,0.01034879982471466
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,fp8,63,0.010361599922180175
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,float16,127,0.01037919968366623
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,fp8,127,0.01037919968366623
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,float16,255,0.01037919968366623
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,fp8,255,0.010443200170993806
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,float16,511,0.010467199981212616
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,fp8,511,0.010382399708032609
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,float16,1023,0.010473600029945374
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,fp8,1023,0.010414399951696397
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,float16,2047,0.010441599786281586
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,fp8,2047,0.010468800365924836
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,float16,4095,0.012516799569129943
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,fp8,4095,0.012411200255155564
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,float16,8191,0.014460800588130951
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,fp8,8191,0.01438560038805008
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,float16,1,0.00843840017914772
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,fp8,1,0.008414400368928909
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,float16,3,0.008403199911117553
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,fp8,15,0.010659199953079224
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,fp8,3,0.008303999900817871
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,float16,7,0.008393599838018417
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,fp8,7,0.008393599838018417
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,fp8,63,0.010480000078678131
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,fp8,15,0.008374399691820144
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,float16,31,0.008422400057315826
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,fp8,31,0.008379200100898742
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,float16,63,0.00844319984316826
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,float16,1023,0.010502400249242783
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,fp8,63,0.008372800052165985
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,float16,127,0.00870719999074936
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,float16,255,0.008414400368928909
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,fp8,255,0.008388800173997879
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,float16,511,0.009544000029563904
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,float16,8191,0.014561599493026734
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,fp8,511,0.008844800293445587
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,fp8,1023,0.008457600325345992
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,float16,2047,0.010491199791431427
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,fp8,2047,0.01048479974269867
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,float16,4095,0.012439999729394913
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,fp8,4095,0.010577599704265594
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,float16,8191,0.014504000544548035
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,fp8,8191,0.012470400333404541
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,float16,1,0.008355200290679932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,fp8,1,0.008388800173997879
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,float16,3,0.008449599891901017
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,fp8,3,0.008430399745702744
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,float16,7,0.008404800295829773
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,fp8,7,0.009348800033330917
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,fp8,7,0.008419200032949447
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,float16,15,0.008427199721336365
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,fp8,15,0.009508799761533737
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,float16,31,0.008399999886751174
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,fp8,31,0.00833439975976944
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,float16,63,0.008425600081682205
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,float16,127,0.008347199857234954
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,fp8,127,0.00843520015478134
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,float16,255,0.008399999886751174
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,fp8,255,0.008414400368928909
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,fp8,511,0.008376000076532364
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,float16,511,0.008457600325345992
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,float16,1023,0.009031999856233597
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,fp8,1023,0.008427199721336365
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,fp8,2047,0.008448000252246856
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,float16,2047,0.010462400317192078
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,float16,4095,0.012467200309038163
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,fp8,4095,0.010406400263309478
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,fp8,8191,0.014263999462127686
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,float16,1,0.010412800312042236
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,fp8,1,0.010396800190210342
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,float16,3,0.010414399951696397
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,fp8,3,0.010393600165843963
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,float16,7,0.010390400141477584
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,fp8,7,0.010361599922180175
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,float16,15,0.010412800312042236
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,fp8,15,0.010406400263309478
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,fp8,31,0.010401599854230881
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,float16,31,0.010420800000429154
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,float16,63,0.010384000092744827
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,fp8,63,0.010422399640083313
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,float16,127,0.010358399897813796
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,fp8,127,0.010425599664449692
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,float16,255,0.010396800190210342
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,fp8,255,0.010396800190210342
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,float16,511,0.01080320030450821
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,float16,15,0.008537600189447403
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,fp8,511,0.010475199669599533
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,float16,1023,0.01443839967250824
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,fp8,1023,0.012521600723266602
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,float16,2047,0.028883200883865357
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,fp8,127,0.008374399691820144
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,float16,4095,0.04731999933719635
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,fp8,2047,0.017360000312328337
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,float16,1023,0.010366400331258773
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,float16,1,0.010440000146627427
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,fp8,1,0.01043360009789467
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,float16,3,0.01045759990811348
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,fp8,3,0.010384000092744827
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,float16,7,0.010583999752998351
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,fp8,7,0.010412800312042236
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,fp8,15,0.010425599664449692
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,float16,31,0.01043199971318245
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,fp8,31,0.010409600287675857
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,float16,63,0.010372799634933472
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,fp8,63,0.0103472001850605
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,float16,127,0.0104592002928257
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,fp8,127,0.01072480008006096
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,float16,255,0.010496000200510025
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,float16,511,0.014142400026321411
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,fp8,15,0.008313599973917007
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,fp8,511,0.012417600303888322
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,fp8,63,0.008801600337028504
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,float16,1023,0.028372800350189208
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,fp8,1023,0.014574399590492249
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,float16,2047,0.047275200486183167
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,float16,4095,0.08274080157279969
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,fp8,4095,0.04893440008163452
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,fp8,1,0.011294399946928024
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,float16,3,0.01249919980764389
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,fp8,3,0.010513599961996078
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,float16,7,0.012518399953842163
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,fp8,7,0.010438399761915207
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,float16,8191,0.02710399925708771
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,float16,15,0.012492799758911132
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,fp8,15,0.010444799810647965
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,float16,31,0.012537600100040435
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,fp8,31,0.01043040007352829
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,float16,63,0.01247360035777092
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,float16,127,0.012833599746227265
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,fp8,127,0.010500799864530563
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,float16,255,0.0124719999730587
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,fp8,255,0.011151999980211259
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,float16,511,0.028887999057769776
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,fp8,511,0.01464959979057312
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,float16,1023,0.047409600019454955
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,fp8,1023,0.02959040105342865
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,float16,2047,0.08467519879341126
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,fp8,2047,0.04852479994297028
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,fp8,4095,0.0308896005153656
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,float16,4095,0.15935200452804565
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,float16,1,0.017187200486660004
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,fp8,4095,0.08438079953193664
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,float16,15,0.010446400195360184
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,float16,3,0.01687519997358322
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,fp8,1,0.014603200554847717
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,float16,7,0.01682559996843338
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,fp8,7,0.015028800070285796
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,float16,15,0.01664319932460785
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,fp8,15,0.014870400726795196
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,fp8,31,0.014892800152301789
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,float16,31,0.016646400094032288
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,float16,63,0.01656640022993088
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,fp8,63,0.014827199280261993
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,fp8,127,0.014564800262451171
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,float16,127,0.016867199540138246
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,fp8,255,0.014707200229167938
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,float16,511,0.0466735988855362
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,fp8,511,0.029051199555397034
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,fp8,2047,0.029633599519729614
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,float16,1023,0.0826367974281311
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,fp8,1023,0.048767998814582825
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,float16,1,0.012494400143623352
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,float16,2047,0.1559664011001587
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,float16,1,0.0125231996178627
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,fp8,4095,0.16762080192565917
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,fp8,1,0.012430399656295776
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,float16,3,0.01247519999742508
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,float16,7,0.012518399953842163
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,fp8,7,0.012425599992275238
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,fp8,63,0.011222399771213531
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,float16,15,0.012505599856376648
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,fp8,15,0.012505599856376648
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,float16,31,0.012489599734544754
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,float16,63,0.012556800246238708
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,fp8,63,0.012531200051307678
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,float16,127,0.012518399953842163
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,fp8,127,0.012401600182056428
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,float16,255,0.012491200119256973
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,fp8,255,0.012494400143623352
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,float16,511,0.016571199893951415
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,fp8,511,0.014563199877738953
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,float16,1023,0.030608001351356506
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,fp8,1023,0.018743999302387238
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,float16,1,0.014567999541759491
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,fp8,3,0.014547200500965118
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,fp8,1,0.012488000094890594
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,float16,3,0.014577600359916686
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,fp8,3,0.012483199685811996
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,fp8,255,0.010462400317192078
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,float16,255,0.028016000986099243
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,float16,7,0.014561599493026734
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,fp8,7,0.012531200051307678
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,float16,15,0.014574399590492249
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,fp8,15,0.012489599734544754
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,float16,31,0.014595200121402741
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,fp8,31,0.012529599666595458
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,float16,63,0.014537599682807923
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,fp8,63,0.012555199861526489
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,float16,127,0.014577600359916686
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,float16,255,0.014555199444293976
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,fp8,255,0.012545600533485413
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,fp8,511,0.016599999368190767
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,float16,511,0.028815999627113342
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,fp8,2047,0.08616160154342652
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,fp8,1023,0.030904000997543334
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,float16,1023,0.04737600088119507
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,float16,1,0.01706400066614151
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,fp8,1,0.014595200121402741
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,float16,3,0.01658879965543747
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,fp8,3,0.014727999269962311
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,float16,4095,0.3030895948410034
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,fp8,7,0.014584000408649444
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,float16,15,0.016828800737857818
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,float16,7,0.017470400035381316
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,fp8,3,0.012441600114107132
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,float16,31,0.01656000018119812
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,fp8,31,0.014641599357128143
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,fp8,15,0.014582400023937226
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,float16,127,0.017590400576591492
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,fp8,127,0.014641599357128143
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,float16,255,0.028380799293518066
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,fp8,255,0.014803199470043183
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,float16,511,0.04738239943981171
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,fp8,511,0.029124799370765685
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,fp8,31,0.012436799705028534
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,float16,1023,0.08409119844436645
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,float16,1,0.027671998739242552
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,fp8,1023,0.0489439994096756
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,fp8,1,0.02284960001707077
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,fp8,3,0.022988800704479218
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,float16,7,0.02688319981098175
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,fp8,7,0.023179200291633607
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,float16,15,0.027585598826408386
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,fp8,15,0.022844800353050233
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,float16,31,0.026897600293159483
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,fp8,31,0.023684799671173096
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,float16,63,0.027020800113677978
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,fp8,63,0.02300959974527359
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,float16,127,0.030828800797462464
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,fp8,127,0.022913600504398345
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,float16,255,0.04748159945011139
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,fp8,255,0.03062559962272644
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,float16,511,0.0841584026813507
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,fp8,511,0.047620800137519834
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,float16,1,0.018628799915313722
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,float16,1023,0.15664000511169435
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,fp8,1,0.017052799463272095
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,fp8,1023,0.08673120141029358
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,fp8,3,0.01684480011463165
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,float16,3,0.018641600012779237
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,float16,7,0.019198399782180787
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,float16,15,0.018680000305175783
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,float16,31,0.018671999871730804
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,fp8,31,0.018459199368953703
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,float16,63,0.01865279972553253
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,fp8,63,0.018475200235843658
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,float16,127,0.018617600202560425
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,fp8,127,0.018409599363803864
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,fp8,127,0.01249919980764389
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,float16,255,0.020076799392700195
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,fp8,255,0.01867839992046356
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,float16,511,0.03174239993095398
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,fp8,511,0.022737599909305573
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,float16,1,0.02274720072746277
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,fp8,1,0.018654400110244752
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,float16,3,0.022716799378395082
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,fp8,3,0.018649600446224213
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,float16,7,0.02271360009908676
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,fp8,7,0.01866399943828583
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,float16,15,0.022652800381183624
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,fp8,15,0.018611200153827667
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,float16,31,0.022655999660491942
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,fp8,31,0.018639999628067016
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,float16,63,0.022672000527381896
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,fp8,63,0.018611200153827667
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,fp8,63,0.015249599516391755
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,float16,127,0.022750400006771088
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,fp8,127,0.018595199286937713
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,float16,255,0.02906399965286255
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,fp8,255,0.018636800348758698
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,float16,511,0.047998398542404175
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,fp8,511,0.030214399099349976
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,float16,1,0.028094398975372314
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,fp8,1,0.024784000217914583
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,float16,3,0.026846399903297423
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,fp8,3,0.02476159930229187
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,float16,7,0.026804798841476442
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,fp8,7,0.024772800505161285
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,float16,3,0.026848000288009644
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,fp8,15,0.024695999920368195
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,float16,31,0.027881601452827455
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,fp8,31,0.024721600115299225
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,float16,63,0.028806400299072266
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,fp8,63,0.0247856006026268
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,float16,127,0.030947199463844298
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,fp8,127,0.024689599871635437
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,fp8,255,0.029504001140594482
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,float16,255,0.04743199944496155
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,float16,511,0.08558880090713501
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,float16,1,0.047332799434661864
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,fp8,1,0.04018400013446808
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,float16,3,0.04757919907569885
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,fp8,3,0.040059199929237364
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,fp8,15,0.018545599281787874
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,float16,7,0.04744639992713928
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,float16,15,0.047460800409317015
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,fp8,7,0.03945119976997376
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,fp8,15,0.04034880101680756
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,float16,31,0.047875198721885684
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,float16,63,0.049291199445724486
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,fp8,63,0.0409168004989624
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,float16,127,0.05138400197029114
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,fp8,127,0.04203839898109436
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,float16,255,0.08489279747009278
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,fp8,255,0.05104159712791443
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,fp8,511,0.08687360286712646
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,float16,511,0.1584928035736084
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,float16,1,0.010497599840164185
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,fp8,1,0.010500799864530563
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,float16,3,0.010502400249242783
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,fp8,3,0.010446400195360184
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,fp8,7,0.010454399883747101
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,float16,15,0.0104592002928257
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,fp8,15,0.01043040007352829
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,float16,63,0.018347199261188506
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,float16,31,0.0124208003282547
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,float16,63,0.01117599979043007
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,fp8,63,0.0104592002928257
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,float16,127,0.01241919994354248
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,fp8,127,0.010476800054311753
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,float16,255,0.010454399883747101
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,float16,15,0.026870399713516235
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,float16,511,0.010465600341558457
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,fp8,511,0.010503999888896942
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,float16,1023,0.010471999645233154
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,fp8,1023,0.010380800068378448
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,float16,2047,0.012383999675512314
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,fp8,2047,0.01141439974308014
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,float16,4095,0.01250080019235611
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,fp8,511,0.04886879920959473
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,fp8,4095,0.01244800016283989
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,float16,8191,0.014683200418949128
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,fp8,8191,0.01446560025215149
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,float16,1,0.010384000092744827
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,fp8,1,0.010366400331258773
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,float16,3,0.010384000092744827
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,fp8,3,0.010387200117111205
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,fp8,7,0.01035040020942688
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,float16,7,0.0104032002389431
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,float16,15,0.010366400331258773
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,fp8,15,0.009382399916648864
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,float16,31,0.010366400331258773
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,fp8,31,0.03937439918518067
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,float16,63,0.010371199995279311
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,fp8,31,0.010177599638700486
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,fp8,63,0.009419199824333192
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,float16,127,0.010398399829864503
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,fp8,127,0.009459199756383896
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,float16,255,0.01043199971318245
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,fp8,255,0.009363199770450591
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,float16,511,0.010414399951696397
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,fp8,511,0.010380800068378448
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,float16,1023,0.010395199805498124
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,fp8,1023,0.010385599732398988
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,fp8,2047,0.010446400195360184
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,float16,2047,0.011191999912261963
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,float16,4095,0.012569600343704223
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,float16,8191,0.015291200578212738
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,fp8,4095,0.01244639977812767
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,fp8,8191,0.014545600116252898
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,fp8,1,0.008420799672603608
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,float16,3,0.008436799794435502
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,fp8,3,0.0083856001496315
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,float16,7,0.008339200168848038
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,fp8,7,0.00833280012011528
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,float16,15,0.008367999643087386
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,fp8,15,0.00841120034456253
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,float16,31,0.00835840031504631
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,float16,63,0.008342400193214417
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,float16,7,0.010465600341558457
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,fp8,63,0.008350399881601333
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,fp8,31,0.008376000076532364
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,float16,127,0.008423999696969987
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,float16,255,0.008324799686670303
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,fp8,127,0.008336000144481659
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,fp8,255,0.00838399976491928
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,fp8,511,0.008371199667453765
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,fp8,1023,0.00843520015478134
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,fp8,31,0.011105599999427795
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,fp8,2047,0.01040479987859726
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,float16,4095,0.012444800138473511
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,fp8,4095,0.010411199927330018
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,float16,8191,0.028326401114463808
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,fp8,255,0.01051200032234192
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,fp8,8191,0.01366879940032959
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,float16,1,0.008428800106048583
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,fp8,1,0.008347199857234954
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,float16,3,0.008367999643087386
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,fp8,3,0.00835840031504631
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,float16,7,0.008423999696969987
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,fp8,7,0.008416000008583068
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,float16,15,0.008430399745702744
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,fp8,15,0.008363199979066848
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,float16,31,0.008403199911117553
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,fp8,31,0.008323200047016144
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,float16,63,0.008446399867534638
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,fp8,63,0.008422400057315826
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,float16,127,0.008392000198364257
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,float16,255,0.008428800106048583
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,fp8,255,0.00833119973540306
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,float16,511,0.008417599648237229
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,fp8,511,0.008339200168848038
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,float16,1023,0.010451199859380722
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,fp8,1023,0.008380799740552902
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,float16,2047,0.012451200187206269
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,fp8,2047,0.0104592002928257
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,float16,4095,0.026867198944091796
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,fp8,4095,0.012814399600028992
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,float16,8191,0.04579679965972901
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,fp8,8191,0.028486400842666626
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,float16,1,0.030969598889350893
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,fp8,1,0.028892800211906433
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,float16,3,0.03089280128479004
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,fp8,3,0.028832000494003297
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,float16,7,0.030963200330734252
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,float16,511,0.008459199965000153
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,fp8,7,0.02882719933986664
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,float16,1023,0.009363199770450591
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,float16,15,0.030895999073982237
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,fp8,15,0.0288783997297287
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,float16,31,0.030883198976516722
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,float16,63,0.030888000130653383
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,float16,2047,0.010411199927330018
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,float16,127,0.03141759932041168
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,fp8,63,0.028889599442481994
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,fp8,127,0.02893120050430298
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,fp8,255,0.029020801186561584
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,float16,1,0.03705599904060364
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,fp8,1,0.028815999627113342
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,float16,3,0.03705919981002807
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,fp8,3,0.028806400299072266
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,float16,7,0.03709439933300018
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,float16,15,0.037049600481987
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,fp8,7,0.028814399242401124
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,fp8,15,0.02885279953479767
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,float16,31,0.037067198753356935
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,fp8,31,0.028862398862838746
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,float16,63,0.03741919994354248
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,fp8,63,0.028947201371192933
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,float16,127,0.03883999884128571
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,fp8,127,0.028889599442481994
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,float16,255,0.04936800003051758
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,fp8,255,0.03289600014686585
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,float16,1,0.04754559993743897
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,fp8,1,0.04120480120182037
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,fp8,127,0.008343999832868576
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,float16,3,0.047628799080848695
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,fp8,3,0.04115520119667053
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,float16,7,0.047577598690986635
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,float16,15,0.04781599938869476
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,fp8,7,0.041212800145149234
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,float16,31,0.049358400702476504
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,fp8,7,0.01719679981470108
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,fp8,31,0.041193601489067075
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,float16,63,0.04939360022544861
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,fp8,63,0.041231998801231386
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,float16,127,0.05262560248374939
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,float16,255,0.08632320165634155
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,fp8,255,0.052070397138595584
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,fp8,127,0.04314239919185638
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,float16,1,0.0866703987121582
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,fp8,1,0.07234560251235962
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,float16,3,0.08700159788131714
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,fp8,3,0.07203680276870728
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,float16,7,0.08661280274391174
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,fp8,7,0.0723743975162506
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,float16,15,0.08830400109291077
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,fp8,15,0.07210400104522705
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,float16,31,0.0883072018623352
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,fp8,31,0.07315359711647033
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,float16,63,0.08849279880523682
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,fp8,63,0.07423359751701356
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,float16,127,0.09434559941291809
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,fp8,127,0.07570239901542664
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,1,128,1,float16,float16,1,0.054364800453186035
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,fp8,255,0.09359040260314941
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,float16,255,0.16135200262069702
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,1,128,1,float16,fp8,1,0.04944159984588623
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,1,128,1,float16,float16,3,0.053585600852966306
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,1,128,1,float16,float16,7,0.05405759811401367
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,1,128,1,float16,fp8,3,0.04941279888153076
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,1,128,1,float16,fp8,7,0.04937599897384644
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,1,128,1,float16,float16,15,0.053972798585891726
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,1,128,1,float16,fp8,15,0.0494271993637085
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,1,128,1,float16,float16,31,0.05379840135574341
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,fp8,31,0.028880000114440918
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,1,128,1,float16,fp8,31,0.04942240118980408
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,1,128,1,float16,float16,63,0.05679839849472046
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,1,128,1,float16,float16,127,0.05924479961395264
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,2,128,1,float16,float16,1,0.06776800155639648
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,1,128,1,float16,fp8,127,0.051583999395370485
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,float16,255,0.03504480123519897
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,2,128,1,float16,fp8,1,0.04942879974842072
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,2,128,1,float16,float16,3,0.06780319809913635
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,2,128,1,float16,float16,7,0.06842880249023438
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,2,128,1,float16,fp8,7,0.04938560128211975
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,2,128,1,float16,float16,15,0.06785280108451844
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,2,128,1,float16,fp8,15,0.04947519898414612
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,2,128,1,float16,float16,31,0.06913120150566102
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,2,128,1,float16,fp8,31,0.04943679869174957
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,2,128,1,float16,float16,63,0.06994240283966065
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,2,128,1,float16,fp8,63,0.050153601169586184
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,2,128,1,float16,float16,127,0.06893600225448608
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,2,128,1,float16,fp8,127,0.05152480006217956
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,4,128,1,float16,fp8,1,0.07397599816322327
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,4,128,1,float16,float16,3,0.08766239881515503
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,4,128,1,float16,fp8,3,0.07396799921989441
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,4,128,1,float16,float16,7,0.08829280138015747
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,4,128,1,float16,fp8,7,0.07409120202064515
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,4,128,1,float16,float16,15,0.08898400068283081
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,4,128,1,float16,fp8,15,0.0741919994354248
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,fp8,15,0.041212800145149234
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,4,128,1,float16,fp8,31,0.07512800097465515
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,4,128,1,float16,float16,63,0.08887199759483337
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,4,128,1,float16,fp8,63,0.0759663999080658
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,4,128,1,float16,float16,31,0.08896639943122864
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,4,128,1,float16,float16,127,0.09616479873657227
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,4,128,1,float16,fp8,127,0.0777679979801178
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,8,128,1,float16,float16,1,0.16524480581283568
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,8,128,1,float16,fp8,1,0.13772640228271485
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,8,128,1,float16,float16,3,0.1663040041923523
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,8,128,1,float16,fp8,3,0.13773280382156372
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,float16,1,0.008404800295829773
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,8,128,1,float16,fp8,7,0.1377408027648926
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,8,128,1,float16,float16,7,0.1663920044898987
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,8,128,1,float16,float16,15,0.1666640043258667
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,8,128,1,float16,fp8,15,0.13865439891815184
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,8,128,1,float16,fp8,31,0.13817440271377562
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,8,128,1,float16,float16,31,0.16640000343322753
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,8,128,1,float16,float16,63,0.16763360500335694
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,8,128,1,float16,fp8,63,0.13904000520706178
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,float16,1,0.01149279996752739
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,fp8,1,0.010500799864530563
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,float16,3,0.01117279976606369
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,8,128,1,float16,fp8,127,0.14191999435424804
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,8,128,1,float16,float16,127,0.17884639501571656
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,float16,7,0.010976000130176545
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,fp8,7,0.010521599650382995
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,float16,31,0.010609599947929382
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,float16,15,0.010468800365924836
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,fp8,31,0.010516799986362457
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,float16,63,0.01120320037007332
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,fp8,63,0.01043040007352829
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,float16,127,0.010470400005578995
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,fp8,127,0.010492800176143647
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,float16,255,0.010480000078678131
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,2,128,1,float16,fp8,3,0.04934079945087433
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,fp8,255,0.010465600341558457
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,float16,511,0.010716799646615982
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,float16,1023,0.01043040007352829
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,fp8,511,0.010486400127410889
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,fp8,1023,0.01043040007352829
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,float16,2047,0.012443199753761292
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,fp8,2047,0.010513599961996078
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,float16,4095,0.014510400593280792
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,fp8,4095,0.013483199477195739
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,fp8,8191,0.015412800014019012
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,fp8,1,0.008432000130414962
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,float16,3,0.010315199941396713
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,fp8,3,0.008423999696969987
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,float16,7,0.009625600278377533
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,fp8,7,0.008408000320196151
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,fp8,15,0.008432000130414962
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,float16,31,0.009408000111579894
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,fp8,31,0.008460800349712371
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,float16,63,0.009852799773216247
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,fp8,63,0.009297599643468856
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,float16,127,0.008839999884366989
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,fp8,127,0.008417599648237229
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,float16,255,0.008460800349712371
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,fp8,255,0.008473599702119828
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,float16,511,0.008364800363779068
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,fp8,511,0.00843840017914772
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,float16,1023,0.01037440001964569
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,fp8,3,0.01050880029797554
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,fp8,1023,0.008462399989366532
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,fp8,15,0.011070399731397628
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,float16,2047,0.01045759990811348
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,fp8,2047,0.01048159971833229
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,float16,4095,0.013657599687576294
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,fp8,4095,0.01242400035262108
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,float16,8191,0.02882240116596222
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,fp8,8191,0.014860799908638
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,float16,1,0.008748800307512284
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,fp8,1,0.008425600081682205
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,float16,3,0.008454400300979614
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,fp8,3,0.008403199911117553
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,float16,7,0.008372800052165985
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,fp8,7,0.008387199789285659
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,float16,8191,0.016568000614643096
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,fp8,15,0.008425600081682205
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,float16,1,0.00843999981880188
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,float16,31,0.008417599648237229
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,fp8,31,0.008336000144481659
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,float16,63,0.008377599716186523
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,float16,127,0.008423999696969987
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,fp8,63,0.008363199979066848
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,float16,255,0.008371199667453765
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,fp8,127,0.008337599784135818
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,fp8,255,0.00844319984316826
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,float16,511,0.008416000008583068
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,fp8,511,0.00843999981880188
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,fp8,1023,0.008406399935483932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,float16,2047,0.012414400279521943
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,float16,4095,0.02860639989376068
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,fp8,4095,0.013777600228786468
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,float16,8191,0.04577920138835907
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,fp8,8191,0.02880159914493561
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,float16,1,0.008366400003433227
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,fp8,1,0.008363199979066848
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,float16,3,0.00835999995470047
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,fp8,3,0.008324799686670303
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,fp8,7,0.008369600027799606
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,float16,15,0.008390399813652038
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,fp8,15,0.008313599973917007
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,float16,31,0.008343999832868576
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,fp8,31,0.008441600203514098
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,float16,63,0.008376000076532364
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,fp8,63,0.008329600095748901
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,float16,127,0.008406399935483932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,fp8,127,0.008302400261163712
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,float16,255,0.008343999832868576
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,1,128,1,float16,fp8,63,0.04937280118465424
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,fp8,255,0.008377599716186523
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,float16,511,0.010356800258159637
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,fp8,511,0.008323200047016144
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,float16,1023,0.012401600182056428
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,fp8,1023,0.010384000092744827
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,float16,2047,0.026921600103378296
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,fp8,2047,0.012600000202655792
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,float16,4095,0.045187199115753175
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,fp8,4095,0.02775680124759674
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,float16,15,0.008363199979066848
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,float16,8191,0.08162239789962769
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,fp8,8191,0.04652960002422333
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,32,4,128,1,float16,float16,1,0.08798400163650513
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,float16,1,0.010438399761915207
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,fp8,1,0.010159999877214432
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,float16,3,0.010388799756765366
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,fp8,3,0.010356800258159637
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,fp8,7,0.010344000160694122
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,float16,7,0.010396800190210342
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,float16,15,0.010012800246477127
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,float16,15,0.010391999781131745
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,fp8,15,0.010375999659299851
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,float16,31,0.010372799634933472
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,fp8,31,0.009620799869298934
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,float16,63,0.010409600287675857
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,fp8,63,0.009374400228261947
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,float16,1023,0.010475199669599533
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,float16,255,0.010406400263309478
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,fp8,2047,0.010449600219726563
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,fp8,511,0.008582399785518646
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,float16,1023,0.010497599840164185
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,fp8,1023,0.010452800244092942
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,float16,2047,0.01252480000257492
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,fp8,2047,0.012443199753761292
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,float16,4095,0.028867200016975403
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,fp8,4095,0.014528000354766845
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,float16,8191,0.04656000137329101
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,fp8,8191,0.028921601176261903
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,float16,16383,0.0821615993976593
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,float16,1,0.008420799672603608
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,fp8,16383,0.04724000096321106
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,fp8,1,0.00841279998421669
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,fp8,3,0.008364800363779068
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,fp8,7,0.008345600217580795
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,float16,15,0.008633600175380706
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,fp8,15,0.008366400003433227
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,float16,7,0.008560000360012055
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,float16,31,0.008364800363779068
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,fp8,31,0.008417599648237229
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,float16,63,0.008387199789285659
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,float16,127,0.00835679993033409
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,float16,127,0.010391999781131745
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,fp8,127,0.009009599685668945
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,fp8,255,0.009040000289678574
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,float16,511,0.010171200335025787
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,float16,255,0.008350399881601333
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,fp8,255,0.008345600217580795
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,fp8,511,0.008417599648237229
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,fp8,127,0.008355200290679932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,float16,1023,0.01082720011472702
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,fp8,1023,0.010345599800348281
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,float16,2047,0.027030399441719054
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,fp8,2047,0.012547199428081513
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,float16,4095,0.04528320133686066
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,fp8,4095,0.02735840082168579
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,float16,8191,0.08101440072059632
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,fp8,8191,0.04618239998817444
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,float16,3,0.008316799998283386
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,float16,7,0.008392000198364257
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,fp8,1,0.008393599838018417
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,float16,16383,0.15248960256576538
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,float16,3,0.010206399857997895
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,fp8,3,0.008392000198364257
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,float16,7,0.009987200051546097
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,float16,15,0.010411199927330018
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,fp8,15,0.008427199721336365
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,fp8,31,0.00840959995985031
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,float16,63,0.01043199971318245
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,fp8,63,0.008327999711036682
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,fp8,63,0.008524800091981888
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,float16,31,0.010420800000429154
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,fp8,127,0.009092800319194794
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,fp8,255,0.008377599716186523
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,float16,511,0.012462399899959564
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,fp8,511,0.010377600044012069
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,float16,1023,0.02707360088825226
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,fp8,1023,0.013515199720859527
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,float16,255,0.010384000092744827
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,float16,2047,0.045377600193023684
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,fp8,2047,0.028935998678207397
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,float16,4095,0.08245279788970947
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,fp8,4095,0.04749760031700134
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,fp8,8191,0.08438559770584106
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,float16,8191,0.15463039875030518
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,fp8,16383,0.1601904034614563
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,float16,1,0.012521600723266602
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,float16,16383,0.3035167932510376
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,float16,3,0.012467200309038163
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,float16,7,0.012465599924325943
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,fp8,3,0.011214400082826615
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,float16,15,0.012559999525547028
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,fp8,15,0.010468800365924836
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,float16,511,0.010188800096511842
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,float16,31,0.012428800016641617
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,fp8,31,0.010486400127410889
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,float16,63,0.01244639977812767
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,fp8,63,0.010462400317192078
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,float16,127,0.012432000041007996
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,fp8,127,0.010473600029945374
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,float16,255,0.012430399656295776
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,fp8,255,0.010476800054311753
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,float16,511,0.027561599016189577
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,fp8,16383,0.0829263985157013
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,fp8,511,0.014580799639225006
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,fp8,7,0.008436799794435502
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,fp8,1023,0.02894560098648071
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,float16,2047,0.08418560028076172
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,float16,4095,0.1574928045272827
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,fp8,4095,0.0864575982093811
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,float16,127,0.010467199981212616
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,fp8,8191,0.16494079828262329
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,float16,8191,0.3061455965042114
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,float16,1,0.010435199737548828
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,fp8,1,0.010480000078678131
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,float16,3,0.010424000024795533
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,fp8,3,0.010396800190210342
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,fp8,16383,0.3258320093154907
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,float16,7,0.010419200360774993
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,float16,15,0.010438399761915207
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,fp8,7,0.008441600203514098
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,fp8,15,0.01045759990811348
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,float16,31,0.009452799707651139
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,fp8,31,0.009080000221729279
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,float16,63,0.00844319984316826
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,fp8,63,0.009004800021648407
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,float16,127,0.008427199721336365
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,fp8,127,0.009495999664068222
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,float16,255,0.008446399867534638
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,fp8,255,0.009217599779367447
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,fp8,1,0.011500799655914306
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,float16,511,0.008425600081682205
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,fp8,511,0.010438399761915207
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,float16,1023,0.010393600165843963
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,fp8,1023,0.008433599770069123
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,float16,2047,0.010388799756765366
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,fp8,2047,0.010422399640083313
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,fp8,7,0.010489600151777268
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,float16,4095,0.012222400307655335
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,fp8,4095,0.0104592002928257
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,float16,8191,0.01311199963092804
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,fp8,8191,0.012462399899959564
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,float16,1,0.008416000008583068
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,fp8,1,0.008396799862384795
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,float16,3,0.008414400368928909
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,fp8,3,0.008428800106048583
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,float16,7,0.008401600271463394
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,float16,1,0.009640000015497207
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,float16,15,0.008382400125265121
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,float16,1023,0.046312001347541806
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,fp8,15,0.00841120034456253
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,float16,31,0.008355200290679932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,fp8,2047,0.047495999932289125
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,fp8,63,0.008387199789285659
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,float16,127,0.008388800173997879
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,fp8,127,0.008404800295829773
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,float16,255,0.008475200086832047
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,fp8,255,0.008352000266313553
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,float16,511,0.008408000320196151
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,fp8,511,0.00843999981880188
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,float16,1023,0.008390399813652038
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,fp8,1023,0.008416000008583068
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,float16,2047,0.008713600039482117
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,fp8,2047,0.008432000130414962
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,float16,4095,0.010497599840164185
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,float16,16383,0.608355188369751
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,fp8,4095,0.010435199737548828
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,fp8,8191,0.012486399710178375
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,float16,8191,0.01252480000257492
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,float16,16383,0.016579200327396394
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,fp8,16383,0.01656160056591034
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,fp8,1,0.008347199857234954
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,float16,3,0.008448000252246856
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,fp8,3,0.008369600027799606
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,float16,7,0.008355200290679932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,fp8,7,0.00844319984316826
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,float16,15,0.008417599648237229
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,fp8,15,0.008353599905967712
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,float16,16383,0.01656160056591034
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,float16,31,0.008473599702119828
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,float16,63,0.008932799845933915
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,fp8,63,0.00844319984316826
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,float16,127,0.008380799740552902
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,fp8,127,0.008406399935483932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,fp8,255,0.00849440023303032
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,float16,255,0.008448000252246856
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,float16,511,0.008745600283145905
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,fp8,511,0.008451200276613235
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,fp8,7,0.008342400193214417
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,fp8,1023,0.008436799794435502
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,float16,2047,0.010388799756765366
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,fp8,2047,0.008457600325345992
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,float16,4095,0.010416000336408614
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,fp8,4095,0.01037440001964569
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,float16,8191,0.01263359934091568
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,fp8,8191,0.012542399764060973
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,float16,63,0.008448000252246856
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,fp8,16383,0.014496000111103058
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,float16,1,0.008825600147247314
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,float16,3,0.008617600053548813
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,fp8,3,0.008406399935483932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,float16,7,0.008404800295829773
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,fp8,7,0.008428800106048583
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,float16,15,0.008798400312662125
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,fp8,15,0.008428800106048583
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,float16,31,0.008416000008583068
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,fp8,31,0.008401600271463394
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,float16,63,0.00840959995985031
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,fp8,63,0.008433599770069123
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,float16,127,0.008828800171613693
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,fp8,127,0.008433599770069123
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,float16,255,0.00883520022034645
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,fp8,255,0.008401600271463394
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,float16,511,0.008401600271463394
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,float16,1,0.00838399976491928
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,float16,1023,0.009096000343561172
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,fp8,511,0.008353599905967712
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,float16,2047,0.010358399897813796
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,fp8,2047,0.010428799688816071
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,float16,4095,0.010435199737548828
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,fp8,4095,0.010460799932479859
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,float16,8191,0.012488000094890594
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,fp8,16383,0.01655520051717758
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,fp8,31,0.008353599905967712
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,float16,16383,0.015376000106334687
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,fp8,16383,0.014326399564743042
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,float16,1,0.01043199971318245
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,fp8,1,0.009347199648618697
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,float16,3,0.010440000146627427
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,fp8,3,0.009382399916648864
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,float16,1023,0.009486400336027146
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,fp8,7,0.00868159979581833
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,float16,15,0.010417599976062775
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,fp8,15,0.008462399989366532
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,float16,31,0.0104032002389431
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,fp8,31,0.00841120034456253
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,float16,63,0.010427200049161912
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,fp8,63,0.008446399867534638
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,float16,16383,0.016256000101566314
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,float16,127,0.010417599976062775
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,fp8,127,0.008476799726486206
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,float16,255,0.010384000092744827
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,fp8,255,0.008993600308895112
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,fp8,1,0.008350399881601333
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,float16,511,0.010385599732398988
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,fp8,511,0.010300800204277039
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,float16,1023,0.010391999781131745
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,fp8,1023,0.009473600238561631
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,float16,2047,0.01048159971833229
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,fp8,2047,0.010476800054311753
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,fp8,4095,0.010926400125026704
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,float16,4095,0.010486400127410889
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,float16,16383,0.01860159933567047
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,fp8,8191,0.012555199861526489
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,fp8,16383,0.016862399876117706
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,fp8,1,0.008345600217580795
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,float16,3,0.008427199721336365
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,fp8,3,0.008363199979066848
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,float16,7,0.008451200276613235
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,fp8,7,0.008446399867534638
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,float16,15,0.00846560001373291
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,fp8,15,0.008449599891901017
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,float16,31,0.00843840017914772
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,fp8,31,0.008430399745702744
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,float16,63,0.00835840031504631
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,fp8,63,0.008340799808502197
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,fp8,1023,0.008489599823951722
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,float16,127,0.008455999940633774
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,fp8,127,0.008455999940633774
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,fp8,255,0.008408000320196151
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,float16,511,0.008423999696969987
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,fp8,511,0.008408000320196151
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,float16,1023,0.0084927998483181
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,fp8,1023,0.008422400057315826
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,float16,2047,0.009473600238561631
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,fp8,2047,0.008417599648237229
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,float16,4095,0.010436800122261048
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,fp8,4095,0.010467199981212616
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,float16,8191,0.012476799637079239
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,fp8,31,0.008367999643087386
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,fp8,8191,0.012451200187206269
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,float16,16383,0.016502399742603303
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,fp8,16383,0.014519999921321868
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,float16,1,0.008441600203514098
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,float16,3,0.008395200222730636
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,fp8,3,0.008404800295829773
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,float16,7,0.00843840017914772
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,float16,8191,0.014284799993038177
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,fp8,7,0.008353599905967712
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,float16,15,0.008414400368928909
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,fp8,15,0.008444800227880477
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,float16,31,0.008459199965000153
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,float16,1,0.008441600203514098
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,fp8,31,0.008404800295829773
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,float16,63,0.008892799913883209
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,float16,127,0.008390399813652038
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,fp8,127,0.008444800227880477
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,float16,255,0.008761599659919739
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,float16,511,0.008548799902200699
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,fp8,255,0.008392000198364257
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,fp8,511,0.008452799916267396
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,float16,1023,0.008454400300979614
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,fp8,1023,0.008448000252246856
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,float16,2047,0.010380800068378448
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,fp8,2047,0.009985599666833878
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,float16,4095,0.0104592002928257
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,fp8,4095,0.01048320010304451
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,float16,8191,0.012559999525547028
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,fp8,8191,0.012432000041007996
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,float16,16383,0.01478559970855713
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,fp8,16383,0.013089600205421447
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,float16,1,0.00843520015478134
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,fp8,1,0.008857599645853042
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,float16,3,0.008428800106048583
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,fp8,3,0.00841120034456253
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,float16,7,0.008830399811267852
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,float16,255,0.008436799794435502
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,fp8,7,0.008420799672603608
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,float16,15,0.008777599781751633
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,float16,31,0.008388800173997879
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,fp8,31,0.008436799794435502
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,float16,63,0.008432000130414962
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,fp8,15,0.008364800363779068
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,float16,127,0.008457600325345992
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,fp8,127,0.008366400003433227
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,float16,255,0.00899680033326149
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,fp8,255,0.00843840017914772
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,fp8,8191,0.01252640038728714
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,fp8,511,0.008478400111198426
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,float16,1023,0.010380800068378448
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,fp8,1023,0.008537600189447403
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,float16,2047,0.010377600044012069
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,fp8,2047,0.010388799756765366
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,float16,4095,0.012491200119256973
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,float16,7,0.010463999956846238
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,float16,8191,0.014571200311183929
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,fp8,8191,0.012484800070524216
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,float16,16383,0.029241600632667543
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,float16,1,0.009431999921798707
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,fp8,1,0.008449599891901017
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,float16,3,0.008382400125265121
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,fp8,1,0.008416000008583068
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,fp8,3,0.008399999886751174
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,float16,7,0.008398400247097015
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,float16,15,0.008420799672603608
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,fp8,15,0.008366400003433227
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,float16,31,0.0083856001496315
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,fp8,31,0.00843840017914772
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,fp8,63,0.008441600203514098
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,float16,63,0.008633600175380706
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,fp8,63,0.008441600203514098
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,float16,127,0.010390400141477584
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,fp8,127,0.008420799672603608
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,float16,255,0.010023999959230423
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,fp8,255,0.008404800295829773
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,float16,511,0.00862400010228157
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,fp8,511,0.00843999981880188
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,float16,1023,0.010377600044012069
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,fp8,1023,0.008417599648237229
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,float16,2047,0.010449600219726563
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,fp8,2047,0.01040000021457672
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,float16,4095,0.012510399520397186
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,fp8,4095,0.012454400211572647
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,fp8,8191,0.014511999487876893
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,float16,8191,0.029316800832748412
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,float16,16383,0.04721759855747223
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,float16,1,0.008406399935483932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,fp8,1,0.008376000076532364
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,float16,3,0.008398400247097015
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,fp8,3,0.008367999643087386
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,fp8,63,0.008425600081682205
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,float16,7,0.008451200276613235
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,fp8,7,0.008355200290679932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,float16,15,0.00836160033941269
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,float16,511,0.009326399862766266
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,fp8,15,0.00835999995470047
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,float16,31,0.008451200276613235
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,fp8,31,0.00840959995985031
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,fp8,63,0.008428800106048583
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,float16,127,0.008417599648237229
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,fp8,127,0.008401600271463394
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,float16,255,0.008382400125265121
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,fp8,255,0.008352000266313553
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,float16,511,0.008510400354862214
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,fp8,4095,0.010524799674749374
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,float16,1023,0.010443200170993806
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,fp8,1023,0.008483199775218964
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,float16,2047,0.012392000108957291
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,fp8,16383,0.014640000462532044
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,fp8,2047,0.010513599961996078
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,float16,4095,0.027033600211143493
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,fp8,4095,0.01249760016798973
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,float16,8191,0.04571360051631927
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,fp8,8191,0.02817600071430206
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,float16,16383,0.08160319924354553
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,float16,1,0.00833119973540306
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,fp8,16383,0.047312000393867494
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,fp8,1,0.008366400003433227
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,fp8,3,0.008374399691820144
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,float16,3,0.008320000022649765
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,fp8,7,0.008345600217580795
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,fp8,15,0.008339200168848038
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,float16,31,0.008369600027799606
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,fp8,31,0.008353599905967712
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,float16,63,0.00846719965338707
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,fp8,63,0.008348800241947174
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,float16,127,0.008372800052165985
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,fp8,127,0.008392000198364257
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,float16,255,0.008366400003433227
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,fp8,255,0.008355200290679932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,float16,511,0.009486400336027146
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,fp8,511,0.008423999696969987
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,float16,1023,0.010526400059461594
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,fp8,1023,0.010409600287675857
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,float16,2047,0.02675040066242218
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,fp8,2047,0.012515200674533844
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,float16,4095,0.045480000972747806
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,fp8,4095,0.026950401067733765
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,float16,8191,0.08155199885368347
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,fp8,8191,0.04544320106506348
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,float16,63,0.008367999643087386
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,float16,16383,0.15392160415649414
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,float16,1,0.010398399829864503
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,fp8,16383,0.08244479894638061
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,fp8,1,0.008643200248479843
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,fp8,3,0.008436799794435502
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,float16,3,0.010393600165843963
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,fp8,7,0.008612799644470214
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,float16,15,0.010414399951696397
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,fp8,15,0.008574400097131729
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,float16,31,0.010369600355625152
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,fp8,31,0.008497600257396699
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,float16,63,0.010428799688816071
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,fp8,63,0.008497600257396699
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,float16,127,0.010420800000429154
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,fp8,127,0.00904960036277771
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,float16,255,0.010396800190210342
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,float16,511,0.01244800016283989
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,float16,7,0.008404800295829773
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,fp8,7,0.00936639979481697
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,float16,1023,0.027420800924301148
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,fp8,1023,0.014271999895572662
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,float16,2047,0.04541760087013245
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,fp8,2047,0.02884320020675659
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,float16,4095,0.08223999738693237
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,fp8,4095,0.04841279983520508
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,float16,8191,0.15554239749908447
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,fp8,8191,0.08933600187301635
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,fp8,16383,0.02967039942741394
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,float16,16383,0.3039024114608765
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,float16,1,0.010424000024795533
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,fp8,1,0.00941760018467903
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,float16,3,0.010425599664449692
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,fp8,3,0.009401600062847137
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,float16,7,0.010425599664449692
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,fp8,511,0.008427199721336365
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,float16,15,0.010371199995279311
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,float16,7,0.010440000146627427
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,fp8,15,0.010417599976062775
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,float16,31,0.010428799688816071
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,fp8,31,0.009134399890899658
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,float16,63,0.010409600287675857
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,fp8,63,0.009247999638319016
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,float16,127,0.010409600287675857
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,float16,255,0.010411199927330018
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,fp8,255,0.008476799726486206
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,fp8,255,0.008575999736785888
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,float16,511,0.010465600341558457
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,fp8,511,0.010480000078678131
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,float16,1023,0.010438399761915207
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,float16,15,0.00836160033941269
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,fp8,1023,0.009911999851465226
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,float16,2047,0.010441599786281586
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,fp8,2047,0.010360000282526016
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,float16,4095,0.01242400035262108
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,fp8,4095,0.010451199859380722
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,float16,8191,0.014497600495815277
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,float16,16383,0.01655520051717758
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,fp8,16383,0.016494399309158324
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,float16,1,0.008396799862384795
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,fp8,1,0.008419200032949447
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,float16,3,0.008408000320196151
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,fp8,3,0.008401600271463394
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,float16,7,0.008444800227880477
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,fp8,7,0.0083856001496315
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,float16,15,0.008444800227880477
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,fp8,15,0.008423999696969987
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,float16,31,0.00840959995985031
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,fp8,31,0.008390399813652038
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,float16,63,0.008408000320196151
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,fp8,63,0.00838399976491928
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,float16,127,0.008414400368928909
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,fp8,127,0.008428800106048583
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,float16,255,0.008425600081682205
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,fp8,255,0.008451200276613235
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,float16,511,0.008423999696969987
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,fp8,511,0.008457600325345992
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,float16,1023,0.010339199751615524
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,fp8,1023,0.008436799794435502
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,float16,2047,0.010489600151777268
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,fp8,2047,0.010335999727249145
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,float16,4095,0.010683199763298035
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,fp8,4095,0.010462400317192078
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,fp8,8191,0.012721599638462066
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,float16,16383,0.015563200414180755
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,fp8,16383,0.013548800349235534
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,float16,1,0.008455999940633774
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,float16,3,0.008379200100898742
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,fp8,3,0.00835679993033409
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,float16,7,0.008476799726486206
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,fp8,16383,0.17189600467681884
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,fp8,7,0.008390399813652038
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,fp8,15,0.008401600271463394
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,float16,31,0.008446399867534638
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,float16,15,0.008726400136947633
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,fp8,31,0.00841279998421669
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,fp8,63,0.008425600081682205
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,fp8,7,0.009393599629402161
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,float16,127,0.00843520015478134
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,fp8,127,0.008376000076532364
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,float16,255,0.00841120034456253
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,fp8,255,0.008382400125265121
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,float16,511,0.008473599702119828
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,fp8,511,0.008444800227880477
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,float16,1023,0.008448000252246856
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,float16,2047,0.01048159971833229
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,fp8,127,0.010382399708032609
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,fp8,2047,0.010139200091361999
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,float16,4095,0.012449599802494049
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,fp8,4095,0.010454399883747101
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,float16,8191,0.014511999487876893
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,fp8,8191,0.012563200294971466
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,float16,16383,0.030423998832702637
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,fp8,16383,0.014555199444293976
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,float16,1,0.008379200100898742
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,fp8,1,0.00838399976491928
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,float16,3,0.008417599648237229
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,fp8,8191,0.013436800241470337
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,fp8,3,0.008401600271463394
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,float16,7,0.008404800295829773
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,fp8,7,0.008318399637937545
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,float16,15,0.008367999643087386
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,fp8,15,0.00833119973540306
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,float16,31,0.008343999832868576
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,fp8,31,0.008350399881601333
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,float16,63,0.008329600095748901
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,fp8,63,0.008329600095748901
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,float16,127,0.008395200222730636
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,fp8,127,0.008432000130414962
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,float16,255,0.008420799672603608
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,fp8,255,0.008355200290679932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,fp8,511,0.008414400368928909
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,float16,511,0.008432000130414962
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,float16,1023,0.008955200016498566
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,fp8,1023,0.008355200290679932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,float16,2047,0.010471999645233154
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,float16,4095,0.012518399953842163
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,fp8,4095,0.010500799864530563
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,float16,8191,0.01295360028743744
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,float16,8191,0.02749119997024536
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,fp8,8191,0.014444799721240997
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,float16,16383,0.04591839909553528
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,float16,1,0.008462399989366532
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,fp8,16383,0.028571200370788575
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,fp8,1,0.008352000266313553
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,float16,3,0.00843840017914772
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,fp8,3,0.008324799686670303
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,float16,7,0.00840959995985031
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,fp8,7,0.008404800295829773
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,float16,15,0.008318399637937545
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,fp8,15,0.008345600217580795
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,float16,31,0.008451200276613235
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,fp8,31,0.00843999981880188
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,float16,63,0.008414400368928909
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,fp8,63,0.008423999696969987
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,float16,127,0.008343999832868576
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,fp8,127,0.008436799794435502
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,float16,255,0.008347199857234954
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,fp8,255,0.008336000144481659
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,float16,511,0.010390400141477584
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,fp8,511,0.0094480000436306
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,float16,1023,0.012411200255155564
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,fp8,1023,0.010531199723482132
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,fp8,2047,0.01448799967765808
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,float16,2047,0.027113598585128785
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,float16,4095,0.04600479900836944
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,fp8,4095,0.028865599632263185
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,fp8,1023,0.008892799913883209
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,float16,1,0.010420800000429154
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,float16,8191,0.0821008026599884
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,fp8,8191,0.04734559953212738
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,fp8,1,0.008708799630403519
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,float16,3,0.010366400331258773
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,fp8,3,0.008414400368928909
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,float16,7,0.010425599664449692
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,fp8,7,0.008491200208663941
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,fp8,511,0.009508799761533737
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,float16,15,0.010344000160694122
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,float16,31,0.01040000021457672
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,fp8,31,0.00936800017952919
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,float16,63,0.01032319962978363
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,fp8,63,0.008856000006198883
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,fp8,127,0.008352000266313553
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,fp8,255,0.00840959995985031
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,float16,511,0.012412799894809723
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,fp8,511,0.010425599664449692
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,float16,1023,0.02770400047302246
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,fp8,1023,0.014401599764823914
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,float16,2047,0.045875200629234315
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,fp8,2047,0.028863999247550964
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,float16,4095,0.0819487988948822
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,fp8,2047,0.009913600236177444
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,float16,8191,0.15409280061721803
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,fp8,8191,0.08576319813728332
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,float16,1,0.012452799826860428
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,fp8,1,0.010856000334024429
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,fp8,1,0.00833119973540306
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,float16,3,0.012516799569129943
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,fp8,3,0.01048320010304451
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,float16,7,0.012470400333404541
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,fp8,7,0.010451199859380722
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,float16,31,0.012399999797344208
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,fp8,31,0.010454399883747101
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,float16,63,0.012439999729394913
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,fp8,63,0.010476800054311753
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,float16,127,0.012425599992275238
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,fp8,127,0.010460799932479859
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,float16,63,0.008641599863767623
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,float16,255,0.012486399710178375
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,fp8,255,0.010454399883747101
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,float16,511,0.028340798616409302
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,fp8,511,0.014563199877738953
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,float16,1023,0.04719679951667786
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,fp8,1023,0.028921601176261903
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,float16,2047,0.0838864028453827
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,fp8,2047,0.04763199985027313
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,fp8,4095,0.08439679741859436
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,float16,4095,0.15828800201416016
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,fp8,15,0.008779200166463852
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,float16,1,0.016572800278663636
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,float16,127,0.010487999767065048
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,fp8,1,0.014539200067520141
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,float16,3,0.016947199404239655
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,fp8,8191,0.1580191969871521
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,fp8,3,0.015315200388431548
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,float16,8191,0.30695040225982667
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,fp8,7,0.014590400457382201
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,float16,7,0.01704320013523102
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,fp8,15,0.014555199444293976
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,float16,31,0.016916799545288085
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,float16,15,0.01661919951438904
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,fp8,31,0.014579200744628906
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,float16,63,0.0166143998503685
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,fp8,63,0.01454399973154068
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,float16,127,0.016948799788951873
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,fp8,127,0.014590400457382201
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,float16,255,0.02736479938030243
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,fp8,255,0.014819200336933135
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,fp8,511,0.02882240116596222
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,fp8,4095,0.0475951999425888
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,float16,1023,0.08260319828987121
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,fp8,2047,0.08538240194320679
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,float16,15,0.012439999729394913
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,fp8,15,0.010468800365924836
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,float16,4095,0.3009376049041748
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,fp8,4095,0.164027202129364
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,float16,1,0.010443200170993806
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,fp8,1,0.01037440001964569
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,float16,3,0.010471999645233154
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,fp8,3,0.010393600165843963
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,fp8,8191,0.32246239185333253
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,float16,7,0.010679999738931656
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,float16,8191,0.5902624130249023
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,float16,15,0.010592000186443329
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,fp8,7,0.01037919968366623
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,float16,31,0.010558400303125381
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,fp8,31,0.010407999902963639
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,float16,63,0.010896000266075134
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,fp8,15,0.010503999888896942
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,fp8,63,0.010384000092744827
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,fp8,127,0.010441599786281586
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,float16,255,0.010516799986362457
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,fp8,255,0.010384000092744827
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,float16,255,0.01045759990811348
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,fp8,511,0.012432000041007996
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,float16,1023,0.02890239953994751
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,fp8,1023,0.014571200311183929
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,float16,2047,0.047337600588798524
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,fp8,2047,0.028913599252700806
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,float16,1,0.01244800016283989
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,fp8,1,0.010499200224876404
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,fp8,3,0.010463999956846238
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,float16,7,0.012452799826860428
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,fp8,7,0.011519999802112579
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,float16,15,0.012503999471664428
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,float16,511,0.045465600490570066
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,fp8,15,0.011614400148391723
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,fp8,31,0.01157120019197464
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,float16,63,0.01252799928188324
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,fp8,63,0.010487999767065048
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,float16,2047,0.1551584005355835
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,fp8,127,0.012142399698495865
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,float16,255,0.012544000148773193
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,fp8,255,0.011383999884128571
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,float16,511,0.028233599662780762
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,fp8,511,0.01459999978542328
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,float16,1023,0.04678719937801361
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,fp8,1023,0.02908160090446472
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,float16,2047,0.08394399881362916
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,float16,1,0.01687999963760376
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,fp8,1,0.014534400403499603
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,float16,3,0.017001600563526155
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,fp8,3,0.014555199444293976
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,float16,7,0.01655520051717758
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,fp8,7,0.014580799639225006
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,float16,15,0.016646400094032288
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,fp8,15,0.014532800018787383
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,float16,127,0.010502400249242783
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,float16,31,0.01660960018634796
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,float16,511,0.014500799775123595
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,fp8,31,0.014596800506114959
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,float16,63,0.016604800522327424
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,fp8,63,0.015110400319099427
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,float16,127,0.01664000004529953
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,float16,255,0.0283376008272171
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,float16,511,0.04674080014228821
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,float16,3,0.012451200187206269
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,fp8,511,0.02879199981689453
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,float16,1023,0.08309760093688964
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,fp8,1023,0.04745120108127594
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,float16,2047,0.1557360053062439
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,fp8,2047,0.08575360178947448
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,float16,31,0.012511999905109405
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,float16,1,0.026899200677871705
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,fp8,1,0.022788800299167633
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,float16,3,0.02714560031890869
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,fp8,3,0.022767999768257143
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,float16,7,0.026867198944091796
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,fp8,7,0.022868800163269042
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,float16,15,0.026855999231338502
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,float16,127,0.012455999851226807
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,fp8,15,0.022780799865722658
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,float16,31,0.026982399821281432
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,fp8,31,0.022808000445365906
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,float16,63,0.026972800493240356
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,fp8,63,0.022767999768257143
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,float16,127,0.030876800417900085
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,fp8,127,0.02449920028448105
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,float16,255,0.046696001291275026
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,fp8,255,0.02975200116634369
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,float16,511,0.08266879916191101
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,fp8,511,0.047600001096725464
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,float16,1023,0.15584479570388793
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,fp8,1023,0.08595520257949829
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,float16,1,0.014496000111103058
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,fp8,1,0.012918399274349212
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,fp8,2047,0.0476175993680954
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,float16,3,0.014484800398349762
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,float16,2047,0.3001807928085327
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,fp8,2047,0.1638815999031067
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,float16,7,0.014511999487876893
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,fp8,7,0.01249760016798973
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,fp8,15,0.012467200309038163
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,float16,31,0.014523200690746307
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,fp8,31,0.012494400143623352
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,float16,63,0.014558400213718414
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,fp8,63,0.012508800625801087
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,float16,127,0.01446399986743927
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,fp8,127,0.012516799569129943
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,float16,255,0.01446399986743927
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,fp8,255,0.012489599734544754
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,float16,511,0.028896000981330872
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,fp8,511,0.016659200191497803
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,float16,1023,0.047625601291656494
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,fp8,1023,0.030987200140953065
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,float16,1,0.016883200407028197
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,fp8,1,0.015974399447441102
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,fp8,127,0.01454080045223236
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,float16,3,0.017433600127696992
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,fp8,3,0.014753599464893342
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,float16,7,0.017377600073814392
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,fp8,7,0.01659200042486191
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,float16,15,0.016675199568271636
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,fp8,15,0.016457599401474
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,float16,31,0.0178399994969368
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,fp8,31,0.015323199331760406
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,float16,63,0.018006399273872375
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,float16,127,0.01868959963321686
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,fp8,127,0.016575999557971954
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,float16,255,0.02887359857559204
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,fp8,255,0.016366399824619293
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,float16,511,0.047286400198936464
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,fp8,1023,0.048102399706840514
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,fp8,1023,0.04895040094852447
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,float16,1,0.026926401257514953
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,fp8,1,0.02449440062046051
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,float16,3,0.027556800842285158
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,fp8,3,0.024382400512695312
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,float16,7,0.027449598908424376
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,fp8,7,0.024371199309825897
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,float16,15,0.027564799785614012
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,fp8,15,0.024769599735736846
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,float16,31,0.026825600862503053
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,fp8,31,0.02449759989976883
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,float16,63,0.02773759961128235
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,fp8,63,0.023403200507164
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,float16,15,0.014521600306034088
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,float16,127,0.03094240128993988
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,fp8,127,0.023396800458431243
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,float16,255,0.04725599884986877
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,fp8,255,0.029849600791931153
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,float16,511,0.08423200249671936
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,fp8,511,0.04835039973258972
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,float16,1,0.04737600088119507
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,float16,1023,0.15777280330657958
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,fp8,1,0.03995839953422546
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,float16,3,0.04741120040416717
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,fp8,3,0.040033599734306334
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,float16,7,0.047366398572921756
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,fp8,7,0.04016799926757812
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,float16,15,0.04736000001430511
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,fp8,255,0.014558400213718414
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,fp8,15,0.04016000032424927
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,float16,31,0.04738239943981171
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,fp8,31,0.03987680077552795
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,float16,63,0.0493151992559433
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,fp8,63,0.04091199934482574
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,fp8,63,0.016457599401474
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,float16,127,0.05131040215492248
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,fp8,127,0.042208001017570496
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,float16,255,0.08409919738769531
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,fp8,255,0.05109120011329651
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,fp8,511,0.08615520000457763
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,float16,511,0.15612000226974487
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,float16,1,0.010412800312042236
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,float16,3,0.01040000021457672
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,fp8,1,0.008841600269079208
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,float16,1023,0.08316640257835388
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,fp8,3,0.010375999659299851
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,float16,7,0.01040000021457672
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,float16,15,0.01040479987859726
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,fp8,7,0.010380800068378448
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,float16,31,0.01037919968366623
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,fp8,31,0.010281600058078766
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,float16,63,0.010371199995279311
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,fp8,63,0.009464000165462495
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,float16,127,0.010358399897813796
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,fp8,127,0.01034879982471466
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,float16,255,0.01035040020942688
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,fp8,255,0.010422399640083313
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,float16,511,0.01037919968366623
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,fp8,511,0.010409600287675857
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,float16,1023,0.010424000024795533
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,fp8,1023,0.010371199995279311
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,float16,2047,0.010702399909496308
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,fp8,2047,0.010444799810647965
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,float16,4095,0.01242239996790886
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,fp8,4095,0.01210239976644516
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,float16,8191,0.014494399726390838
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,fp8,8191,0.012857599556446076
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,float16,16383,0.016527999937534333
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,fp8,1023,0.08511999845504761
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,fp8,16383,0.014511999487876893
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,float16,1,0.008420799672603608
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,fp8,1,0.008372800052165985
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,float16,3,0.008414400368928909
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,float16,7,0.008403199911117553
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,fp8,7,0.008417599648237229
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,float16,15,0.008444800227880477
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,fp8,15,0.008343999832868576
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,float16,31,0.008390399813652038
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,fp8,31,0.008379200100898742
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,float16,63,0.00843520015478134
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,fp8,63,0.00841120034456253
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,float16,127,0.00844319984316826
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,fp8,511,0.029473599791526795
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,fp8,127,0.0083856001496315
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,float16,255,0.008823999762535095
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,fp8,255,0.008423999696969987
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,float16,1023,0.3008255958557129
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,fp8,1023,0.1611199975013733
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,fp8,15,0.010372799634933472
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,float16,2047,0.01045600026845932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,fp8,2047,0.010396800190210342
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,float16,4095,0.012449599802494049
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,fp8,4095,0.010806400328874588
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,float16,8191,0.014524799585342408
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,fp8,8191,0.012515200674533844
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,float16,16383,0.0290367990732193
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,fp8,16383,0.015910400450229643
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,float16,1,0.008326400071382523
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,float16,3,0.008459199965000153
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,fp8,1,0.00835999995470047
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,fp8,3,0.008380799740552902
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,float16,7,0.008369600027799606
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,fp8,7,0.008339200168848038
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,float16,15,0.008342400193214417
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,fp8,15,0.00833119973540306
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,float16,31,0.008316799998283386
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,float16,63,0.008417599648237229
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,fp8,63,0.008372800052165985
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,float16,127,0.008369600027799606
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,fp8,127,0.008340799808502197
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,float16,255,0.008329600095748901
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,fp8,255,0.008320000022649765
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,float16,511,0.008337599784135818
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,fp8,511,0.00833280012011528
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,float16,1023,0.00894080027937889
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,fp8,1023,0.008441600203514098
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,fp8,3,0.008353599905967712
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,float16,2047,0.010465600341558457
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,fp8,2047,0.008432000130414962
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,float16,4095,0.012436799705028534
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,fp8,4095,0.010467199981212616
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,float16,8191,0.027267199754714967
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,fp8,8191,0.014000000059604644
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,float16,16383,0.04540640115737915
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,fp8,16383,0.028835201263427736
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,fp8,1,0.008401600271463394
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,float16,1,0.008459199965000153
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,float16,3,0.008414400368928909
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,fp8,3,0.008454400300979614
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,float16,7,0.008422400057315826
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,fp8,7,0.00844319984316826
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,float16,15,0.008403199911117553
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,fp8,15,0.008352000266313553
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,float16,31,0.008486399799585343
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,fp8,31,0.008350399881601333
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,float16,511,0.00928800031542778
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,float16,1023,0.010131199657917023
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,fp8,63,0.00843520015478134
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,float16,127,0.008414400368928909
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,fp8,127,0.008432000130414962
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,fp8,1023,0.008894400298595428
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,fp8,511,0.00835999995470047
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,float16,63,0.008427199721336365
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,float16,1023,0.010476800054311753
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,fp8,1023,0.009169600158929824
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,float16,2047,0.012476799637079239
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,fp8,2047,0.010465600341558457
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,float16,4095,0.027505600452423097
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,fp8,4095,0.012651200592517852
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,float16,8191,0.045414400100708005
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,fp8,8191,0.028207999467849732
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,float16,16383,0.08193759918212891
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,fp8,3,0.013147200644016265
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,fp8,16383,0.04665440022945404
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,float16,1,0.022679999470710754
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,fp8,1,0.0186256006360054
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,float16,3,0.022654399275779724
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,fp8,3,0.01868479996919632
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,float16,7,0.022716799378395082
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,fp8,7,0.01868959963321686
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,float16,15,0.022686399519443512
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,fp8,15,0.01857600063085556
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,float16,31,0.022644799947738648
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,fp8,31,0.018607999384403228
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,float16,63,0.022763200104236603
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,fp8,63,0.018691200017929076
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,float16,127,0.022729599475860597
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,fp8,127,0.018697600066661834
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,float16,255,0.029259198904037477
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,fp8,255,0.018699200451374055
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,float16,511,0.047814399003982544
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,fp8,511,0.030881598591804504
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,float16,1,0.026836800575256347
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,fp8,1,0.024716800451278685
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,float16,3,0.026830399036407472
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,fp8,3,0.024740800261497498
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,float16,7,0.026790401339530943
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,fp8,7,0.024806399643421174
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,float16,15,0.026940798759460448
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,fp8,15,0.02473440021276474
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,float16,31,0.026952001452445983
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,fp8,31,0.024710400402545928
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,float16,63,0.02829279899597168
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,fp8,63,0.02476319968700409
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,float16,127,0.030943998694419862
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,fp8,127,0.024779200553894043
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,float16,255,0.047295999526977536
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,fp8,255,0.03118560016155243
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,fp8,511,0.008475200086832047
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,fp8,511,0.0494704008102417
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,float16,255,0.00833119973540306
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,float16,1,0.047444799542427064
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,fp8,1,0.04123679995536804
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,float16,511,0.008390399813652038
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,float16,3,0.04745439887046814
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,fp8,3,0.04120000004768372
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,float16,7,0.04737919867038727
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,fp8,7,0.04118399918079376
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,float16,15,0.04735040068626404
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,float16,31,0.04779999852180481
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,fp8,31,0.04115520119667053
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,float16,63,0.049377599358558656
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,fp8,63,0.04103519916534424
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,float16,127,0.05144320130348205
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,fp8,127,0.043243199586868286
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,float16,255,0.08475840091705322
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,fp8,255,0.050627201795578
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,float16,511,0.15848159790039062
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,fp8,511,0.08623200058937072
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,fp8,31,0.00841279998421669
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,fp8,1,0.07326239943504334
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,float16,3,0.08684639930725098
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,fp8,3,0.07256640195846557
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,fp8,7,0.07264479994773865
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,float16,15,0.08727359771728516
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,fp8,15,0.07277920246124267
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,float16,31,0.08804479837417603
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,fp8,31,0.07308800220489502
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,float16,63,0.08833280205726624
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,fp8,63,0.07397599816322327
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,float16,127,0.09300640225410461
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,fp8,127,0.07545599937438965
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,fp8,255,0.09303200244903564
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,float16,255,0.15880160331726073
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,float16,1,0.03712640106678009
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,fp8,511,0.16166720390319825
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,fp8,1,0.028996801376342772
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,float16,3,0.03718400001525879
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,fp8,3,0.030073601007461547
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,float16,7,0.037115201354026794
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,fp8,7,0.029713600873947144
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,float16,511,0.08448320031166076
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,float16,15,0.03718880116939545
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,fp8,15,0.029791998863220214
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,float16,31,0.037089601159095764
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,fp8,31,0.029785600304603577
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,float16,63,0.03902559876441956
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,fp8,63,0.030036801099777223
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,float16,127,0.039105600118637084
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,fp8,127,0.02991040050983429
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,float16,255,0.04936800003051758
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,fp8,255,0.032923200726509096
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,float16,1,0.0477647989988327
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,fp8,1,0.0417248010635376
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,float16,3,0.04745439887046814
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,float16,7,0.047779199481010434
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,fp8,7,0.041377601027488706
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,float16,15,0.04778720140457153
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,fp8,15,0.04128159880638123
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,float16,31,0.04938080012798309
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,float16,1,0.08669440150260925
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,float16,63,0.049403199553489686
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,fp8,63,0.04140160083770752
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,float16,7,0.08681759834289551
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,float16,127,0.05194240212440491
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,fp8,127,0.044126400351524354
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,float16,255,0.08617439866065979
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,fp8,255,0.05246080160140991
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,float16,1,0.08702239990234376
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,fp8,1,0.07396000027656555
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,float16,3,0.08680959939956664
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,fp8,3,0.07397119998931885
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,float16,7,0.08695679903030396
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,fp8,7,0.07395679950714111
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,float16,15,0.0883679986000061
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,float16,31,0.088400000333786
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,fp8,31,0.07396640181541443
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,float16,511,0.30337278842926024
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,fp8,63,0.0748799979686737
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,float16,127,0.09440320134162902
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,fp8,255,0.008374399691820144
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,fp8,127,0.07611839771270752
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,fp8,15,0.04118080139160156
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,fp8,255,0.09326080083847046
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,float16,255,0.16062239408493043
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,fp8,3,0.04257600009441376
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,float16,1,0.16442400217056274
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,fp8,1,0.1379680037498474
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,float16,3,0.16535359621047974
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,fp8,3,0.13799519538879396
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,float16,7,0.1662768006324768
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,fp8,31,0.04121440052986145
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,fp8,15,0.1382159948348999
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,float16,31,0.16624799966812134
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,fp8,63,0.13871840238571168
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,float16,63,0.16694079637527465
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,float16,127,0.17630720138549805
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,float16,255,0.3086719989776611
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,fp8,255,0.17619199752807618
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,float16,63,0.08847839832305908
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,float16,1,0.010446400195360184
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,fp8,1,0.009143999963998794
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,float16,3,0.010441599786281586
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,fp8,3,0.009353599697351455
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,float16,7,0.010451199859380722
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,float16,15,0.010416000336408614
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,fp8,15,0.009385599941015243
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,float16,31,0.01037919968366623
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,fp8,31,0.009438399970531464
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,float16,63,0.01034879982471466
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,fp8,63,0.009108799695968627
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,float16,127,0.01037440001964569
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,fp8,127,0.009436800330877303
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,float16,255,0.010335999727249145
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,fp8,255,0.009363199770450591
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,float16,511,0.010335999727249145
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,fp8,511,0.010177599638700486
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,float16,1023,0.010467199981212616
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,fp8,1023,0.010454399883747101
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,float16,2047,0.01157120019197464
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,fp8,2047,0.010521599650382995
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,float16,4095,0.012617599964141846
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,fp8,4095,0.012492799758911132
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,float16,8191,0.01478559970855713
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,fp8,8191,0.014535999298095703
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,fp8,7,0.1380895972251892
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,float16,15,0.16639039516448975
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,float16,16383,0.030806401371955873
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,fp8,16383,0.016729600727558136
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,float16,3,0.008416000008583068
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,fp8,3,0.008427199721336365
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,fp8,31,0.13855199813842772
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,float16,15,0.008427199721336365
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,float16,7,0.008316799998283386
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,fp8,15,0.00840959995985031
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,float16,31,0.008449599891901017
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,fp8,31,0.008327999711036682
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,float16,63,0.008318399637937545
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,fp8,63,0.008348800241947174
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,float16,127,0.008366400003433227
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,fp8,127,0.008337599784135818
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,float16,255,0.008395200222730636
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,fp8,255,0.008414400368928909
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,fp8,127,0.14058560132980347
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,float16,511,0.008398400247097015
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,float16,1023,0.00840959995985031
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,fp8,1023,0.008432000130414962
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,float16,2047,0.010487999767065048
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,fp8,2047,0.009145600348711013
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,float16,4095,0.012443199753761292
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,fp8,4095,0.010478399693965912
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,float16,8191,0.027156800031661987
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,fp8,8191,0.014457599818706512
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,fp8,15,0.07394400238990784
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,float16,16383,0.04532000124454498
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,float16,1,0.008392000198364257
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,fp8,16383,0.02877599895000458
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,fp8,1,0.008430399745702744
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,float16,7,0.008420799672603608
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,fp8,3,0.008382400125265121
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,float16,3,0.008406399935483932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,float16,15,0.008436799794435502
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,fp8,15,0.00835999995470047
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,float16,31,0.008416000008583068
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,fp8,31,0.00840959995985031
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,fp8,7,0.009337600320577621
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,float16,63,0.008337599784135818
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,fp8,63,0.008363199979066848
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,float16,127,0.008428800106048583
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,fp8,127,0.008363199979066848
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,float16,255,0.008452799916267396
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,fp8,255,0.008417599648237229
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,float16,511,0.008396799862384795
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,float16,1023,0.010441599786281586
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,fp8,1023,0.009454400092363358
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,float16,2047,0.012433599680662155
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,fp8,2047,0.010444799810647965
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,float16,4095,0.02842560112476349
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,fp8,4095,0.01393280029296875
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,float16,8191,0.04548319876194
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,float16,1,0.00836160033941269
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,fp8,8191,0.028352001309394838
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,fp8,1,0.008420799672603608
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,fp8,16383,0.045875200629234315
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,fp8,7,0.008399999886751174
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,fp8,1,0.008296000212430954
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,float16,3,0.008379200100898742
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,fp8,3,0.008339200168848038
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,float16,7,0.00844319984316826
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,fp8,7,0.008315200358629227
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,float16,15,0.008395200222730636
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,fp8,15,0.008366400003433227
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,float16,31,0.008451200276613235
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,fp8,31,0.00828320011496544
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,float16,63,0.008427199721336365
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,fp8,63,0.008310399949550629
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,float16,127,0.008379200100898742
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,fp8,127,0.008387199789285659
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,float16,255,0.008433599770069123
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,fp8,255,0.008310399949550629
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,float16,511,0.010451199859380722
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,fp8,511,0.008401600271463394
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,float16,1023,0.012123200297355651
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,fp8,1023,0.010380800068378448
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,float16,2047,0.02689119875431061
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,fp8,2047,0.012715199589729309
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,float16,4095,0.04505920112133026
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,fp8,4095,0.0273391991853714
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,float16,8191,0.08157600164413452
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,fp8,8191,0.04665760099887848
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,fp8,7,0.008340799808502197
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,float16,1,0.008388800173997879
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,fp8,16383,0.08345119953155518
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,float16,3,0.008324799686670303
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,fp8,1,0.00840959995985031
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,fp8,3,0.00843520015478134
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,float16,7,0.008401600271463394
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,fp8,7,0.008320000022649765
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,float16,15,0.008387199789285659
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,fp8,15,0.008310399949550629
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,float16,31,0.008404800295829773
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,fp8,31,0.008436799794435502
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,float16,63,0.008393599838018417
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,fp8,63,0.008406399935483932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,float16,127,0.008444800227880477
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,float16,255,0.008340799808502197
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,fp8,255,0.00835679993033409
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,float16,511,0.00835840031504631
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,fp8,511,0.00843520015478134
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,fp8,511,0.00835999995470047
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,float16,1023,0.010366400331258773
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,fp8,1023,0.010252799838781357
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,fp8,2047,0.01037440001964569
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,fp8,4095,0.01250240057706833
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,float16,8191,0.04580000042915344
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,fp8,8191,0.02885279953479767
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,float16,16383,0.08212479948997498
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,float16,16383,0.08195679783821105
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,float16,1,0.008387199789285659
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,float16,32767,0.15336320400238038
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,fp8,32767,0.08269280195236206
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,float16,1,0.008372800052165985
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,fp8,511,0.00833280012011528
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,fp8,3,0.008336000144481659
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,float16,7,0.008345600217580795
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,fp8,7,0.008339200168848038
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,float16,15,0.008339200168848038
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,fp8,15,0.008339200168848038
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,float16,31,0.008337599784135818
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,fp8,31,0.008355200290679932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,float16,63,0.008420799672603608
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,fp8,63,0.008315200358629227
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,float16,127,0.008444800227880477
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,float16,16383,0.15364960432052613
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,float16,255,0.00840959995985031
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,fp8,255,0.008324799686670303
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,float16,511,0.0094991996884346
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,fp8,511,0.008399999886751174
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,float16,1023,0.0124208003282547
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,float16,2047,0.012430399656295776
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,fp8,1023,0.010375999659299851
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,float16,4095,0.027588799595832825
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,float16,2047,0.026977598667144775
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,fp8,2047,0.0124719999730587
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,float16,4095,0.04518879950046539
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,fp8,4095,0.02800000011920929
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,fp8,8191,0.045902401208877563
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,fp8,16383,0.04673759937286377
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,float16,16383,0.1528591990470886
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,fp8,16383,0.0834991991519928
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,fp8,1,0.008374399691820144
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,float16,3,0.008350399881601333
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,float16,1,0.010417599976062775
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,float16,32767,0.29618399143218993
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,float16,3,0.010428799688816071
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,fp8,32767,0.15720959901809692
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,float16,7,0.010446400195360184
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,fp8,3,0.008448000252246856
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,float16,15,0.010443200170993806
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,float16,31,0.01045759990811348
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,fp8,7,0.008423999696969987
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,fp8,31,0.008377599716186523
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,float16,63,0.01037440001964569
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,fp8,63,0.008398400247097015
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,float16,127,0.010372799634933472
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,fp8,127,0.008884800225496292
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,float16,255,0.010366400331258773
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,fp8,255,0.008403199911117553
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,float16,511,0.012403199821710587
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,fp8,511,0.010467199981212616
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,float16,1023,0.02717759907245636
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,fp8,127,0.008343999832868576
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,fp8,1023,0.01403840035200119
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,float16,2047,0.04610399901866913
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,fp8,2047,0.028803199529647827
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,float16,4095,0.08213599920272827
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,fp8,4095,0.04720959961414337
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,float16,8191,0.1546671986579895
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,fp8,8191,0.08438400030136109
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,fp8,127,0.008432000130414962
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,float16,1,0.008371199667453765
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,fp8,16383,0.15900319814682007
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,fp8,1,0.008345600217580795
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,fp8,3,0.008323200047016144
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,float16,3,0.008432000130414962
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,float16,7,0.008404800295829773
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,fp8,7,0.008395200222730636
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,float16,15,0.008406399935483932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,fp8,15,0.008376000076532364
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,float16,31,0.008420799672603608
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,fp8,32767,0.3120687961578369
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,float16,63,0.008427199721336365
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,float16,32767,0.5921696186065674
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,float16,127,0.008363199979066848
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,float16,255,0.008313599973917007
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,fp8,63,0.008390399813652038
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,float16,8191,0.08175039887428284
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,float16,511,0.008390399813652038
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,fp8,127,0.008444800227880477
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,float16,1023,0.008355200290679932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,fp8,1023,0.008459199965000153
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,fp8,511,0.008452799916267396
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,fp8,2047,0.008414400368928909
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,float16,4095,0.010502400249242783
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,fp8,4095,0.010428799688816071
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,float16,8191,0.012457600235939026
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,fp8,8191,0.01250080019235611
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,float16,16383,0.01653439998626709
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,fp8,16383,0.016495999693870545
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,float16,32767,0.018492799997329713
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,fp8,32767,0.016631999611854555
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,float16,1,0.008363199979066848
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,fp8,1,0.008419200032949447
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,float16,3,0.00835679993033409
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,fp8,1,0.008441600203514098
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,fp8,3,0.008388800173997879
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,float16,7,0.008451200276613235
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,fp8,7,0.008428800106048583
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,fp8,15,0.008326400071382523
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,float16,31,0.008452799916267396
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,fp8,31,0.008363199979066848
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,float16,63,0.008347199857234954
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,fp8,63,0.008339200168848038
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,float16,127,0.00838399976491928
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,fp8,127,0.00835999995470047
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,float16,255,0.008427199721336365
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,float16,511,0.00846880003809929
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,fp8,511,0.008355200290679932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,float16,1023,0.008408000320196151
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,fp8,1023,0.008460800349712371
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,float16,2047,0.01040479987859726
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,fp8,2047,0.009064000099897385
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,float16,4095,0.010411199927330018
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,fp8,4095,0.010367999970912933
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,float16,8191,0.012479999661445617
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,fp8,8191,0.012438400089740754
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,float16,16383,0.016625599563121797
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,fp8,16383,0.01655679941177368
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,float16,32767,0.018611200153827667
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,fp8,31,0.008382400125265121
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,fp8,32767,0.016595199704170227
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,float16,1,0.008478400111198426
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,fp8,1,0.008404800295829773
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,fp8,255,0.008321599662303924
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,float16,3,0.008398400247097015
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,fp8,3,0.00846560001373291
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,float16,7,0.008399999886751174
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,fp8,7,0.008387199789285659
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,float16,2047,0.009388799965381622
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,float16,15,0.008401600271463394
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,fp8,15,0.008414400368928909
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,float16,31,0.00838399976491928
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,float16,63,0.008718399703502655
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,fp8,63,0.008392000198364257
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,float16,127,0.008433599770069123
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,fp8,127,0.008347199857234954
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,float16,255,0.008388800173997879
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,fp8,255,0.008366400003433227
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,float16,511,0.008374399691820144
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,fp8,511,0.008376000076532364
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,float16,1023,0.00841279998421669
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,fp8,1023,0.008406399935483932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,float16,2047,0.010364799946546554
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,fp8,2047,0.008951999992132188
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,float16,4095,0.010463999956846238
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,float16,15,0.008446399867534638
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,fp8,4095,0.010478399693965912
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,float16,8191,0.01281599998474121
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,fp8,8191,0.012520000338554382
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,float16,16383,0.014630399644374847
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,fp8,16383,0.014531199634075165
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,float16,32767,0.01828320026397705
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,fp8,32767,0.015985600650310516
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,float16,1,0.008398400247097015
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,fp8,15,0.008454400300979614
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,fp8,1,0.008395200222730636
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,float16,3,0.008352000266313553
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,fp8,3,0.00840959995985031
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,float16,7,0.00844319984316826
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,fp8,7,0.008387199789285659
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,fp8,255,0.008408000320196151
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,fp8,15,0.00843520015478134
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,float16,31,0.008444800227880477
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,fp8,31,0.00843999981880188
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,float16,63,0.008454400300979614
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,fp8,63,0.00836160033941269
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,fp8,127,0.008433599770069123
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,float16,255,0.00846719965338707
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,float16,511,0.008752000331878663
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,float16,1023,0.008459199965000153
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,fp8,511,0.008460800349712371
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,fp8,1023,0.008404800295829773
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,float16,2047,0.010412800312042236
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,float16,4095,0.010462400317192078
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,fp8,2047,0.008425600081682205
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,fp8,4095,0.01043199971318245
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,float16,8191,0.012478400021791458
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,fp8,8191,0.012532800436019897
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,float16,16383,0.016627199947834015
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,fp8,16383,0.016601599752902985
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,float16,32767,0.01855359971523285
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,fp8,32767,0.016631999611854555
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,float16,1,0.008406399935483932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,fp8,1,0.008382400125265121
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,float16,16383,0.3011744022369385
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,fp8,3,0.008363199979066848
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,float16,7,0.008436799794435502
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,fp8,7,0.008452799916267396
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,float16,15,0.008441600203514098
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,fp8,15,0.008404800295829773
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,fp8,31,0.00843520015478134
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,float16,63,0.008425600081682205
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,fp8,63,0.008401600271463394
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,float16,127,0.008427199721336365
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,fp8,127,0.008404800295829773
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,float16,255,0.008427199721336365
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,fp8,255,0.008406399935483932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,float16,511,0.008406399935483932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,fp8,511,0.008395200222730636
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,fp8,31,0.008404800295829773
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,float16,1023,0.00902239978313446
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,fp8,1023,0.008377599716186523
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,float16,2047,0.010438399761915207
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,float16,4095,0.01043360009789467
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,fp8,4095,0.010441599786281586
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,float16,8191,0.013251200318336487
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,fp8,8191,0.01252640038728714
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,float16,16383,0.016527999937534333
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,fp8,16383,0.01454080045223236
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,float16,32767,0.016599999368190767
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,fp8,32767,0.016200000047683717
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,float16,1,0.008417599648237229
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,fp8,1,0.008399999886751174
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,float16,3,0.008398400247097015
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,fp8,3,0.00841279998421669
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,fp8,255,0.0083856001496315
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,float16,15,0.008392000198364257
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,fp8,15,0.008372800052165985
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,float16,31,0.008396799862384795
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,float16,3,0.008399999886751174
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,fp8,31,0.008423999696969987
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,float16,63,0.008414400368928909
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,fp8,63,0.008347199857234954
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,float16,31,0.008476799726486206
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,float16,127,0.008444800227880477
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,fp8,127,0.008414400368928909
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,float16,255,0.00841120034456253
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,fp8,255,0.00843999981880188
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,fp8,511,0.008454400300979614
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,float16,1023,0.00888800024986267
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,fp8,1023,0.008408000320196151
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,float16,2047,0.010476800054311753
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,fp8,2047,0.0104032002389431
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,float16,4095,0.010505600273609162
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,fp8,4095,0.010412800312042236
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,float16,8191,0.013041600584983826
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,fp8,2047,0.009372799843549728
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,fp8,8191,0.012513600289821625
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,float16,16383,0.015254400670528412
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,fp8,16383,0.014483200013637542
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,float16,32767,0.031121599674224853
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,fp8,32767,0.016624000668525696
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,float16,1,0.008380799740552902
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,fp8,1,0.008323200047016144
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,float16,3,0.00838399976491928
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,fp8,3,0.008337599784135818
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,float16,7,0.008395200222730636
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,fp8,7,0.00835679993033409
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,float16,15,0.008403199911117553
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,fp8,15,0.00833280012011528
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,float16,31,0.00841279998421669
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,float16,15,0.008350399881601333
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,float16,63,0.008369600027799606
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,float16,127,0.008380799740552902
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,float16,127,0.008401600271463394
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,fp8,127,0.008388800173997879
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,float16,255,0.008392000198364257
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,fp8,7,0.008425600081682205
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,fp8,255,0.00841279998421669
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,float16,511,0.008398400247097015
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,fp8,511,0.008388800173997879
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,float16,1023,0.008737599849700928
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,fp8,1023,0.008428800106048583
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,float16,2047,0.010388799756765366
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,fp8,2047,0.010367999970912933
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,float16,4095,0.012451200187206269
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,float16,8191,0.028270399570465087
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,float16,16383,0.04679679870605469
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,fp8,8191,0.014523200690746307
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,fp8,16383,0.0288783997297287
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,float16,32767,0.08203840255737305
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,fp8,32767,0.047244799137115476
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,float16,1,0.00841279998421669
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,fp8,1,0.0083856001496315
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,float16,3,0.008420799672603608
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,fp8,3,0.008392000198364257
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,float16,7,0.00843999981880188
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,float16,15,0.008340799808502197
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,fp8,7,0.00838399976491928
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,fp8,15,0.00835999995470047
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,float16,31,0.008376000076532364
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,float16,63,0.008473599702119828
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,float16,511,0.008432000130414962
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,fp8,63,0.008433599770069123
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,float16,127,0.008425600081682205
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,fp8,127,0.008393599838018417
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,float16,255,0.008441600203514098
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,fp8,255,0.008416000008583068
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,float16,511,0.008523199707269669
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,fp8,511,0.008449599891901017
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,float16,1023,0.010463999956846238
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,fp8,1023,0.009665600210428237
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,float16,2047,0.012520000338554382
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,fp8,2047,0.01045600026845932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,float16,4095,0.028307199478149414
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,fp8,4095,0.013209599256515502
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,float16,8191,0.04563199877738953
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,fp8,8191,0.028856000304222106
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,float16,16383,0.08194400072097778
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,fp8,16383,0.04710240066051483
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,fp8,32767,0.08272160291671753
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,float16,32767,0.15368959903717042
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,fp8,1,0.008320000022649765
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,float16,3,0.008340799808502197
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,fp8,3,0.008318399637937545
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,float16,7,0.008369600027799606
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,fp8,63,0.008422400057315826
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,fp8,7,0.008350399881601333
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,float16,15,0.008382400125265121
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,float16,7,0.008713600039482117
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,float16,31,0.008392000198364257
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,fp8,31,0.008347199857234954
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,float16,63,0.00833280012011528
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,fp8,63,0.008348800241947174
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,float16,127,0.008364800363779068
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,fp8,4095,0.010539200156927109
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,fp8,255,0.008369600027799606
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,float16,511,0.010391999781131745
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,fp8,511,0.008388800173997879
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,float16,1023,0.01231200024485588
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,fp8,1023,0.010382399708032609
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,float16,2047,0.027726399898529052
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,fp8,2047,0.012505599856376648
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,float16,4095,0.04528320133686066
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,fp8,4095,0.02736319899559021
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,fp8,31,0.00835679993033409
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,float16,8191,0.08193920254707336
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,fp8,8191,0.04588479995727539
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,fp8,16383,0.08379359841346741
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,float16,16383,0.15374720096588135
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,fp8,32767,0.15550240278244018
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,float16,32767,0.2978111982345581
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,float16,1,0.00843840017914772
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,fp8,1,0.008408000320196151
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,float16,3,0.008382400125265121
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,fp8,3,0.008406399935483932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,float16,7,0.008441600203514098
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,fp8,7,0.008313599973917007
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,float16,15,0.008446399867534638
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,fp8,15,0.008414400368928909
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,float16,31,0.00880960002541542
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,float16,1,0.008406399935483932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,float16,63,0.008404800295829773
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,fp8,63,0.008406399935483932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,float16,127,0.008419200032949447
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,fp8,127,0.008387199789285659
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,fp8,31,0.008348800241947174
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,float16,255,0.009259200096130371
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,fp8,255,0.008697599917650223
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,float16,511,0.008448000252246856
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,float16,1023,0.008705600351095199
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,float16,2047,0.01045600026845932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,fp8,1023,0.008451200276613235
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,fp8,15,0.008342400193214417
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,fp8,2047,0.009041599929332733
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,float16,4095,0.010465600341558457
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,fp8,4095,0.01037440001964569
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,float16,8191,0.01279519945383072
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,fp8,8191,0.012534399330615998
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,float16,16383,0.015118399262428283
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,float16,32767,0.018292799592018127
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,fp8,127,0.008369600027799606
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,fp8,32767,0.016489599645137788
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,fp8,1,0.008364800363779068
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,float16,3,0.008417599648237229
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,float16,7,0.008454400300979614
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,fp8,7,0.008420799672603608
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,float16,15,0.00840959995985031
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,fp8,15,0.008425600081682205
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,float16,31,0.008408000320196151
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,fp8,31,0.008324799686670303
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,float16,63,0.008427199721336365
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,fp8,63,0.008419200032949447
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,float16,127,0.008459199965000153
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,fp8,127,0.008428800106048583
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,float16,255,0.008665599673986436
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,fp8,255,0.008414400368928909
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,float16,511,0.008449599891901017
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,fp8,511,0.008423999696969987
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,float16,1023,0.009369599819183349
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,fp8,1023,0.008387199789285659
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,float16,2047,0.010390400141477584
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,fp8,2047,0.010465600341558457
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,float16,4095,0.010985600203275681
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,fp8,4095,0.010468800365924836
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,float16,8191,0.012492799758911132
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,float16,16383,0.01467359960079193
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,fp8,16383,0.013152000308036805
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,float16,32767,0.031439998745918275
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,fp8,32767,0.016484799981117248
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,float16,1,0.008459199965000153
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,fp8,31,0.008403199911117553
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,fp8,1,0.008420799672603608
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,float16,3,0.008432000130414962
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,fp8,3,0.008459199965000153
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,float16,7,0.008843199908733368
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,fp8,7,0.008441600203514098
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,fp8,511,0.008918400108814239
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,float16,15,0.008833599835634231
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,fp8,15,0.008455999940633774
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,float16,31,0.008486399799585343
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,fp8,31,0.008427199721336365
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,float16,63,0.008479999750852585
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,fp8,63,0.008427199721336365
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,fp8,127,0.00843520015478134
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,float16,255,0.008449599891901017
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,fp8,255,0.00841120034456253
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,float16,511,0.008419200032949447
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,fp8,511,0.008404800295829773
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,float16,255,0.008420799672603608
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,float16,1,0.008425600081682205
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,fp8,1023,0.008376000076532364
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,float16,2047,0.010427200049161912
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,fp8,2047,0.010380800068378448
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,fp8,4095,0.010443200170993806
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,fp8,3,0.008451200276613235
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,fp8,8191,0.01249919980764389
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,float16,16383,0.030353599786758424
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,fp8,16383,0.015334400534629821
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,float16,32767,0.048151999711990356
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,fp8,32767,0.030638399720191955
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,float16,1,0.008366400003433227
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,fp8,1,0.00835999995470047
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,float16,3,0.008436799794435502
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,fp8,3,0.008355200290679932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,float16,7,0.00833439975976944
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,fp8,7,0.008390399813652038
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,float16,15,0.008427199721336365
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,fp8,15,0.008363199979066848
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,float16,31,0.008414400368928909
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,fp8,31,0.008352000266313553
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,fp8,8191,0.012439999729394913
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,float16,63,0.008340799808502197
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,fp8,63,0.008320000022649765
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,float16,127,0.008449599891901017
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,fp8,127,0.00830880030989647
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,float16,255,0.008463999629020691
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,fp8,255,0.008315200358629227
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,float16,511,0.009558399766683578
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,fp8,511,0.008423999696969987
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,float16,1023,0.011121600121259689
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,fp8,1023,0.010367999970912933
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,float16,2047,0.027009600400924684
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,fp8,2047,0.012520000338554382
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,float16,4095,0.04533919990062714
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,float16,127,0.008416000008583068
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,fp8,4095,0.0280815988779068
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,fp8,16383,0.014535999298095703
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,float16,8191,0.08132479786872863
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,float16,1023,0.009254399687051773
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,float16,16383,0.15341440439224244
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,float16,1,0.009600000083446502
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,fp8,1,0.008403199911117553
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,fp8,16383,0.08345760107040405
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,fp8,3,0.008460800349712371
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,float16,7,0.0104032002389431
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,float16,15,0.010364799946546554
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,fp8,7,0.008388800173997879
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,fp8,15,0.00841120034456253
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,float16,31,0.010420800000429154
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,fp8,31,0.008463999629020691
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,float16,63,0.010422399640083313
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,fp8,63,0.008991999924182892
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,fp8,127,0.008721599727869034
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,fp8,255,0.00889120027422905
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,float16,255,0.010387200117111205
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,float16,511,0.012457600235939026
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,fp8,511,0.010411199927330018
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,float16,1023,0.02728480100631714
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,fp8,1023,0.013060800731182098
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,float16,2047,0.04567840099334717
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,fp8,2047,0.028854399919509888
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,float16,4095,0.082014399766922
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,fp8,4095,0.0472927987575531
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,fp8,8191,0.0848464012145996
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,float16,8191,0.15386559963226318
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,fp8,16383,0.1599984049797058
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,float16,16383,0.29959518909454347
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,float16,1,0.012492799758911132
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,float16,3,0.012443199753761292
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,fp8,1,0.011224000155925751
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,float16,7,0.012467200309038163
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,fp8,7,0.010627199709415436
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,float16,15,0.012518399953842163
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,float16,31,0.012476799637079239
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,fp8,31,0.010667199641466141
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,float16,63,0.012462399899959564
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,fp8,63,0.01101600006222725
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,float16,127,0.012503999471664428
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,fp8,127,0.010548800230026245
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,float16,255,0.012492799758911132
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,fp8,8191,0.04646719992160797
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,float16,511,0.02892799973487854
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,float16,4095,0.0125231996178627
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,fp8,511,0.014567999541759491
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,float16,8191,0.014571200311183929
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,float16,1023,0.04701760113239288
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,float16,2047,0.08362879753112792
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,fp8,2047,0.04723680019378662
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,float16,127,0.010440000146627427
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,fp8,4095,0.08417919874191285
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,float16,4095,0.15818560123443604
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,float16,8191,0.3052592039108276
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,fp8,8191,0.15767040252685546
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,float16,1,0.009967999905347824
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,fp8,1,0.0083856001496315
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,float16,3,0.010273600369691849
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,fp8,3,0.008425600081682205
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,float16,7,0.010353600233793258
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,float16,16383,0.6099135875701904
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,fp8,16383,0.3044640064239502
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,fp8,7,0.008419200032949447
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,float16,31,0.010353600233793258
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,fp8,31,0.008508799970149994
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,float16,15,0.010398399829864503
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,fp8,15,0.00844319984316826
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,float16,63,0.01037919968366623
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,float16,127,0.010470400005578995
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,fp8,127,0.008455999940633774
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,fp8,3,0.011401599645614624
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,float16,255,0.010443200170993806
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,fp8,255,0.008441600203514098
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,float16,511,0.012495999783277511
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,fp8,511,0.010476800054311753
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,float16,1023,0.02744320034980774
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,fp8,15,0.011191999912261963
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,fp8,1023,0.014535999298095703
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,float16,2047,0.04731999933719635
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,fp8,2047,0.02881920039653778
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,float16,1,0.012462399899959564
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,fp8,4095,0.04765760004520416
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,fp8,1,0.01149120032787323
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,float16,3,0.012491200119256973
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,fp8,3,0.010503999888896942
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,float16,7,0.012486399710178375
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,fp8,7,0.010467199981212616
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,float16,15,0.01247519999742508
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,fp8,15,0.01045759990811348
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,float16,31,0.012515200674533844
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,fp8,255,0.010527999699115753
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,fp8,31,0.010460799932479859
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,float16,63,0.012537600100040435
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,float16,127,0.012545600533485413
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,fp8,63,0.010438399761915207
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,fp8,127,0.010465600341558457
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,float16,255,0.012561599910259246
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,fp8,255,0.010480000078678131
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,fp8,1023,0.029216000437736513
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,fp8,511,0.014585599303245544
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,float16,1023,0.046184000372886655
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,float16,2047,0.0836239993572235
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,fp8,2047,0.0473008006811142
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,float16,4095,0.157532799243927
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,fp8,4095,0.08481600284576415
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,float16,1,0.016633599996566772
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,fp8,1,0.014612799882888794
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,float16,3,0.016569599509239197
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,fp8,3,0.0149167999625206
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,float16,7,0.016575999557971954
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,fp8,7,0.01531360000371933
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,float16,15,0.01663679927587509
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,fp8,15,0.014655999839305878
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,float16,31,0.016624000668525696
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,fp8,31,0.015564799308776855
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,float16,63,0.016631999611854555
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,float16,127,0.017268800735473634
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,fp8,63,0.015385599434375763
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,fp8,127,0.01462399959564209
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,float16,255,0.02813439965248108
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,fp8,255,0.015041600167751312
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,float16,511,0.04567520022392273
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,fp8,511,0.02892960011959076
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,fp8,1023,0.04849439859390259
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,float16,1023,0.08279359936714173
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,fp8,63,0.008747199922800064
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,fp8,2047,0.08510559797286987
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,float16,2047,0.15610719919204713
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,float16,1,0.012427199631929398
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,float16,4095,0.30237278938293455
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,fp8,4095,0.159825599193573
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,float16,4095,0.0820527970790863
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,float16,3,0.012465599924325943
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,fp8,3,0.01249919980764389
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,float16,7,0.012518399953842163
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,float16,15,0.01250240057706833
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,fp8,15,0.012380799651145935
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,float16,31,0.012540799379348756
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,fp8,31,0.011881600320339202
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,float16,63,0.012540799379348756
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,fp8,63,0.011479999870061874
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,fp8,127,0.012187200039625168
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,float16,255,0.012478400021791458
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,float16,3,0.010382399708032609
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,fp8,255,0.011601600050926208
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,float16,511,0.02879199981689453
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,fp8,1023,0.02890079915523529
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,fp8,511,0.014582400023937226
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,fp8,1023,0.028908801078796387
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,float16,1023,0.047307199239730834
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,float16,2047,0.08407679796218873
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,float16,1,0.016622400283813475
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,fp8,2047,0.04766879975795746
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,fp8,1,0.014529600739479065
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,float16,3,0.016596800088882445
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,float16,7,0.017662400007247926
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,fp8,3,0.014567999541759491
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,fp8,7,0.014556799829006196
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,float16,15,0.017691199481487275
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,fp8,15,0.014681600034236908
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,float16,31,0.01656160056591034
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,fp8,31,0.016467200219631196
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,float16,63,0.016616000235080718
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,float16,127,0.017161600291728973
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,fp8,63,0.015726399421691895
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,fp8,255,0.014547200500965118
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,float16,255,0.027603200078010558
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,float16,511,0.046172800660133365
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,fp8,511,0.028944000601768494
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,float16,1023,0.08206719756126404
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,fp8,1023,0.049262401461601255
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,float16,2047,0.15544799566268921
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,fp8,2047,0.08664320111274719
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,float16,1,0.02690880000591278
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,fp8,1,0.02431679964065552
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,fp8,3,0.022889600694179536
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,float16,7,0.026873600482940675
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,fp8,7,0.022894400358200073
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,float16,15,0.026862400770187377
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,fp8,15,0.023367999494075774
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,fp8,1,0.012455999851226807
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,float16,31,0.02685759961605072
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,fp8,31,0.024556800723075867
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,float16,63,0.027460798621177673
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,fp8,7,0.011559999734163284
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,float16,127,0.030534398555755616
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,fp8,127,0.024135999381542206
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,float16,255,0.047044798731803894
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,fp8,255,0.029094401001930236
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,float16,127,0.012507200241088867
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,float16,511,0.08304160237312316
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,fp8,511,0.04785279929637909
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,float16,511,0.028046399354934692
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,fp8,1023,0.08479679822921753
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,float16,1023,0.15624959468841554
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,float16,1,0.00843840017914772
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,fp8,1,0.008374399691820144
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,float16,3,0.008404800295829773
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,fp8,3,0.008444800227880477
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,float16,2047,0.30266239643096926
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,float16,7,0.008448000252246856
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,fp8,2047,0.15809919834136962
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,fp8,7,0.00838399976491928
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,fp8,15,0.00843520015478134
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,fp8,31,0.00841279998421669
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,float16,63,0.00841279998421669
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,float16,31,0.008396799862384795
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,fp8,63,0.008420799672603608
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,fp8,127,0.008455999940633774
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,float16,255,0.008367999643087386
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,fp8,255,0.008417599648237229
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,float16,511,0.008491200208663941
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,fp8,511,0.008399999886751174
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,fp8,127,0.016487999260425566
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,float16,1023,0.00952960029244423
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,fp8,1023,0.008367999643087386
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,float16,2047,0.01043040007352829
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,fp8,2047,0.010420800000429154
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,float16,4095,0.01167839989066124
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,float16,8191,0.013782399892807006
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,fp8,8191,0.012411200255155564
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,float16,16383,0.016476799547672272
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,fp8,16383,0.014457599818706512
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,float16,32767,0.030875200033187868
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,float16,1,0.008640000224113464
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,fp8,1,0.008393599838018417
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,float16,3,0.008366400003433227
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,float16,3,0.027036800980567932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,fp8,3,0.008371199667453765
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,float16,7,0.008408000320196151
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,fp8,7,0.008408000320196151
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,float16,15,0.008372800052165985
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,fp8,15,0.008398400247097015
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,float16,31,0.008441600203514098
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,fp8,31,0.008374399691820144
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,float16,63,0.008432000130414962
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,fp8,63,0.008369600027799606
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,float16,127,0.008406399935483932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,fp8,127,0.008422400057315826
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,float16,255,0.008617600053548813
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,fp8,255,0.008355200290679932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,float16,511,0.009017600119113922
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,fp8,511,0.008462399989366532
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,fp8,63,0.023102399706840516
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,float16,2047,0.010414399951696397
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,fp8,2047,0.010385599732398988
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,float16,4095,0.012455999851226807
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,float16,8191,0.014593599736690522
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,fp8,8191,0.012459199875593185
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,float16,16383,0.030616000294685364
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,fp8,16383,0.016417600214481354
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,float16,32767,0.047443199157714847
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,fp8,32767,0.030932798981666565
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,float16,1,0.00835679993033409
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,fp8,1,0.008401600271463394
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,float16,15,0.00843520015478134
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,float16,3,0.008363199979066848
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,float16,127,0.00843840017914772
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,fp8,3,0.008324799686670303
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,float16,7,0.00835999995470047
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,float16,15,0.00835679993033409
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,fp8,7,0.008403199911117553
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,float16,31,0.008327999711036682
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,fp8,31,0.008414400368928909
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,fp8,15,0.00833439975976944
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,float16,63,0.00846880003809929
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,float16,127,0.00833280012011528
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,fp8,127,0.008364800363779068
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,float16,255,0.00844319984316826
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,fp8,4095,0.010440000146627427
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,fp8,255,0.008345600217580795
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,float16,511,0.008427199721336365
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,fp8,511,0.008352000266313553
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,float16,1023,0.009030400216579438
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,fp8,1023,0.008352000266313553
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,float16,2047,0.010460799932479859
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,fp8,2047,0.008420799672603608
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,fp8,4095,0.01048320010304451
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,float16,8191,0.028521600365638732
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,fp8,8191,0.014481599628925323
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,float16,16383,0.045737600326538085
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,fp8,16383,0.028838399052619933
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,float16,32767,0.08258240222930908
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,fp8,32767,0.04740320146083832
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,float16,1,0.01693120002746582
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,float16,1023,0.009348800033330917
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,fp8,1,0.016595199704170227
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,fp8,1023,0.008500800281763077
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,fp8,3,0.016607999801635742
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,float16,7,0.01786399930715561
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,fp8,7,0.016519999504089354
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,float16,15,0.017683200538158417
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,fp8,15,0.016564799845218657
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,fp8,4095,0.0105103999376297
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,fp8,31,0.016539199650287627
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,float16,63,0.01717280000448227
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,fp8,63,0.016499200463294984
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,float16,127,0.017340800166130065
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,fp8,127,0.016519999504089354
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,float16,255,0.028638398647308348
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,fp8,255,0.016518400609493257
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,float16,511,0.04729120135307312
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,fp8,511,0.02909280061721802
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,float16,1023,0.08387839794158936
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,float16,1,0.027161601185798644
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,fp8,1,0.0247296005487442
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,float16,3,0.02720159888267517
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,fp8,3,0.024758400022983552
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,float16,7,0.02688640058040619
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,fp8,7,0.02476159930229187
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,float16,15,0.026819199323654175
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,fp8,63,0.008312000334262848
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,fp8,15,0.024728000164031982
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,float16,31,0.02727360129356384
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,fp8,31,0.024784000217914583
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,float16,63,0.02828640043735504
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,fp8,63,0.024742400646209715
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,fp8,32767,0.016516800224781036
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,fp8,127,0.02476159930229187
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,float16,127,0.030865600705146788
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,float16,4095,0.01252480000257492
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,float16,255,0.04710879921913147
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,fp8,255,0.031031998991966247
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,fp8,511,0.04884800016880035
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,fp8,1023,0.08614879846572876
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,float16,1,0.04737119972705841
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,fp8,1,0.041206398606300355
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,float16,3,0.04822399914264679
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,fp8,3,0.04112800061702728
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,float16,7,0.047332799434661864
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,fp8,7,0.041131201386451724
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,float16,15,0.04738720059394837
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,float16,31,0.04743199944496155
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,fp8,31,0.041115200519561766
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,float16,63,0.049291199445724486
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,float16,31,0.016689600050449373
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,fp8,63,0.041196799278259276
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,float16,127,0.05141119956970215
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,fp8,127,0.04302560091018677
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,float16,255,0.08392959833145142
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,fp8,255,0.04976800084114075
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,fp8,1023,0.04935680031776428
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,float16,511,0.15613119602203368
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,fp8,511,0.0858128011226654
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,float16,1,0.02774080038070679
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,fp8,1,0.024772800505161285
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,float16,1023,0.30480639934539794
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,fp8,3,0.02595840096473694
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,float16,7,0.027777600288391113
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,fp8,7,0.02481919974088669
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,float16,15,0.02773280143737793
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,fp8,15,0.024817599356174468
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,float16,31,0.028262400627136232
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,fp8,31,0.02484800070524216
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,float16,63,0.0281792014837265
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,fp8,63,0.024823999404907225
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,float16,127,0.030931198596954347
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,fp8,127,0.02524000108242035
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,float16,255,0.04739840030670166
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,float16,511,0.08345760107040405
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,float16,511,0.08458399772644043
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,fp8,511,0.049644801020622256
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,float16,1,0.04741599857807159
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,float16,3,0.04731520116329193
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,fp8,3,0.04124319851398468
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,fp8,15,0.04118399918079376
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,float16,7,0.04734239876270294
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,fp8,7,0.04118399918079376
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,float16,15,0.047513601183891294
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,fp8,15,0.04126240015029907
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,float16,31,0.04835039973258972
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,fp8,31,0.04114400148391724
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,float16,63,0.049246400594711304
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,fp8,63,0.04126879870891571
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,float16,127,0.05133119821548462
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,fp8,127,0.04345600008964538
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,float16,255,0.08437439799308777
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,fp8,255,0.05190879702568054
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,float16,3,0.028596800565719605
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,float16,511,0.15667999982833863
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,fp8,511,0.08749120235443116
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,float16,1,0.08691679835319518
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,fp8,1,0.07398719787597656
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,float16,3,0.08675680160522461
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,fp8,3,0.07401760220527649
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,float16,7,0.08680959939956664
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,fp8,7,0.07396320104599
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,float16,15,0.08733760118484497
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,fp8,15,0.07398719787597656
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,float16,31,0.08801760077476502
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,fp8,31,0.07455199956893921
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,float16,1023,0.15518239736557007
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,float16,63,0.08836640119552612
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,fp8,63,0.075382399559021
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,fp8,1,0.04120959937572479
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,float16,127,0.09389280080795288
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,fp8,127,0.07610719799995422
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,float16,255,0.15881119966506957
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,fp8,255,0.091702401638031
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,float16,1,0.008441600203514098
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,fp8,1,0.008432000130414962
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,float16,3,0.008417599648237229
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,fp8,3,0.008320000022649765
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,float16,7,0.008455999940633774
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,fp8,511,0.16084959506988525
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,fp8,7,0.008363199979066848
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,float16,15,0.008459199965000153
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,float16,31,0.008414400368928909
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,fp8,31,0.008406399935483932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,fp8,15,0.008419200032949447
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,float16,63,0.008406399935483932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,fp8,63,0.008486399799585343
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,float16,127,0.008399999886751174
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,fp8,127,0.00835840031504631
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,float16,255,0.008427199721336365
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,float16,511,0.00867680013179779
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,fp8,511,0.008449599891901017
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,float16,1023,0.010388799756765366
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,fp8,1023,0.008481600135564805
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,float16,2047,0.010505600273609162
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,fp8,2047,0.01043360009789467
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,float16,4095,0.012531200051307678
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,fp8,4095,0.010476800054311753
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,float16,8191,0.014611199498176575
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,fp8,8191,0.01255359947681427
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,float16,16383,0.029631999135017396
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,fp8,16383,0.014787200093269347
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,float16,32767,0.04909439980983734
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,float16,1,0.008416000008583068
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,fp8,1023,0.15879199504852295
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,fp8,1,0.008340799808502197
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,fp8,3,0.008377599716186523
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,float16,3,0.008399999886751174
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,float16,7,0.00835679993033409
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,fp8,7,0.008339200168848038
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,float16,15,0.008353599905967712
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,fp8,15,0.008369600027799606
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,float16,31,0.008380799740552902
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,fp8,31,0.00833119973540306
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,float16,63,0.00841120034456253
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,fp8,63,0.00838399976491928
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,float16,127,0.008367999643087386
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,fp8,127,0.008422400057315826
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,float16,255,0.008380799740552902
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,fp8,255,0.008364800363779068
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,float16,511,0.008390399813652038
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,fp8,511,0.008343999832868576
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,float16,1023,0.009188800305128097
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,fp8,1023,0.008403199911117553
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,float16,2047,0.010531199723482132
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,fp8,2047,0.01043040007352829
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,float16,4095,0.012483199685811996
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,fp8,4095,0.010503999888896942
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,float16,8191,0.02736000120639801
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,fp8,255,0.031086400151252747
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,float16,16383,0.04556800127029419
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,fp8,16383,0.028832000494003297
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,float16,32767,0.08214240074157715
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,float16,1,0.008793599903583527
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,fp8,32767,0.04717119932174683
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,fp8,1,0.00835840031504631
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,fp8,3,0.00843520015478134
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,float16,7,0.008414400368928909
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,fp8,7,0.008396799862384795
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,float16,15,0.008452799916267396
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,fp8,15,0.0083856001496315
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,float16,31,0.008448000252246856
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,fp8,31,0.008406399935483932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,float16,63,0.008452799916267396
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,fp8,63,0.008406399935483932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,float16,127,0.00841120034456253
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,float16,3,0.01661120057106018
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,fp8,127,0.008350399881601333
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,float16,255,0.008433599770069123
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,fp8,255,0.00841120034456253
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,float16,511,0.008420799672603608
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,fp8,511,0.008419200032949447
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,float16,1023,0.010363200306892395
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,fp8,1023,0.008448000252246856
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,float16,2047,0.012425599992275238
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,fp8,2047,0.010467199981212616
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,fp8,255,0.008446399867534638
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,float16,4095,0.027657601237297057
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,fp8,4095,0.01255359947681427
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,float16,8191,0.04553439915180206
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,fp8,8191,0.02871040105819702
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,float16,16383,0.08214719891548157
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,fp8,16383,0.046823999285697936
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,float16,32767,0.15350719690322875
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,fp8,32767,0.0849183976650238
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,3,0.008428800106048583
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,1,0.008432000130414962
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,3,0.00835999995470047
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,fp8,32767,0.030700799822807313
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,7,0.008428800106048583
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,15,0.008454400300979614
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,15,0.008451200276613235
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,31,0.008420799672603608
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,31,0.008342400193214417
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,63,0.008455999940633774
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,63,0.008347199857234954
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,127,0.008489599823951722
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,127,0.00835840031504631
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,255,0.008367999643087386
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,255,0.00843520015478134
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,511,0.008830399811267852
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,1023,0.010401599854230881
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,1023,0.009700799733400345
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,2047,0.012518399953842163
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,2047,0.010396800190210342
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,4095,0.027804800868034364
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,fp8,8191,0.01316000074148178
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,4095,0.012665599584579468
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,8191,0.02879680097103119
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,float16,3,0.008460800349712371
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,16383,0.08155840039253234
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,16383,0.04666399955749512
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,32767,0.15300960540771485
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,32767,0.0829967975616455
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,float16,511,0.30409600734710696
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,1,0.008342400193214417
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,65535,0.2967184066772461
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,1,0.008398400247097015
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,65535,0.15486079454421997
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,3,0.008310399949550629
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,3,0.008350399881601333
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,7,0.008292800188064576
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,7,0.008367999643087386
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,15,0.008340799808502197
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,31,0.008323200047016144
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,63,0.008339200168848038
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,31,0.00836160033941269
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,63,0.008318399637937545
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,127,0.008352000266313553
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,1,0.00846719965338707
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,255,0.008353599905967712
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,255,0.008369600027799606
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,511,0.010435199737548828
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,1023,0.011022400110960007
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,1023,0.010471999645233154
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,2047,0.02696320116519928
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,7,0.008401600271463394
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,2047,0.01255040019750595
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,4095,0.04525760114192963
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,4095,0.027260801196098326
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,8191,0.04624640047550201
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,8191,0.08094239830970765
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,16383,0.152457594871521
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,16383,0.08348960280418397
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,511,0.008427199721336365
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,32767,0.29639520645141604
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,32767,0.15653120279312133
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,1,0.008414400368928909
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,1,0.008406399935483932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,3,0.008382400125265121
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,3,0.008340799808502197
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,7,0.008423999696969987
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,65535,0.582371187210083
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,15,0.008425600081682205
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,65535,0.3125279903411865
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,31,0.00841120034456253
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,63,0.008393599838018417
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,15,0.008382400125265121
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,31,0.008326400071382523
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,63,0.008427199721336365
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,127,0.008430399745702744
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,127,0.008432000130414962
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,255,0.008428800106048583
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,511,0.008419200032949447
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,1023,0.008747199922800064
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,15,0.008352000266313553
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,1023,0.008436799794435502
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,127,0.008327999711036682
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,2047,0.010416000336408614
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,2047,0.008366400003433227
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,4095,0.010468800365924836
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,4095,0.00939679965376854
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,8191,0.012484800070524216
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,8191,0.011400000005960465
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,16383,0.016673600673675536
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,16383,0.015345600247383118
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,32767,0.01860959976911545
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,32767,0.016543999314308167
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,65535,0.019756799936294554
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,1,0.00846880003809929
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,65535,0.016624000668525696
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,3,0.008444800227880477
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,3,0.008428800106048583
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,7,0.008371199667453765
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,7,0.008343999832868576
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,15,0.008428800106048583
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,15,0.00843999981880188
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,31,0.00843999981880188
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,31,0.00835840031504631
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,63,0.008470399677753449
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,8191,0.04565599858760834
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,127,0.008423999696969987
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,127,0.00835999995470047
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,255,0.008428800106048583
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,255,0.008436799794435502
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,255,0.008345600217580795
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,511,0.008489599823951722
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,511,0.008433599770069123
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,511,0.008444800227880477
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,2047,0.009867200255393982
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,2047,0.008684799820184708
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,4095,0.010487999767065048
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,4095,0.01045759990811348
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,8191,0.012403199821710587
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,8191,0.012441600114107132
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,16383,0.016568000614643096
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,16383,0.016606399416923524
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,32767,0.01785759925842285
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,32767,0.016624000668525696
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,65535,0.020230400562286376
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,511,0.008396799862384795
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,65535,0.018592000007629395
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,1,0.00862559974193573
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,1,0.008337599784135818
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,3,0.00844319984316826
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,3,0.008441600203514098
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,7,0.008432000130414962
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,7,0.008408000320196151
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,15,0.008380799740552902
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,15,0.008396799862384795
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,31,0.008433599770069123
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,31,0.008408000320196151
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,63,0.008433599770069123
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,1,0.008420799672603608
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,127,0.008425600081682205
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,255,0.00846719965338707
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,255,0.008577600121498108
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,511,0.008414400368928909
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,511,0.008417599648237229
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,1023,0.008432000130414962
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,2047,0.010367999970912933
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,7,0.008399999886751174
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,2047,0.008425600081682205
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,4095,0.010438399761915207
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,4095,0.010351999849081039
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,8191,0.012513600289821625
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,8191,0.012414400279521943
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,16383,0.01656319946050644
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,32767,0.01812800019979477
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,16383,0.016531200706958772
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,32767,0.016627199947834015
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,65535,0.020427200198173522
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,65535,0.018622399866580965
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,1,0.008417599648237229
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,1,0.008406399935483932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,1023,0.009335999935865402
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,3,0.00841120034456253
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,7,0.008414400368928909
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,7,0.008387199789285659
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,15,0.00841279998421669
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,15,0.008441600203514098
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,31,0.008425600081682205
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,31,0.008377599716186523
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,63,0.009328000247478485
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,63,0.008414400368928909
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,127,0.008528000116348267
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,127,0.00838399976491928
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,255,0.008528000116348267
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,255,0.008396799862384795
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,511,0.008401600271463394
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,1023,0.00843840017914772
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,1023,0.008403199911117553
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,2047,0.010361599922180175
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,2047,0.008881600201129913
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,4095,0.01045759990811348
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,63,0.008318399637937545
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,4095,0.010384000092744827
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,127,0.008462399989366532
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,8191,0.012544000148773193
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,16383,0.014804799854755402
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,16383,0.014507199823856353
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,32767,0.018459199368953703
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,32767,0.01659200042486191
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,1023,0.008470399677753449
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,65535,0.03488959968090057
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,65535,0.018617600202560425
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,1,0.008347199857234954
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,3,0.008336000144481659
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,3,0.008379200100898742
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,7,0.008395200222730636
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,7,0.008367999643087386
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,15,0.00835679993033409
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,15,0.008372800052165985
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,31,0.008340799808502197
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,3,0.008455999940633774
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,1023,0.008430399745702744
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,31,0.00836160033941269
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,63,0.008343999832868576
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,127,0.00838399976491928
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,127,0.00843520015478134
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,255,0.008396799862384795
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,255,0.008423999696969987
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,511,0.008392000198364257
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,511,0.008320000022649765
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,1023,0.008801600337028504
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,511,0.008393599838018417
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,1023,0.008363199979066848
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,2047,0.010446400195360184
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,4095,0.01255040019750595
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,4095,0.010627199709415436
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,8191,0.028035199642181395
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,8191,0.014535999298095703
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,16383,0.045326399803161624
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,16383,0.02885279953479767
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,32767,0.04731839895248413
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,65535,0.15333919525146483
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,65535,0.08440319895744323
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,8191,0.012510399520397186
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,1,0.008380799740552902
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,1,0.008350399881601333
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,3,0.008459199965000153
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,3,0.008420799672603608
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,7,0.008408000320196151
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,7,0.0083856001496315
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,15,0.008377599716186523
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,15,0.008336000144481659
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,31,0.008420799672603608
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,1,0.008423999696969987
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,63,0.00841279998421669
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,63,0.008380799740552902
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,127,0.008390399813652038
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,127,0.008455999940633774
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,255,0.008395200222730636
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,255,0.008451200276613235
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,511,0.00846880003809929
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,511,0.008419200032949447
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,1023,0.010412800312042236
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,1023,0.008396799862384795
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,2047,0.01043199971318245
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,2047,0.012460800260305405
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,4095,0.012908799946308136
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,4095,0.027670401334762573
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,8191,0.04532159864902496
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,16383,0.08202880024909973
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,16383,0.04652799963951111
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,63,0.008376000076532364
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,32767,0.1536080002784729
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,32767,0.08292319774627685
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,1,0.008537600189447403
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,1,0.008367999643087386
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,3,0.00840959995985031
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,65535,0.1570016026496887
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,65535,0.29613280296325684
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,2047,0.010436800122261048
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,7,0.008433599770069123
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,7,0.008420799672603608
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,15,0.008423999696969987
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,31,0.008420799672603608
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,63,0.008408000320196151
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,63,0.008404800295829773
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,127,0.008425600081682205
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,32767,0.08170880079269409
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,127,0.008321599662303924
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,255,0.008432000130414962
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,255,0.008420799672603608
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,511,0.008451200276613235
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,511,0.008401600271463394
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,1023,0.008454400300979614
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,1023,0.008367999643087386
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,2047,0.010364799946546554
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,2047,0.009536000341176987
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,4095,0.010502400249242783
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,4095,0.010369600355625152
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,8191,0.01449279934167862
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,8191,0.012488000094890594
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,16383,0.016574400663375854
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,16383,0.014692799746990204
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,32767,0.018615999817848207
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,32767,0.01653759926557541
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,31,0.008419200032949447
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,65535,0.034923198819160464
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,65535,0.018347199261188506
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,1,0.008428800106048583
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,63,0.008350399881601333
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,1,0.008417599648237229
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,3,0.008417599648237229
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,3,0.008353599905967712
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,7,0.008462399989366532
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,15,0.008470399677753449
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,15,0.008472000062465668
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,31,0.008473599702119828
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,31,0.00835999995470047
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,63,0.008452799916267396
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,8191,0.028420799970626832
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,63,0.008423999696969987
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,127,0.008462399989366532
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,127,0.008377599716186523
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,255,0.00835679993033409
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,511,0.008419200032949447
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,511,0.008367999643087386
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,1023,0.008396799862384795
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,1023,0.008382400125265121
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,2047,0.010473600029945374
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,2047,0.009788800030946732
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,4095,0.010731200128793717
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,4095,0.010500799864530563
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,8191,0.012608000636100769
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,8191,0.012465599924325943
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,16383,0.01454080045223236
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,3,0.008454400300979614
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,32767,0.03101919889450073
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,32767,0.01653439998626709
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,15,0.008398400247097015
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,65535,0.0513264000415802
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,65535,0.032204800844192506
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,float16,1,0.008436799794435502
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,fp8,1,0.008403199911117553
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,31,0.008425600081682205
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,float16,3,0.008393599838018417
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,float16,7,0.008414400368928909
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,fp8,3,0.008364800363779068
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,float16,15,0.008350399881601333
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,fp8,7,0.008348800241947174
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,fp8,15,0.008403199911117553
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,float16,31,0.008372800052165985
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,fp8,31,0.008369600027799606
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,float16,63,0.008345600217580795
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,fp8,63,0.00835840031504631
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,float16,127,0.008454400300979614
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,fp8,127,0.008294399827718735
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,fp8,255,0.00835840031504631
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,fp8,511,0.008420799672603608
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,float16,511,0.009617599844932555
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,float16,1023,0.011107199639081956
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,fp8,1023,0.010411199927330018
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,fp8,2047,0.012529599666595458
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,float16,4095,0.04520640075206757
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,fp8,4095,0.027724799513816834
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,fp8,8191,0.0463808000087738
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,float16,16383,0.15316640138626098
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,fp8,16383,0.0831167995929718
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,7,0.008406399935483932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,float16,32767,0.29667840003967283
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,float16,1,0.010447999835014344
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,fp8,1,0.00841120034456253
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,fp8,32767,0.15430879592895508
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,float16,3,0.009705600142478944
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,float16,7,0.01048159971833229
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,fp8,3,0.008396799862384795
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,fp8,7,0.00841120034456253
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,float16,15,0.01032480001449585
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,fp8,15,0.008399999886751174
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,fp8,31,0.00843999981880188
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,float16,63,0.01045759990811348
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,fp8,63,0.008422400057315826
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,float16,127,0.010497599840164185
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,fp8,127,0.008406399935483932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,16383,0.012943999469280243
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,fp8,255,0.008416000008583068
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,float16,511,0.012479999661445617
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,fp8,511,0.010475199669599533
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,float16,1023,0.027608001232147218
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,fp8,1023,0.014049600064754485
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,float16,2047,0.04561919867992401
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,fp8,2047,0.028884801268577575
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,float16,255,0.008336000144481659
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,float16,4095,0.08217120170593262
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,fp8,4095,0.04773600101470947
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,float16,2047,0.026867198944091796
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,float16,8191,0.15382239818572999
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,float16,8191,0.08102399706840516
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,float16,16383,0.2980272054672241
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,fp8,16383,0.16152960062026978
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,float16,1,0.010427200049161912
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,fp8,1,0.008377599716186523
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,float16,3,0.010366400331258773
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,fp8,3,0.008353599905967712
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,float16,7,0.010391999781131745
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,float16,32767,0.5877855777740478
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,fp8,7,0.008416000008583068
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,fp8,15,0.008366400003433227
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,float16,15,0.010345599800348281
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,fp8,31,0.008827199786901474
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,float16,63,0.01045600026845932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,fp8,63,0.009145600348711013
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,255,0.008423999696969987
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,float16,127,0.01040479987859726
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,float16,255,0.01040000021457672
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,fp8,127,0.008713600039482117
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,fp8,255,0.008459199965000153
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,float16,511,0.012534399330615998
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,float16,31,0.010345599800348281
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,fp8,511,0.010451199859380722
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,fp8,1023,0.014455999433994293
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,float16,1023,0.027379199862480164
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,float16,2047,0.04591360092163086
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,fp8,2047,0.02884800136089325
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,fp8,4095,0.047563201189041136
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,float16,4095,0.08195840120315552
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,float16,255,0.010435199737548828
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,fp8,1,0.011502400040626526
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,float16,8191,0.15457119941711425
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,float16,3,0.012476799637079239
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,fp8,8191,0.08403199911117554
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,float16,7,0.012464000284671784
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,fp8,3,0.010505600273609162
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,fp8,7,0.011475200206041336
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,float16,15,0.012476799637079239
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,fp8,15,0.010576000064611435
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,fp8,31,0.011563199758529662
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,float16,63,0.0124208003282547
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,fp8,63,0.011456000059843064
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,float16,127,0.012521600723266602
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,fp8,127,0.011553599685430526
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,float16,255,0.012518399953842163
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,fp8,255,0.010676799714565277
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,float16,511,0.02794240117073059
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,fp8,511,0.014556799829006196
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,fp8,8191,0.08421599864959717
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,fp8,1023,0.028969600796699524
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,fp8,2047,0.047244799137115476
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,float16,4095,0.1565600037574768
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,fp8,4095,0.08422080278396607
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,fp8,32767,0.310860800743103
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,float16,8191,0.3050607919692993
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,float16,31,0.010310400277376175
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,fp8,8191,0.1568992018699646
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,fp8,1,0.010982400178909302
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,float16,3,0.012532800436019897
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,float16,1,0.0124719999730587
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,fp8,3,0.011224000155925751
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,float16,7,0.012511999905109405
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,fp8,7,0.012516799569129943
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,float16,15,0.012436799705028534
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,fp8,15,0.012398400157690049
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,float16,31,0.01252799928188324
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,fp8,31,0.012430399656295776
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,float16,63,0.01252799928188324
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,fp8,63,0.011086399853229522
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,fp8,127,0.0112015999853611
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,float16,255,0.012483199685811996
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,fp8,255,0.011128000169992446
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,float16,511,0.02741599977016449
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,fp8,511,0.014470399916172027
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,fp8,1023,0.028968000411987306
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,float16,1023,0.046235200762748715
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,float16,2047,0.08410720229148864
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,fp8,2047,0.047593599557876586
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,float16,1,0.012436799705028534
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,float16,1,0.016599999368190767
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,float16,4095,0.1567456007003784
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,float16,3,0.01659359931945801
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,float16,7,0.016899199783802034
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,fp8,3,0.015663999319076537
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,float16,31,0.012479999661445617
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,fp8,7,0.016059200465679168
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,float16,15,0.01740639954805374
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,fp8,15,0.014580799639225006
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,float16,31,0.017500799894332886
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,fp8,31,0.01629440039396286
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,float16,63,0.01661919951438904
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,fp8,63,0.016407999396324157
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,fp8,127,0.01462559998035431
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,float16,127,0.017947199940681457
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,float16,255,0.027321600914001466
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,fp8,255,0.014790399372577668
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,fp8,511,0.028913599252700806
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,float16,1023,0.046942400932312014
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,float16,1023,0.08196480274200439
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,fp8,1023,0.04822239875793457
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,float16,2047,0.08279359936714173
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,float16,2047,0.15513919591903685
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,1,0.00843520015478134
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,float16,4095,0.2999903917312622
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,fp8,4095,0.15938080549240113
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,3,0.00843999981880188
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,3,0.008460800349712371
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,7,0.008404800295829773
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,7,0.008401600271463394
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,15,0.008393599838018417
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,15,0.008369600027799606
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,31,0.008484800159931184
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,31,0.008473599702119828
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,63,0.008417599648237229
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,63,0.00843999981880188
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,127,0.008379200100898742
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,127,0.008457600325345992
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,float16,127,0.01247360035777092
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,255,0.008475200086832047
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,255,0.008404800295829773
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,511,0.008408000320196151
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,1023,0.00843840017914772
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,511,0.008350399881601333
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,1023,0.008448000252246856
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,2047,0.010393600165843963
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,2047,0.00952799990773201
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,4095,0.011593600362539291
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,4095,0.010497599840164185
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,8191,0.012985600531101227
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,8191,0.012535999715328216
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,fp8,4095,0.08411679863929748
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,16383,0.01594240069389343
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,fp8,1,0.014593599736690522
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,32767,0.031307199597358705
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,65535,0.050097602605819705
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,32767,0.0165583997964859
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,65535,0.03235679864883423
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,1,0.008462399989366532
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,1,0.008416000008583068
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,3,0.00841279998421669
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,3,0.008444800227880477
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,7,0.00843840017914772
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,7,0.008444800227880477
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,15,0.008462399989366532
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,15,0.00841120034456253
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,31,0.008532799780368805
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,31,0.008427199721336365
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,63,0.009265600144863129
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,127,0.008428800106048583
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,float16,511,0.045470398664474485
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,255,0.008414400368928909
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,255,0.008414400368928909
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,511,0.00936800017952919
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,511,0.008376000076532364
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,1023,0.010367999970912933
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,1023,0.008475200086832047
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,fp8,2047,0.08541439771652222
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,2047,0.010401599854230881
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,4095,0.012481600046157837
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,4095,0.011158400028944016
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,8191,0.014531199634075165
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,1,0.008414400368928909
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,8191,0.012513600289821625
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,16383,0.030348798632621764
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,16383,0.0160303995013237
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,32767,0.04899680018424988
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,65535,0.08426560163497925
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,65535,0.04999839961528778
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,float16,1,0.01658879965543747
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,fp8,1,0.01682240068912506
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,float16,3,0.01659359931945801
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,fp8,3,0.01656000018119812
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,float16,7,0.01709440052509308
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,fp8,7,0.016543999314308167
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,float16,15,0.016847999393939973
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,fp8,15,0.01659359931945801
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,float16,31,0.016894400119781494
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,fp8,31,0.01658879965543747
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,float16,63,0.01661120057106018
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,fp8,63,0.01658080071210861
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,float16,127,0.01826400011777878
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,fp8,127,0.016575999557971954
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,float16,255,0.027959999442100526
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,fp8,255,0.01653439998626709
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,float16,511,0.04716640114784241
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,fp8,511,0.029067200422286988
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,float16,1023,0.08315359950065612
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,fp8,1023,0.04809919893741608
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,fp8,2047,0.0857807993888855
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,float16,2047,0.1564703941345215
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,16383,0.01358720064163208
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,float16,1,0.027137601375579835
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,float16,3,0.026833599805831908
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,fp8,1,0.02473759949207306
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,fp8,3,0.024719999730587007
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,fp8,7,0.024809600412845613
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,float16,15,0.027811199426651
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,fp8,15,0.024747200310230255
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,float16,31,0.026900801062583923
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,fp8,31,0.024809600412845613
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,float16,63,0.028217598795890808
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,fp8,63,0.024795199930667877
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,63,0.00835999995470047
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,fp8,127,0.024736000597476958
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,float16,255,0.046979200839996335
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,fp8,255,0.030350399017333985
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,float16,511,0.08252639770507812
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,fp8,511,0.047779199481010434
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,2047,0.010401599854230881
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,float16,1023,0.15486880540847778
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,fp8,1023,0.08510559797286987
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,float16,1,0.027211201190948487
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,fp8,1,0.024809600412845613
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,32767,0.030876800417900085
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,float16,3,0.027105599641799927
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,fp8,3,0.024846400320529937
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,float16,7,0.02735840082168579
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,fp8,7,0.024831999838352204
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,float16,15,0.02714560031890869
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,fp8,15,0.02481600046157837
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,float16,31,0.027238398790359497
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,fp8,31,0.024771200120449068
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,float16,63,0.028832000494003297
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,fp8,63,0.02475679963827133
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,float16,127,0.03094559907913208
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,fp8,127,0.024827200174331664
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,float16,255,0.0472351998090744
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,fp8,255,0.031227201223373413
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,float16,511,0.08410720229148864
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,float16,1023,0.15603840351104736
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,fp8,1023,0.08643199801445008
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,float16,1,0.04745120108127594
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,fp8,1,0.041340801119804385
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,float16,3,0.047270399332046506
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,fp8,3,0.04124319851398468
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,float16,7,0.027657601237297057
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,float16,7,0.047409600019454955
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,fp8,7,0.04127680063247681
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,float16,15,0.04735200107097626
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,float16,127,0.030895999073982237
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,fp8,15,0.04124639928340912
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,float16,31,0.04766879975795746
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,fp8,31,0.0416047990322113
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,float16,63,0.049369600415229795
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,fp8,63,0.04141600131988525
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,fp8,127,0.043463999032974245
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,float16,255,0.08411679863929748
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,fp8,255,0.052167999744415286
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,float16,511,0.15585119724273683
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,fp8,511,0.08680959939956664
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,float16,2047,0.3001055955886841
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,fp8,2047,0.15873759984970093
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,1,0.008750399947166443
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,1,0.008455999940633774
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,3,0.008451200276613235
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,float16,1023,0.2987871885299683
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,3,0.00838399976491928
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,15,0.008366400003433227
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,7,0.008363199979066848
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,15,0.008372800052165985
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,31,0.008432000130414962
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,31,0.008366400003433227
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,63,0.008366400003433227
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,127,0.008457600325345992
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,127,0.008430399745702744
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,255,0.008449599891901017
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,511,0.008367999643087386
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,255,0.008414400368928909
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,511,0.008423999696969987
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,1023,0.008801600337028504
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,1023,0.008664000034332275
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,2047,0.010441599786281586
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,2047,0.01040479987859726
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,4095,0.01252480000257492
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,4095,0.011343999952077865
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,8191,0.014552000164985656
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,8191,0.012476799637079239
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,16383,0.030374398827552794
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,16383,0.015425600111484528
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,32767,0.047907200455665586
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,32767,0.030888000130653383
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,65535,0.08416000008583069
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,65535,0.049588799476623535
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,1,0.008403199911117553
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,3,0.008408000320196151
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,3,0.008422400057315826
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,127,0.00835999995470047
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,7,0.008392000198364257
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,7,0.00838399976491928
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,15,0.008377599716186523
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,31,0.008379200100898742
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,float16,127,0.05121279954910278
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,31,0.008420799672603608
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,63,0.008363199979066848
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,63,0.008399999886751174
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,127,0.008337599784135818
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,127,0.008428800106048583
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,255,0.008376000076532364
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,255,0.00844319984316826
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,511,0.008401600271463394
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,1023,0.009124799817800521
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,1023,0.008376000076532364
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,2047,0.010465600341558457
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,2047,0.008929599821567536
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,4095,0.012484800070524216
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,fp8,1023,0.15991679430007935
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,7,0.00838399976491928
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,8191,0.027233600616455078
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,8191,0.01308639943599701
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,63,0.008363199979066848
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,16383,0.04563519954681396
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,16383,0.02881760001182556
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,32767,0.08213279843330383
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,fp8,511,0.04925119876861572
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,32767,0.047147199511528015
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,1,0.00843520015478134
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,65535,0.15408480167388916
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,65535,0.08422719836235046
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,1,0.008340799808502197
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,7,0.008446399867534638
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,3,0.00843999981880188
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,7,0.008372800052165985
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,15,0.008372800052165985
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,15,0.00843999981880188
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,31,0.008451200276613235
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,31,0.008339200168848038
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,63,0.008417599648237229
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,63,0.008363199979066848
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,127,0.008423999696969987
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,127,0.008427199721336365
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,255,0.0089199997484684
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,255,0.008420799672603608
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,511,0.008363199979066848
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,511,0.00844319984316826
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,1023,0.010382399708032609
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,1023,0.00843840017914772
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,2047,0.012433599680662155
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,2047,0.01048479974269867
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,4095,0.027801600098609925
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,1,0.008377599716186523
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,4095,0.01265760064125061
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,8191,0.04633280038833618
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,8191,0.02874560058116913
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,16383,0.08174719810485839
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,16383,0.0471455991268158
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,15,0.008414400368928909
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,32767,0.15343840122222902
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,32767,0.08256000280380249
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,65535,0.2972127914428711
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,65535,0.15564160346984862
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,511,0.008433599770069123
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,1,0.008446399867534638
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,3,0.008371199667453765
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,1,0.008420799672603608
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,3,0.00849440023303032
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,7,0.008395200222730636
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,7,0.008403199911117553
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,15,0.008433599770069123
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,15,0.008376000076532364
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,31,0.008420799672603608
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,131071,0.30285120010375977
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,63,0.008379200100898742
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,63,0.0083856001496315
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,4095,0.010454399883747101
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,127,0.008448000252246856
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,255,0.008832000195980072
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,127,0.008350399881601333
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,255,0.008392000198364257
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,511,0.008479999750852585
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,1023,0.008327999711036682
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,1023,0.00835999995470047
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,2047,0.010388799756765366
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,2047,0.008462399989366532
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,4095,0.010468800365924836
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,8191,0.012511999905109405
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,8191,0.01249760016798973
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,16383,0.01661120057106018
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,16383,0.01658560037612915
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,32767,0.01650079935789108
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,32767,0.01675039976835251
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,65535,0.018593600392341612
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,65535,0.01865759938955307
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,131071,0.020712000131607056
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,131071,0.020657600462436677
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,3,0.008388800173997879
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,3,0.008472000062465668
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,3,0.008416000008583068
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,7,0.008420799672603608
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,7,0.008417599648237229
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,15,0.008363199979066848
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,15,0.008339200168848038
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,31,0.008449599891901017
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,31,0.0083856001496315
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,63,0.008401600271463394
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,127,0.008387199789285659
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,63,0.008403199911117553
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,127,0.008323200047016144
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,255,0.00838399976491928
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,255,0.008444800227880477
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,511,0.008340799808502197
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,1023,0.008428800106048583
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,1023,0.008427199721336365
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,2047,0.010407999902963639
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,2047,0.009361600130796432
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,4095,0.01050880029797554
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,31,0.008427199721336365
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,4095,0.010366400331258773
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,131071,0.5836016178131104
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,511,0.008444800227880477
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,16383,0.016524800658226015
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,16383,0.016551999747753142
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,8191,0.012468799948692322
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,32767,0.018699200451374055
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,32767,0.017052799463272095
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,65535,0.02064319998025894
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,65535,0.01860480010509491
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,131071,0.03920640051364899
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,131071,0.0209647998213768
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,1,0.008345600217580795
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,1,0.00836160033941269
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,1,0.008377599716186523
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,3,0.008324799686670303
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,7,0.008366400003433227
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,7,0.008340799808502197
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,15,0.008406399935483932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,15,0.008371199667453765
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,31,0.008390399813652038
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,31,0.008348800241947174
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,63,0.008347199857234954
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,63,0.008348800241947174
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,127,0.008380799740552902
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,127,0.00833119973540306
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,255,0.008390399813652038
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,255,0.008387199789285659
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,511,0.00835840031504631
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,511,0.008423999696969987
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,1023,0.009115199744701385
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,1023,0.00843840017914772
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,2047,0.010492800176143647
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,2047,0.010390400141477584
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,511,0.008430399745702744
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,4095,0.01250240057706833
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,4095,0.010603199899196624
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,8191,0.014575999975204468
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,8191,0.02724800109863281
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,16383,0.04551520049571991
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,16383,0.028856000304222106
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,32767,0.08202880024909973
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,32767,0.04726719856262207
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,65535,0.15387040376663208
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,65535,0.08560320138931274
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,1,0.008764799684286118
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,1,0.008380799740552902
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,131071,0.29801440238952637
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,3,0.008388800173997879
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,131071,0.15773279666900636
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,3,0.008452799916267396
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,15,0.008406399935483932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,7,0.008420799672603608
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,15,0.008353599905967712
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,31,0.008470399677753449
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,31,0.008372800052165985
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,63,0.008406399935483932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,63,0.008399999886751174
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,127,0.008423999696969987
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,127,0.008436799794435502
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,255,0.008472000062465668
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,255,0.008367999643087386
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,511,0.008476799726486206
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,511,0.00888959988951683
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,1023,0.008446399867534638
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,4095,0.010475199669599533
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,1023,0.008340799808502197
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,2047,0.010417599976062775
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,4095,0.010420800000429154
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,2047,0.008449599891901017
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,4095,0.010486400127410889
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,8191,0.013801600039005279
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,8191,0.012547199428081513
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,16383,0.0144896000623703
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,32767,0.01865279972553253
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,1,0.008419200032949447
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,65535,0.018619200587272643
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,65535,0.03488160073757172
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,3,0.008347199857234954
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,131071,0.05267040133476257
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,1,0.00841120034456253
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,1,0.008315200358629227
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,131071,0.035036799311637876
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,3,0.008343999832868576
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,3,0.008327999711036682
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,7,0.008425600081682205
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,7,0.008388800173997879
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,15,0.008452799916267396
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,15,0.008779200166463852
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,31,0.008408000320196151
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,63,0.00843840017914772
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,63,0.008321599662303924
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,127,0.00835999995470047
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,127,0.008388800173997879
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,255,0.008263999968767166
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,511,0.01040000021457672
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,511,0.008367999643087386
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,1023,0.010409600287675857
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,2047,0.027055999636650084
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,7,0.008449599891901017
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,2047,0.012604799866676331
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,4095,0.04529280066490173
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,8191,0.012542399764060973
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,8191,0.08130239844322204
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,8191,0.046009600162506104
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,16383,0.016230399906635284
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,32767,0.016415999829769136
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,16383,0.15326399803161622
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,16383,0.0826095998287201
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,31,0.008422400057315826
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,32767,0.15501439571380615
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,255,0.008342400193214417
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,float16,1,0.01035040020942688
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,fp8,1,0.00835999995470047
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,1023,0.012452799826860428
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,float16,3,0.010377600044012069
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,fp8,3,0.008430399745702744
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,float16,7,0.010470400005578995
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,fp8,7,0.008417599648237229
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,float16,15,0.01040000021457672
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,65535,0.5835567951202393
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,float16,31,0.010367999970912933
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,fp8,31,0.009081599861383438
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,fp8,63,0.00883840024471283
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,65535,0.3010143995285034
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,fp8,127,0.008955200016498566
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,float16,127,0.01043199971318245
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,float16,63,0.01043040007352829
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,fp8,255,0.008388800173997879
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,float16,255,0.010414399951696397
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,float16,1023,0.026932799816131593
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,fp8,1023,0.014555199444293976
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,float16,2047,0.04534560143947601
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,float16,4095,0.08178399801254273
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,fp8,4095,0.046726399660110475
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,float16,8191,0.15436160564422607
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,4095,0.027470400929450987
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,fp8,8191,0.08408960103988647
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,float16,16383,0.29920639991760256
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,fp8,16383,0.1585103988647461
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,float16,1,0.01247519999742508
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,fp8,1,0.012441600114107132
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,float16,3,0.012537600100040435
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,fp8,3,0.01249760016798973
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,fp8,7,0.011580800265073776
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,float16,15,0.012495999783277511
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,fp8,15,0.01146399974822998
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,float16,31,0.01247519999742508
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,fp8,31,0.011710400134325028
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,32767,0.2962368011474609
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,fp8,63,0.011423999816179276
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,float16,127,0.012432000041007996
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,fp8,127,0.012383999675512314
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,float16,255,0.012470400333404541
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,fp8,255,0.0124208003282547
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,float16,511,0.02789919972419739
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,fp8,15,0.008448000252246856
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,fp8,511,0.014572800695896148
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,float16,1023,0.04609119892120361
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,float16,2047,0.0836736023426056
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,fp8,2047,0.04753440022468567
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,fp8,511,0.01045600026845932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,float16,511,0.01252480000257492
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,fp8,4095,0.08515040278434753
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,fp8,2047,0.028780800104141236
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,1,0.008451200276613235
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,3,0.008387199789285659
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,float16,8191,0.3018815994262695
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,fp8,8191,0.15640319585800172
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,3,0.008329600095748901
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,15,0.008408000320196151
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,7,0.008908800035715102
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,15,0.008454400300979614
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,31,0.009409599751234055
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,31,0.008398400247097015
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,63,0.008611200004816055
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,63,0.008372800052165985
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,127,0.0084927998483181
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,127,0.00841120034456253
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,255,0.0084927998483181
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,255,0.008430399745702744
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,511,0.009068799763917923
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,511,0.008463999629020691
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,1023,0.008451200276613235
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,2047,0.0104032002389431
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,2047,0.010422399640083313
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,4095,0.010924799740314484
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,4095,0.010471999645233154
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,8191,0.012705600261688233
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,8191,0.012479999661445617
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,16383,0.014614400267601014
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,16383,0.013681599497795105
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,32767,0.03094559907913208
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,32767,0.016550399363040924
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,float16,63,0.012465599924325943
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,65535,0.050995200872421265
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,65535,0.032548800110816956
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,131071,0.08614400029182434
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,131071,0.050913602113723755
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,float16,1,0.0173552006483078
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,fp8,1,0.016543999314308167
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,fp8,1023,0.02881920039653778
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,float16,3,0.016574400663375854
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,fp8,3,0.0165583997964859
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,float16,7,0.016638399660587312
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,fp8,7,0.01653279960155487
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,float16,15,0.01661760061979294
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,fp8,15,0.016518400609493257
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,float16,31,0.016627199947834015
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,float16,4095,0.1582576036453247
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,fp8,31,0.016523200273513793
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,float16,63,0.016652800142765045
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,float16,127,0.01664000004529953
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,fp8,127,0.016513599455356597
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,float16,255,0.02693600058555603
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,fp8,255,0.016539199650287627
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,float16,511,0.04618239998817444
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,fp8,511,0.0288783997297287
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,float16,1023,0.08276640176773072
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,fp8,1023,0.04855999946594238
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,7,0.008420799672603608
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,fp8,2047,0.08536319732666016
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,float16,2047,0.15676000118255615
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,float16,1,0.02810719907283783
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,float16,4095,0.2985343933105469
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,fp8,1,0.02503040134906769
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,fp8,4095,0.16068480014801026
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,fp8,3,0.024780799448490144
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,float16,3,0.027289599180221558
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,float16,7,0.02863360047340393
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,float16,15,0.026995199918746948
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,fp8,15,0.024846400320529937
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,float16,31,0.02690559923648834
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,fp8,31,0.024835200607776643
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,float16,63,0.02704479992389679
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,fp8,63,0.02476000040769577
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,float16,7,0.0124719999730587
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,float16,127,0.030878400802612303
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,fp8,127,0.02481919974088669
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,float16,255,0.04700320065021515
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,fp8,255,0.031031998991966247
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,float16,511,0.08255040049552917
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,fp8,511,0.049332800507545474
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,float16,1023,0.15460799932479857
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,fp8,1023,0.08581920266151429
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,1,0.008401600271463394
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,1,0.008392000198364257
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,3,0.008420799672603608
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,3,0.008416000008583068
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,7,0.008406399935483932
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,fp8,2047,0.15864640474319458
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,float16,2047,0.3012703895568848
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,15,0.008422400057315826
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,31,0.008414400368928909
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,31,0.008372800052165985
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,63,0.008449599891901017
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,63,0.008419200032949447
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,127,0.008419200032949447
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,127,0.008399999886751174
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,255,0.008481600135564805
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,255,0.00840959995985031
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,511,0.008816000074148178
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,1,0.008452799916267396
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,1023,0.008478400111198426
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,fp8,63,0.01653279960155487
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,2047,0.010460799932479859
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,2047,0.010452800244092942
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,4095,0.012559999525547028
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,4095,0.010443200170993806
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,8191,0.012495999783277511
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,16383,0.03028320074081421
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,16383,0.015993599593639374
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,32767,0.04827040135860443
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,32767,0.030880001187324525
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,65535,0.08466879725456238
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,fp8,7,0.024795199930667877
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,65535,0.04933440089225769
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,131071,0.0868224024772644
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,131071,0.15646079778671265
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,7,0.008352000266313553
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,15,0.008419200032949447
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,511,0.008398400247097015
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,1023,0.008444800227880477
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,8191,0.014553600549697876
SGLang,0.5.9,NVIDIA B200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,1023,0.010131199657917023
