framework,version,device,op_name,kernel_source,batch_size,isl,num_heads,num_key_value_heads,head_dim,beam_width,attn_dtype,kv_cache_dtype,step,latency
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,64,128,1,float16,float16,1,0.15316959619522094
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,64,128,1,float16,float16,3,0.15299199819564818
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,64,128,1,float16,float16,7,0.15346879959106446
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,64,128,1,float16,float16,15,0.1568511962890625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,64,128,1,float16,float16,31,0.16351360082626343
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,64,128,1,float16,float16,63,0.17355680465698242
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,64,128,1,float16,float16,127,0.24891200065612792
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,64,128,1,float16,float16,255,0.41984000205993655
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,64,128,1,float16,float16,511,1.0444191932678222
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,64,128,1,float16,float16,1023,1.8938480377197267
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,64,128,1,float16,float16,2047,3.582872009277344
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,64,128,1,float16,float16,7,0.01613440066576004
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,64,128,1,float16,float16,15,0.01616799980401993
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,64,128,1,float16,float16,31,0.016118399798870087
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,64,128,1,float16,float16,63,0.016091200709342956
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,64,128,1,float16,float16,127,0.018875199556350707
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,64,128,1,float16,float16,255,0.023643200099468232
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,64,128,1,float16,float16,1023,0.07354559898376464
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,64,128,1,float16,float16,2047,0.11393760442733765
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,64,128,1,float16,float16,1,0.01812320053577423
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,64,128,1,float16,float16,3,0.01809120029211044
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,64,128,1,float16,float16,7,0.018136000633239745
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,64,128,1,float16,float16,15,0.018225599825382233
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,64,128,1,float16,float16,31,0.018329599499702455
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,64,128,1,float16,float16,63,0.01850239932537079
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,64,128,1,float16,float16,127,0.02259040027856827
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,64,128,1,float16,float16,255,0.030694401264190672
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,64,128,1,float16,float16,511,0.07232800126075745
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,64,128,1,float16,float16,1023,0.11729120016098023
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,64,128,1,float16,float16,1,0.08411039710044861
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,64,128,1,float16,float16,7,0.0843616008758545
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,64,128,1,float16,float16,15,0.08465920090675354
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,64,128,1,float16,float16,31,0.08921120166778565
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,64,128,1,float16,float16,127,0.13425920009613038
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,64,128,1,float16,float16,255,0.2182624101638794
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,64,128,1,float16,float16,1,0.015993599593639374
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,64,128,1,float16,float16,1023,0.9581007957458496
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,64,128,1,float16,float16,1,0.023596799373626708
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,64,128,1,float16,float16,3,0.023654399812221526
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,64,128,1,float16,float16,7,0.023609599471092223
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,64,128,1,float16,float16,15,0.023839999735355378
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,64,128,1,float16,float16,31,0.023955200612545014
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,64,128,1,float16,float16,63,0.02455199956893921
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,64,128,1,float16,float16,2047,1.7188079833984375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,64,128,1,float16,float16,127,0.030409601330757142
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,64,128,1,float16,float16,255,0.05224800109863281
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,64,128,1,float16,float16,511,0.10138239860534667
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,64,128,1,float16,float16,1023,0.17607519626617432
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,64,128,1,float16,float16,2047,0.28999359607696534
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,64,128,1,float16,float16,1,0.30488479137420654
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,64,128,1,float16,float16,3,0.29917919635772705
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,64,128,1,float16,float16,7,0.3101151943206787
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,64,128,1,float16,float16,15,0.3042543888092041
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,64,128,1,float16,float16,31,0.3219856023788452
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,64,128,1,float16,float16,63,0.3247663974761963
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,64,128,1,float16,float16,127,0.47484002113342283
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,64,128,1,float16,float16,255,0.8040255546569824
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,64,128,1,float16,float16,3,0.08456000089645385
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,64,128,1,float16,float16,1,0.5798655986785889
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,64,128,1,float16,float16,3,0.5819087982177734
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,64,128,1,float16,float16,511,2.098944091796875
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,64,128,1,float16,float16,7,0.5870783805847168
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,64,128,1,float16,float16,15,0.6112143993377686
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,64,128,1,float16,float16,31,0.6049888134002686
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,64,128,1,float16,float16,63,0.6427343845367431
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,64,128,1,float16,float16,127,0.9205727577209473
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,64,128,1,float16,float16,1023,3.859040069580078
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,64,128,1,float16,float16,3,0.0159183993935585
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,64,128,1,float16,float16,255,1.6169376373291016
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,64,64,128,1,float16,float16,1,1.1538240432739257
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,64,64,128,1,float16,float16,3,1.1927007675170898
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,64,64,128,1,float16,float16,7,1.1566672325134277
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,64,64,128,1,float16,float16,15,1.2288592338562012
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,64,64,128,1,float16,float16,31,1.1896384239196778
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,64,64,128,1,float16,float16,63,1.2593567848205567
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,64,128,1,float16,float16,1,0.031142398715019226
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,64,128,1,float16,float16,7,0.03115839958190918
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,64,128,1,float16,float16,3,0.030772799253463747
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,64,128,1,float16,float16,15,0.03136799931526184
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,64,128,1,float16,float16,31,0.03168799877166748
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,64,128,1,float16,float16,63,0.03243359923362732
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,64,128,1,float16,float16,127,0.05208799839019775
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,64,128,1,float16,float16,255,0.07434080243110656
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,64,64,128,1,float16,float16,127,1.826655960083008
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,64,128,1,float16,float16,511,0.15468640327453614
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,64,128,1,float16,float16,1023,0.26784799098968504
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,64,128,1,float16,float16,2047,0.46586880683898924
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,64,128,1,float16,float16,511,0.04759680032730103
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,64,64,128,1,float16,float16,3,2.295667266845703
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,64,64,128,1,float16,float16,7,2.3030607223510744
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,64,64,128,1,float16,float16,1,2.314147186279297
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,64,64,128,1,float16,float16,15,2.3256927490234376
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,64,64,128,1,float16,float16,31,2.3634048461914063
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,64,64,128,1,float16,float16,63,2.4701904296875
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,64,64,128,1,float16,float16,1,4.542379379272461
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,64,64,128,1,float16,float16,3,4.564870452880859
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,64,128,1,float16,float16,3,0.04773600101470947
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,64,128,1,float16,float16,1,0.04782559871673584
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,64,128,1,float16,float16,7,0.04797599911689758
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,64,64,128,1,float16,float16,7,4.558556747436524
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,64,128,1,float16,float16,15,0.04820320010185242
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,64,128,1,float16,float16,31,0.04844639897346496
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,64,128,1,float16,float16,63,0.058006399869918825
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,64,128,1,float16,float16,127,0.07835040092468262
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,64,128,1,float16,float16,255,0.12088960409164429
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,64,128,1,float16,float16,2047,0.1962463974952698
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,64,128,1,float16,float16,511,0.28706400394439696
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,64,128,1,float16,float16,1023,0.5084991931915284
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,64,128,1,float16,float16,2047,0.9045968055725098
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,64,64,128,1,float16,float16,15,4.6294097900390625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,64,64,128,1,float16,float16,31,4.721163177490235
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,48,48,128,1,float16,float16,1,0.11900320053100585
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,48,48,128,1,float16,float16,63,0.13473600149154663
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,48,48,128,1,float16,float16,127,0.19273439645767212
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,48,48,128,1,float16,float16,255,0.3219696044921875
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,64,128,1,float16,float16,63,0.09717919826507568
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,48,48,128,1,float16,float16,511,0.7977920055389405
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,48,48,128,1,float16,float16,15,0.12234879732131958
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,48,48,128,1,float16,float16,1,0.01539199948310852
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,48,48,128,1,float16,float16,3,0.015449599921703338
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,48,48,128,1,float16,float16,7,0.01547040045261383
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,48,48,128,1,float16,float16,15,0.01541759967803955
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,48,48,128,1,float16,float16,2047,2.6647312164306642
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,48,48,128,1,float16,float16,31,0.015508800745010376
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,48,48,128,1,float16,float16,127,0.01812479943037033
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,48,48,128,1,float16,float16,63,0.01594720035791397
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,48,48,128,1,float16,float16,511,0.033390399813652036
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,48,48,128,1,float16,float16,1023,0.045335999131202696
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,48,48,128,1,float16,float16,2047,0.06955999732017518
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,48,48,128,1,float16,float16,1,0.01606079936027527
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,48,48,128,1,float16,float16,3,0.015807999670505522
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,48,48,128,1,float16,float16,15,0.015851199626922607
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,48,48,128,1,float16,float16,7,0.015974399447441102
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,48,48,128,1,float16,float16,31,0.015940800309181213
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,48,48,128,1,float16,float16,63,0.016318400204181672
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,48,48,128,1,float16,float16,127,0.018796800076961516
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,48,48,128,1,float16,float16,255,0.02396959960460663
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,48,48,128,1,float16,float16,511,0.04343520104885101
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,48,48,128,1,float16,float16,2047,0.10723520517349243
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,48,48,128,1,float16,float16,1023,0.06911200284957886
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,48,48,128,1,float16,float16,1,0.06666079759597779
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,48,48,128,1,float16,float16,3,0.06739040017127991
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,48,48,128,1,float16,float16,7,0.06694080233573914
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,48,48,128,1,float16,float16,15,0.06808639764785766
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,48,48,128,1,float16,float16,31,0.06993280053138733
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,48,48,128,1,float16,float16,63,0.0787551999092102
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,64,128,1,float16,float16,511,0.5350607872009278
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,48,48,128,1,float16,float16,127,0.10659040212631225
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,48,48,128,1,float16,float16,255,0.17598719596862794
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,48,48,128,1,float16,float16,1023,0.7493167877197265
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,48,48,128,1,float16,float16,3,0.11962399482727051
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,48,48,128,1,float16,float16,1,0.017518399655818938
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,48,48,128,1,float16,float16,3,0.017689600586891174
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,48,48,128,1,float16,float16,7,0.017611199617385866
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,48,48,128,1,float16,float16,15,0.017795200645923614
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,48,48,128,1,float16,float16,31,0.01786559969186783
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,48,48,128,1,float16,float16,63,0.018310399353504182
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,48,48,128,1,float16,float16,127,0.022414399683475493
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,48,48,128,1,float16,float16,255,0.03321920037269592
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,48,48,128,1,float16,float16,511,0.0745360016822815
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,48,48,128,1,float16,float16,2047,1.35283203125
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,48,48,128,1,float16,float16,1023,0.11822240352630616
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,48,48,128,1,float16,float16,3,0.22586560249328613
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,48,48,128,1,float16,float16,2047,0.1987712025642395
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,48,48,128,1,float16,float16,1,0.23222720623016357
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,48,48,128,1,float16,float16,7,0.2263040065765381
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,48,48,128,1,float16,float16,15,0.231606388092041
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,48,48,128,1,float16,float16,31,0.24408318996429443
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,48,48,128,1,float16,float16,63,0.2502032041549683
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,48,48,128,1,float16,float16,127,0.3608031988143921
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,48,48,128,1,float16,float16,255,0.6247951984405518
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,48,48,128,1,float16,float16,1,0.4437535762786865
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,48,48,128,1,float16,float16,3,0.44542560577392576
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,48,48,128,1,float16,float16,511,1.5639552116394042
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,48,48,128,1,float16,float16,7,0.44151840209960935
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,48,48,128,1,float16,float16,255,0.023132799565792082
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,48,48,128,1,float16,float16,15,0.45886878967285155
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,48,48,128,1,float16,float16,1023,2.8999711990356447
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,48,48,128,1,float16,float16,31,0.4582223892211914
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,48,48,128,1,float16,float16,63,0.48053441047668455
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,48,48,128,1,float16,float16,127,0.7165103912353515
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,48,48,128,1,float16,float16,1,0.8708736419677734
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,48,48,128,1,float16,float16,3,0.8753680229187012
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,48,48,128,1,float16,float16,255,1.2545200347900392
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,48,48,128,1,float16,float16,7,0.8839008331298828
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,48,48,128,1,float16,float16,15,0.88721923828125
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,48,48,128,1,float16,float16,1,0.025153601169586183
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,48,48,128,1,float16,float16,3,0.025227200984954835
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,48,48,128,1,float16,float16,7,0.02534720003604889
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,48,48,128,1,float16,float16,15,0.02543199956417084
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,48,48,128,1,float16,float16,31,0.9021936416625976
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,48,48,128,1,float16,float16,31,0.0256879985332489
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,48,48,128,1,float16,float16,127,0.036257600784301756
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,48,48,128,1,float16,float16,63,0.02611680030822754
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,48,48,128,1,float16,float16,63,0.9869503974914551
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,48,48,128,1,float16,float16,255,0.05792800188064575
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,48,48,128,1,float16,float16,511,0.12640320062637328
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,48,48,128,1,float16,float16,1023,0.22077438831329346
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,48,48,128,1,float16,float16,2047,0.3715712070465088
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,48,48,128,1,float16,float16,127,1.3795647621154785
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,48,48,128,1,float16,float16,31,0.1280079960823059
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,48,48,128,1,float16,float16,1,1.7225936889648437
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,48,48,128,1,float16,float16,3,1.7566944122314454
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,48,48,128,1,float16,float16,7,1.733951950073242
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,48,48,128,1,float16,float16,15,1.7694511413574219
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,48,48,128,1,float16,float16,31,1.7871759414672852
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,48,48,128,1,float16,float16,63,1.8825471878051758
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,48,48,128,1,float16,float16,1,3.41890869140625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,48,48,128,1,float16,float16,3,3.439235305786133
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,48,48,128,1,float16,float16,7,3.4479808807373047
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,48,48,128,1,float16,float16,1,0.03988640010356903
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,48,48,128,1,float16,float16,3,0.039743998646736146
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,48,48,128,1,float16,float16,15,0.04004800021648407
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,48,48,128,1,float16,float16,7,0.040043199062347413
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,48,48,128,1,float16,float16,31,0.040452799201011656
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,48,48,128,1,float16,float16,15,3.5088062286376953
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,48,48,128,1,float16,float16,1023,1.4341135978698731
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,48,48,128,1,float16,float16,63,0.04460160136222839
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,48,48,128,1,float16,float16,127,0.06457120180130005
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,48,48,128,1,float16,float16,255,0.09874719977378846
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,48,48,128,1,float16,float16,511,0.23276159763336182
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,48,48,128,1,float16,float16,1023,0.40900478363037107
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,48,48,128,1,float16,float16,2047,0.730841588973999
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,48,48,128,1,float16,float16,31,3.5507568359375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,48,48,128,1,float16,float16,7,0.11953439712524414
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,40,40,128,1,float16,float16,1,0.10053440332412719
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,40,40,128,1,float16,float16,127,0.16242079734802245
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,40,40,128,1,float16,float16,255,0.2727855920791626
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,40,40,128,1,float16,float16,1023,1.2019536018371582
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,40,40,128,1,float16,float16,31,0.10809279680252075
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,40,40,128,1,float16,float16,1,0.01515520066022873
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,40,40,128,1,float16,float16,3,0.015063999593257904
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,40,40,128,1,float16,float16,7,0.015311999619007111
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,40,40,128,1,float16,float16,15,0.015204800665378571
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,40,40,128,1,float16,float16,31,0.015145599842071533
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,40,40,128,1,float16,float16,63,0.015364800393581391
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,40,40,128,1,float16,float16,127,0.017811200022697447
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,48,48,128,1,float16,float16,511,0.4199552059173584
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,40,40,128,1,float16,float16,255,0.022547200322151184
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,40,40,128,1,float16,float16,1023,0.04309599995613098
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,40,40,128,1,float16,float16,2047,2.190995216369629
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,40,40,128,1,float16,float16,2047,0.08222560286521911
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,40,40,128,1,float16,float16,3,0.015387199819087982
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,40,40,128,1,float16,float16,1,0.016131199896335602
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,40,40,128,1,float16,float16,7,0.015441599488258361
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,40,40,128,1,float16,float16,15,0.016200000047683717
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,40,40,128,1,float16,float16,31,0.015520000457763672
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,40,40,128,1,float16,float16,127,0.01825760006904602
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,40,40,128,1,float16,float16,63,0.01643040031194687
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,40,40,128,1,float16,float16,255,0.02311519980430603
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,40,40,128,1,float16,float16,511,0.043761599063873294
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,40,40,128,1,float16,float16,1023,0.07259520292282104
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,40,40,128,1,float16,float16,1,0.05571200251579285
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,40,40,128,1,float16,float16,2047,0.11770399808883666
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,40,40,128,1,float16,float16,3,0.05565919876098633
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,40,40,128,1,float16,float16,15,0.10182720422744751
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,40,40,128,1,float16,float16,15,0.05612800121307373
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,40,40,128,1,float16,float16,7,0.05716639757156372
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,40,40,128,1,float16,float16,31,0.05732160210609436
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,40,40,128,1,float16,float16,63,0.06591519713401794
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,40,40,128,1,float16,float16,127,0.0928816020488739
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,40,40,128,1,float16,float16,255,0.1483183979988098
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,40,40,128,1,float16,float16,511,0.34213919639587403
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,40,40,128,1,float16,float16,1,0.01716800034046173
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,40,40,128,1,float16,float16,3,0.017238399386405943
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,40,40,128,1,float16,float16,7,0.017140799760818483
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,40,40,128,1,float16,float16,1023,0.6156943798065185
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,40,40,128,1,float16,float16,15,0.017315199971199034
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,40,40,128,1,float16,float16,31,0.017688000202178956
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,40,40,128,1,float16,float16,63,0.017712000012397765
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,40,40,128,1,float16,float16,127,0.02227199971675873
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,40,40,128,1,float16,float16,255,0.030774399638175964
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,40,40,128,1,float16,float16,511,0.07278879880905151
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,40,40,128,1,float16,float16,1023,0.11544959545135498
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,40,40,128,1,float16,float16,2047,1.0841024398803711
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,40,40,128,1,float16,float16,2047,0.19791200160980224
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,40,40,128,1,float16,float16,1,0.18944480419158935
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,40,40,128,1,float16,float16,3,0.19016799926757813
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,40,40,128,1,float16,float16,15,0.19439200162887574
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,40,40,128,1,float16,float16,7,0.19309279918670655
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,40,40,128,1,float16,float16,31,0.20331840515136718
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,40,40,128,1,float16,float16,63,0.21151359081268312
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,40,40,128,1,float16,float16,127,0.3120703935623169
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,40,40,128,1,float16,float16,255,0.5216415882110595
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,40,40,128,1,float16,float16,1,0.3688096046447754
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,40,40,128,1,float16,float16,511,1.3048895835876464
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,40,40,128,1,float16,float16,3,0.3697535991668701
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,40,40,128,1,float16,float16,7,0.37845759391784667
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,40,40,128,1,float16,float16,511,0.6664112091064454
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,40,40,128,1,float16,float16,15,0.3816224098205566
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,40,40,128,1,float16,float16,31,0.3924288034439087
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,40,40,128,1,float16,float16,63,0.40781121253967284
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,40,40,128,1,float16,float16,127,0.5879487991333008
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,40,40,128,1,float16,float16,1023,2.4091632843017576
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,40,40,128,1,float16,float16,1,0.7352335929870606
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,40,40,128,1,float16,float16,3,0.7355135917663574
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,40,40,128,1,float16,float16,255,1.012384033203125
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,40,40,128,1,float16,float16,7,0.762062406539917
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,40,40,128,1,float16,float16,15,0.7472991943359375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,40,40,128,1,float16,float16,1,0.023292799293994904
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,40,40,128,1,float16,float16,31,0.7601119995117187
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,40,40,128,1,float16,float16,63,0.8457440376281739
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,40,40,128,1,float16,float16,3,0.023371200263500213
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,40,40,128,1,float16,float16,15,0.02353920042514801
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,40,40,128,1,float16,float16,7,0.024344000220298766
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,40,40,128,1,float16,float16,31,0.024352000653743745
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,40,40,128,1,float16,float16,63,0.026392000913619994
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,40,40,128,1,float16,float16,127,1.1844816207885742
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,40,40,128,1,float16,float16,127,0.031748801469802856
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,40,40,128,1,float16,float16,255,0.06503199934959411
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,40,40,128,1,float16,float16,1023,0.16688159704208375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,40,40,128,1,float16,float16,511,0.10644639730453491
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,40,40,128,1,float16,float16,511,0.0326447993516922
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,40,40,128,1,float16,float16,2047,0.29241600036621096
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,40,40,128,1,float16,float16,1,1.4443455696105958
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,40,40,128,1,float16,float16,3,1.488640022277832
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,40,40,128,1,float16,float16,7,1.4555055618286132
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,40,40,128,1,float16,float16,15,1.4984479904174806
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,40,40,128,1,float16,float16,31,1.498633575439453
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,40,40,128,1,float16,float16,63,1.5821344375610351
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,40,40,128,1,float16,float16,1,2.8687711715698243
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,40,40,128,1,float16,float16,3,2.8820512771606444
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,40,40,128,1,float16,float16,1,0.032636800408363344
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,40,40,128,1,float16,float16,3,0.032739201188087465
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,40,40,128,1,float16,float16,7,0.03275519907474518
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,40,40,128,1,float16,float16,15,0.032969599962234496
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,40,40,128,1,float16,float16,31,0.03336319923400879
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,40,40,128,1,float16,float16,63,0.03487519919872284
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,40,40,128,1,float16,float16,7,2.8895488739013673
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,40,40,128,1,float16,float16,127,0.05315679907798767
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,40,40,128,1,float16,float16,255,0.082259202003479
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,40,40,128,1,float16,float16,511,0.18201119899749757
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,40,40,128,1,float16,float16,15,2.9487071990966798
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,40,40,128,1,float16,float16,1023,0.3144576072692871
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,40,40,128,1,float16,float16,2047,0.5518864154815674
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,40,40,128,1,float16,float16,31,2.9886112213134766
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,40,40,128,1,float16,float16,3,0.10049760341644287
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,32,128,1,float16,float16,1,0.08437439799308777
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,32,128,1,float16,float16,127,0.13319040536880494
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,32,128,1,float16,float16,255,0.22050879001617432
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,32,128,1,float16,float16,511,0.5377471923828125
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,40,40,128,1,float16,float16,63,0.11580640077590942
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,32,128,1,float16,float16,1023,0.9687423706054688
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,40,40,128,1,float16,float16,7,0.10145920515060425
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,32,128,1,float16,float16,63,0.09643200039863586
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,32,128,1,float16,float16,3,0.01478240042924881
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,32,128,1,float16,float16,7,0.014860799908638
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,32,128,1,float16,float16,15,0.014800000190734863
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,32,128,1,float16,float16,31,0.015052799880504609
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,32,128,1,float16,float16,63,0.014998400211334228
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,32,128,1,float16,float16,127,0.017694400250911714
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,32,128,1,float16,float16,4095,3.4141422271728517
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,32,128,1,float16,float16,255,0.02252320051193237
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,32,128,1,float16,float16,1023,0.03752320110797882
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,32,128,1,float16,float16,511,0.030211201310157774
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,32,128,1,float16,float16,2047,0.05469759702682495
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,32,128,1,float16,float16,4095,0.08598880171775818
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,32,128,1,float16,float16,1,0.014955200254917145
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,32,128,1,float16,float16,7,0.015092800557613372
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,32,128,1,float16,float16,15,0.015072000026702882
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,32,128,1,float16,float16,3,0.015936000645160674
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,32,128,1,float16,float16,31,0.01528480052947998
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,32,128,1,float16,float16,63,0.01578879952430725
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,32,128,1,float16,float16,127,0.017852799594402315
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,32,128,1,float16,float16,255,0.023268799483776092
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,32,128,1,float16,float16,511,0.04602240025997162
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,32,128,1,float16,float16,2047,0.11180800199508667
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,32,128,1,float16,float16,1023,0.07241439819335938
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,32,128,1,float16,float16,1,0.048537600040435794
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,32,128,1,float16,float16,4095,0.1919535994529724
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,32,128,1,float16,float16,3,0.04867999851703644
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,32,128,1,float16,float16,7,0.04812160134315491
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,32,128,1,float16,float16,31,0.048767998814582825
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,32,128,1,float16,float16,15,0.048956799507141116
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,32,128,1,float16,float16,63,0.056513601541519166
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,32,128,1,float16,float16,127,0.07874240279197693
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,32,128,1,float16,float16,255,0.12026560306549072
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,32,128,1,float16,float16,1,0.014920000731945039
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,32,128,1,float16,float16,511,0.2908911943435669
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,32,128,1,float16,float16,1023,0.5065296173095704
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,32,128,1,float16,float16,1,0.01693439930677414
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,32,128,1,float16,float16,3,0.01712159961462021
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,32,128,1,float16,float16,2047,0.9092608451843261
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,32,128,1,float16,float16,7,0.017047999799251555
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,32,128,1,float16,float16,15,0.01722559928894043
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,32,128,1,float16,float16,31,0.017185600101947786
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,32,128,1,float16,float16,63,0.017451199889183044
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,32,128,1,float16,float16,127,0.0217616006731987
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,32,128,1,float16,float16,255,0.030022400617599487
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,32,128,1,float16,float16,511,0.07020800113677979
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,32,128,1,float16,float16,1023,0.11604800224304199
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,32,128,1,float16,float16,2047,0.19318560361862183
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,32,128,1,float16,float16,1,0.15500639677047728
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,32,128,1,float16,float16,4095,1.7275503158569336
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,32,128,1,float16,float16,3,0.15527039766311646
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,32,128,1,float16,float16,4095,0.3554960012435913
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,32,128,1,float16,float16,15,0.15720959901809692
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,32,128,1,float16,float16,7,0.16613119840621948
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,32,128,1,float16,float16,31,0.16631200313568115
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,32,128,1,float16,float16,63,0.17318559885025026
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,32,128,1,float16,float16,127,0.2577471971511841
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,32,128,1,float16,float16,3,0.08473280072212219
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,32,128,1,float16,float16,255,0.4123040199279785
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,32,128,1,float16,float16,511,1.0853520393371583
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,32,128,1,float16,float16,1,0.30788960456848147
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,32,128,1,float16,float16,1023,1.9234464645385743
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,32,128,1,float16,float16,3,0.3023983955383301
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,32,128,1,float16,float16,7,0.3159424066543579
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,32,128,1,float16,float16,15,0.31297600269317627
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,32,128,1,float16,float16,31,0.3241791963577271
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,32,128,1,float16,float16,63,0.33410561084747314
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,32,128,1,float16,float16,127,0.48480000495910647
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,32,128,1,float16,float16,2047,3.56051025390625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,32,128,1,float16,float16,255,0.8195568084716797
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,32,128,1,float16,float16,7,0.0849568009376526
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,32,128,1,float16,float16,1,0.6243631839752197
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,32,128,1,float16,float16,3,0.5951632022857666
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,32,128,1,float16,float16,7,0.5948448181152344
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,32,128,1,float16,float16,15,0.6243807792663574
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,32,128,1,float16,float16,31,0.6160783767700195
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,32,128,1,float16,float16,511,2.102836799621582
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,32,128,1,float16,float16,63,0.635913610458374
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,32,128,1,float16,float16,1,0.023639999330043793
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,32,128,1,float16,float16,3,0.02274879962205887
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,32,128,1,float16,float16,7,0.023553599417209626
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,32,128,1,float16,float16,127,0.9596688270568847
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,32,128,1,float16,float16,15,0.02285120040178299
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,32,128,1,float16,float16,31,0.023521600663661955
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,32,128,1,float16,float16,63,0.02374880015850067
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,32,128,1,float16,float16,127,0.030193600058555602
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,32,128,1,float16,float16,511,0.09969760179519653
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,32,128,1,float16,float16,255,0.04815039932727814
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,32,128,1,float16,float16,1023,0.16401760578155516
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,32,128,1,float16,float16,2047,0.2884320020675659
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,32,128,1,float16,float16,4095,0.5286240100860595
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,32,128,1,float16,float16,255,1.6169120788574218
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,32,32,128,1,float16,float16,1,1.163060760498047
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,32,32,128,1,float16,float16,3,1.188385581970215
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,32,32,128,1,float16,float16,7,1.1864416122436523
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,32,32,128,1,float16,float16,15,1.2465503692626954
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,32,32,128,1,float16,float16,31,1.2017248153686524
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,32,32,128,1,float16,float16,63,1.2786543846130372
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,32,128,1,float16,float16,15,0.08565760254859925
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,32,32,128,1,float16,float16,127,1.854128074645996
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,32,32,128,1,float16,float16,1,2.3236976623535157
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,32,32,128,1,float16,float16,3,2.3087392807006837
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,32,32,128,1,float16,float16,7,2.353438377380371
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,32,128,1,float16,float16,1,0.030740800499916076
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,32,128,1,float16,float16,3,0.030532801151275636
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,32,128,1,float16,float16,7,0.030668801069259642
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,32,128,1,float16,float16,31,0.031196799874305726
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,32,128,1,float16,float16,15,0.030822399258613586
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,32,128,1,float16,float16,63,0.032251200079917906
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,32,128,1,float16,float16,127,0.04630880057811737
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,32,128,1,float16,float16,255,0.07121919989585876
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,32,128,1,float16,float16,511,0.15569759607315065
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,32,32,128,1,float16,float16,15,2.3488544464111327
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,32,128,1,float16,float16,1023,0.2634704113006592
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,32,32,128,1,float16,float16,31,2.3815120697021483
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,32,128,1,float16,float16,2047,0.46173601150512694
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,32,128,1,float16,float16,4095,0.8643759727478028
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,32,32,128,1,float16,float16,63,2.5430576324462892
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,24,24,128,1,float16,float16,3,0.06697279810905457
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,24,24,128,1,float16,float16,127,0.1060752034187317
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,32,128,1,float16,float16,2047,1.7206239700317383
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,24,24,128,1,float16,float16,255,0.16980960369110107
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,24,24,128,1,float16,float16,1023,0.7555056095123291
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,24,24,128,1,float16,float16,7,0.06735519766807556
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,24,24,128,1,float16,float16,2047,1.357652759552002
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,32,128,1,float16,float16,31,0.09001439809799194
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,24,24,128,1,float16,float16,1,0.015039999783039094
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,24,24,128,1,float16,float16,7,0.014884799718856812
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,24,24,128,1,float16,float16,15,0.014883199334144592
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,24,24,128,1,float16,float16,31,0.014856000244617463
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,24,24,128,1,float16,float16,63,0.014982399344444276
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,24,24,128,1,float16,float16,127,0.017633600533008574
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,24,24,128,1,float16,float16,255,0.022500799596309663
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,24,24,128,1,float16,float16,511,0.028071999549865723
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,24,24,128,1,float16,float16,1023,0.033190399408340454
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,24,24,128,1,float16,float16,2047,0.044228801131248476
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,24,24,128,1,float16,float16,4095,2.5739568710327148
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,24,24,128,1,float16,float16,4095,0.06910240054130554
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,24,24,128,1,float16,float16,1,0.01518079936504364
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,24,24,128,1,float16,float16,3,0.015015999972820281
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,24,24,128,1,float16,float16,7,0.014990399777889251
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,24,24,128,1,float16,float16,15,0.014838400483131408
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,24,24,128,1,float16,float16,1,0.06732320189476013
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,24,24,128,1,float16,float16,63,0.015048000216484069
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,24,24,128,1,float16,float16,31,0.015254400670528412
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,24,24,128,1,float16,float16,255,0.02266079932451248
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,24,24,128,1,float16,float16,127,0.01767680048942566
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,24,24,128,1,float16,float16,511,0.033262398838996884
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,24,24,128,1,float16,float16,1023,0.043828800320625305
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,24,24,128,1,float16,float16,2047,0.06945760250091552
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,24,24,128,1,float16,float16,1,0.03981119990348816
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,24,24,128,1,float16,float16,3,0.04004960060119629
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,24,24,128,1,float16,float16,7,0.039961600303649904
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,24,24,128,1,float16,float16,15,0.04040960073471069
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,24,24,128,1,float16,float16,31,0.040505599975585935
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,24,24,128,1,float16,float16,4095,0.10951839685440064
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,24,24,128,1,float16,float16,63,0.04345119893550873
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,24,24,128,1,float16,float16,127,0.06324959993362426
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,24,24,128,1,float16,float16,255,0.09664959907531738
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,24,24,128,1,float16,float16,511,0.23238720893859863
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,24,24,128,1,float16,float16,1023,0.40786237716674806
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,24,24,128,1,float16,float16,2047,0.7244991779327392
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,24,24,128,1,float16,float16,1,0.015027199685573579
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,24,24,128,1,float16,float16,3,0.015107199549674988
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,24,24,128,1,float16,float16,7,0.01518079936504364
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,24,24,128,1,float16,float16,15,0.015078400075435639
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,24,24,128,1,float16,float16,4095,1.3697104454040527
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,24,24,128,1,float16,float16,31,0.015323199331760406
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,24,24,128,1,float16,float16,127,0.01814880073070526
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,24,24,128,1,float16,float16,63,0.015751999616622925
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,24,24,128,1,float16,float16,255,0.022971199452877046
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,24,24,128,1,float16,float16,511,0.0429392009973526
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,24,24,128,1,float16,float16,1023,0.06633920073509217
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,24,24,128,1,float16,float16,2047,0.10821599960327148
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,24,24,128,1,float16,float16,4095,0.18479839563369752
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,24,24,128,1,float16,float16,1,0.12249120473861694
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,24,24,128,1,float16,float16,3,0.014833599328994751
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,24,24,128,1,float16,float16,7,0.12283200025558472
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,24,24,128,1,float16,float16,3,0.1204640030860901
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,24,24,128,1,float16,float16,15,0.1219215989112854
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,24,24,128,1,float16,float16,31,0.12917920351028442
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,24,24,128,1,float16,float16,63,0.13577760457992555
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,24,24,128,1,float16,float16,127,0.19363679885864257
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,24,24,128,1,float16,float16,255,0.31658880710601806
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,24,24,128,1,float16,float16,511,0.8027711868286133
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,24,24,128,1,float16,float16,1,0.22910559177398682
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,24,24,128,1,float16,float16,3,0.22933440208435057
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,24,24,128,1,float16,float16,1023,1.4523743629455566
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,24,24,128,1,float16,float16,7,0.23088479042053223
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,24,24,128,1,float16,float16,15,0.2406912088394165
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,24,24,128,1,float16,float16,31,0.2425328016281128
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,24,24,128,1,float16,float16,63,0.26072800159454346
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,24,24,128,1,float16,float16,127,0.3653359889984131
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,24,24,128,1,float16,float16,2047,2.674519920349121
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,24,24,128,1,float16,float16,255,0.6092895984649658
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,24,24,128,1,float16,float16,1,0.4480559825897217
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,24,24,128,1,float16,float16,3,0.48372640609741213
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,24,24,128,1,float16,float16,7,0.4554719924926758
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,24,24,128,1,float16,float16,15,0.46104159355163576
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,24,24,128,1,float16,float16,511,0.4245823860168457
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,24,24,128,1,float16,float16,31,0.4922944068908691
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,24,24,128,1,float16,float16,63,0.4945536136627197
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,24,24,128,1,float16,float16,511,1.5779040336608887
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,24,24,128,1,float16,float16,1,0.01730400025844574
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,24,24,128,1,float16,float16,127,0.7124127864837646
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,24,24,128,1,float16,float16,15,0.01746399998664856
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,24,24,128,1,float16,float16,3,0.016947199404239655
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,24,24,128,1,float16,float16,31,0.01794240027666092
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,24,24,128,1,float16,float16,63,0.017785599827766417
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,24,24,128,1,float16,float16,7,0.017057600617408752
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,24,24,128,1,float16,float16,127,0.021731199324131013
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,24,24,128,1,float16,float16,255,0.03249439895153046
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,24,24,128,1,float16,float16,255,1.2284768104553223
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,24,24,128,1,float16,float16,511,0.07200319766998291
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,24,24,128,1,float16,float16,1023,0.11804800033569336
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,24,24,128,1,float16,float16,2047,0.19385600090026855
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,24,24,128,1,float16,float16,4095,0.36218559741973877
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,24,24,128,1,float16,float16,1,0.9067008018493652
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,24,24,128,1,float16,float16,3,0.8917087554931641
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,24,24,128,1,float16,float16,7,0.9079680442810059
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,24,24,128,1,float16,float16,15,0.9005167961120606
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,24,24,128,1,float16,float16,31,0.9281760215759277
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,24,24,128,1,float16,float16,31,0.07009279727935791
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,24,24,128,1,float16,float16,63,0.9784928321838379
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,24,24,128,1,float16,float16,127,1.4031904220581055
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,24,24,128,1,float16,float16,3,1.7469600677490233
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,24,24,128,1,float16,float16,1,1.7595344543457032
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,24,24,128,1,float16,float16,1,0.024798400700092316
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,24,24,128,1,float16,float16,7,1.7770496368408204
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,24,24,128,1,float16,float16,3,0.02489279955625534
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,24,24,128,1,float16,float16,7,0.024956800043582916
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,24,24,128,1,float16,float16,15,1.789616012573242
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,24,24,128,1,float16,float16,31,1.830129623413086
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,24,24,128,1,float16,float16,63,0.025918400287628172
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,24,24,128,1,float16,float16,15,0.025121599435806274
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,24,24,128,1,float16,float16,31,0.025523200631141663
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,24,24,128,1,float16,float16,127,0.03592160046100616
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,24,24,128,1,float16,float16,255,0.05724800229072571
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,24,24,128,1,float16,float16,511,0.12479360103607177
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,24,24,128,1,float16,float16,1023,0.2132783889770508
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,24,24,128,1,float16,float16,2047,0.3714080095291138
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,24,24,128,1,float16,float16,4095,0.6886879920959472
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,24,24,128,1,float16,float16,63,1.9161376953125
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,16,128,1,float16,float16,1,0.048607999086380006
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,24,24,128,1,float16,float16,15,0.0676800012588501
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,16,128,1,float16,float16,127,0.07736160159111023
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,16,128,1,float16,float16,511,0.28967359066009524
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,16,128,1,float16,float16,1023,0.5136240005493165
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,16,128,1,float16,float16,2047,0.914691162109375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,16,128,1,float16,float16,255,0.11996959447860718
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,16,128,1,float16,float16,4095,1.7161712646484375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,16,128,1,float16,float16,1,0.014417600631713868
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,16,128,1,float16,float16,3,0.014414399862289429
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,16,128,1,float16,float16,7,0.014504000544548035
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,16,128,1,float16,float16,15,0.014431999623775482
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,16,128,1,float16,float16,31,0.014542399346828461
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,24,24,128,1,float16,float16,63,0.07797759771347046
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,16,128,1,float16,float16,8191,3.4490734100341798
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,16,128,1,float16,float16,127,0.018441599607467652
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,16,128,1,float16,float16,255,0.02284799963235855
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,16,128,1,float16,float16,511,0.02452159970998764
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,16,128,1,float16,float16,3,0.0484607994556427
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,16,128,1,float16,float16,1023,0.029174399375915528
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,16,128,1,float16,float16,2047,0.03678399920463562
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,16,128,1,float16,float16,4095,0.055009597539901735
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,16,128,1,float16,float16,8191,0.08176640272140503
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,16,128,1,float16,float16,1,0.014900800585746766
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,16,128,1,float16,float16,3,0.014310400187969207
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,16,128,1,float16,float16,15,0.014443199336528777
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,16,128,1,float16,float16,7,0.014923200011253357
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,16,128,1,float16,float16,31,0.01438560038805008
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,16,128,1,float16,float16,63,0.014873600006103516
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,16,128,1,float16,float16,127,0.017292800545692443
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,16,128,1,float16,float16,511,0.028137600421905516
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,16,128,1,float16,float16,255,0.022422400116920472
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,16,128,1,float16,float16,1023,0.03751679956912994
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,16,128,1,float16,float16,2047,0.05485919713973999
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,16,128,1,float16,float16,4095,0.08139520287513732
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,16,128,1,float16,float16,1,0.03043360114097595
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,16,128,1,float16,float16,3,0.030457600951194763
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,16,128,1,float16,float16,8191,0.13637280464172363
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,16,128,1,float16,float16,7,0.030616000294685364
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,16,128,1,float16,float16,31,0.031148800253868104
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,16,128,1,float16,float16,15,0.031219199299812317
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,16,128,1,float16,float16,63,0.031825599074363706
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,16,128,1,float16,float16,127,0.04737919867038727
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,16,128,1,float16,float16,255,0.0690288007259369
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,16,128,1,float16,float16,511,0.1568287968635559
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,16,128,1,float16,float16,1023,0.26416161060333254
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,16,128,1,float16,float16,2047,0.46760001182556155
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,16,128,1,float16,float16,7,0.048583999276161194
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,16,128,1,float16,float16,1,0.014744000136852264
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,16,128,1,float16,float16,4095,0.8629728317260742
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,16,128,1,float16,float16,3,0.014718399941921234
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,16,128,1,float16,float16,31,0.014670400321483612
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,16,128,1,float16,float16,7,0.014715200662612915
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,16,128,1,float16,float16,15,0.014822399616241455
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,16,128,1,float16,float16,63,0.014913600683212281
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,16,128,1,float16,float16,255,0.022305600345134735
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,16,128,1,float16,float16,127,0.01759359985589981
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,16,128,1,float16,float16,511,0.045742401480674745
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,16,128,1,float16,float16,1023,0.07031199932098389
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,16,128,1,float16,float16,2047,0.11194720268249511
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,16,128,1,float16,float16,4095,0.1903040051460266
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,16,128,1,float16,float16,1,0.08636159896850586
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,16,128,1,float16,float16,3,0.08640639781951905
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,16,128,1,float16,float16,8191,0.35375199317932127
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,16,128,1,float16,float16,7,0.08666880130767822
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,16,128,1,float16,float16,15,0.087363201379776
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,16,128,1,float16,float16,31,0.09099680185317993
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,16,128,1,float16,float16,8191,1.699470329284668
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,16,128,1,float16,float16,127,0.13496799468994142
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,16,128,1,float16,float16,63,0.09828959703445435
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,16,128,1,float16,float16,255,0.21912639141082763
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,16,128,1,float16,float16,511,0.5608543872833252
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,16,128,1,float16,float16,1023,0.964350414276123
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,16,128,1,float16,float16,1,0.16225919723510743
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,16,128,1,float16,float16,7,0.16089119911193847
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,16,128,1,float16,float16,3,0.1588528037071228
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,16,128,1,float16,float16,31,0.05003839731216431
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,16,128,1,float16,float16,15,0.1636896014213562
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,16,128,1,float16,float16,31,0.16941440105438232
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,16,128,1,float16,float16,2047,1.7265375137329102
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,16,128,1,float16,float16,63,0.17923840284347534
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,16,128,1,float16,float16,127,0.2569391965866089
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,16,128,1,float16,float16,255,0.4216928005218506
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,16,128,1,float16,float16,1,0.3094336032867432
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,16,128,1,float16,float16,3,0.3069727897644043
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,16,128,1,float16,float16,7,0.3135760068893433
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,16,128,1,float16,float16,511,1.091710376739502
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,16,128,1,float16,float16,15,0.3166815996170044
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,16,128,1,float16,float16,31,0.32710559368133546
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,16,128,1,float16,float16,4095,3.3779216766357423
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,16,128,1,float16,float16,63,0.34072160720825195
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,16,128,1,float16,float16,127,0.4843264102935791
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,16,128,1,float16,float16,1,0.016678400337696075
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,16,128,1,float16,float16,7,0.016729600727558136
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,16,128,1,float16,float16,1023,1.9299840927124023
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,16,128,1,float16,float16,3,0.01658560037612915
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,16,128,1,float16,float16,15,0.016896000504493712
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,16,128,1,float16,float16,255,0.8222224235534668
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,16,128,1,float16,float16,63,0.017088000476360322
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,16,128,1,float16,float16,31,0.017310400307178498
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,16,128,1,float16,float16,127,0.021558399498462676
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,16,128,1,float16,float16,255,0.02932640016078949
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,16,128,1,float16,float16,1023,0.11586560010910034
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,16,128,1,float16,float16,511,0.06908159852027893
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,16,128,1,float16,float16,4095,0.35269439220428467
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,16,128,1,float16,float16,511,2.1530927658081054
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,16,128,1,float16,float16,2047,0.1918063998222351
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,16,128,1,float16,float16,63,0.014572800695896148
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,16,128,1,float16,float16,1,0.5949135780334472
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,16,128,1,float16,float16,3,0.598364782333374
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,16,128,1,float16,float16,7,0.6145247936248779
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,16,128,1,float16,float16,8191,0.687172794342041
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,16,128,1,float16,float16,15,0.6113056182861328
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,16,128,1,float16,float16,31,0.6236159801483154
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,16,128,1,float16,float16,63,0.6438127994537354
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,16,128,1,float16,float16,127,0.9446047782897949
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,16,16,128,1,float16,float16,1,1.191748809814453
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,16,16,128,1,float16,float16,3,1.1864383697509766
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,16,16,128,1,float16,float16,7,1.192251205444336
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,16,128,1,float16,float16,255,1.646763229370117
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,16,128,1,float16,float16,1,0.02452960014343262
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,16,16,128,1,float16,float16,15,1.2219296455383302
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,16,128,1,float16,float16,3,0.02449440062046051
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,16,16,128,1,float16,float16,31,1.2300687789916993
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,16,128,1,float16,float16,15,0.024564799666404725
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,16,128,1,float16,float16,7,0.02252960056066513
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,16,128,1,float16,float16,31,0.02290560007095337
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,16,128,1,float16,float16,63,0.02407200038433075
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,16,128,1,float16,float16,127,0.02937760055065155
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,16,128,1,float16,float16,255,0.046433600783348086
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,16,16,128,1,float16,float16,63,1.265401554107666
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,16,128,1,float16,float16,15,0.04919840097427368
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,16,128,1,float16,float16,511,0.10124640464782715
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,16,128,1,float16,float16,1023,0.1637279987335205
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,16,128,1,float16,float16,2047,0.2888688087463379
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,16,128,1,float16,float16,4095,0.5326240062713623
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,16,16,128,1,float16,float16,127,1.863484764099121
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,16,128,1,float16,float16,8191,1.020308780670166
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,12,12,128,1,float16,float16,1,0.04100959897041321
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,16,128,1,float16,float16,63,0.05753600001335144
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,12,12,128,1,float16,float16,127,0.06519200205802918
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,12,12,128,1,float16,float16,511,0.23653759956359863
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,12,12,128,1,float16,float16,1023,0.41454081535339354
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,12,12,128,1,float16,float16,4095,1.3692720413208008
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,12,12,128,1,float16,float16,255,0.09921119809150696
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,12,12,128,1,float16,float16,1,0.014630399644374847
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,12,12,128,1,float16,float16,3,0.01443679928779602
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,12,12,128,1,float16,float16,7,0.014617599546909332
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,12,12,128,1,float16,float16,15,0.014640000462532044
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,12,12,128,1,float16,float16,31,0.014655999839305878
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,12,12,128,1,float16,float16,63,0.01475519984960556
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,12,12,128,1,float16,float16,127,0.017294399440288544
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,12,12,128,1,float16,float16,255,0.02210880070924759
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,12,12,128,1,float16,float16,511,0.02470560073852539
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,12,12,128,1,float16,float16,1023,0.029014399647712706
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,12,12,128,1,float16,float16,2047,0.03388960063457489
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,12,12,128,1,float16,float16,4095,0.045465600490570066
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,12,12,128,1,float16,float16,8191,0.06939839720726013
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,12,12,128,1,float16,float16,1,0.014825600385665893
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,12,12,128,1,float16,float16,8191,2.7232080459594727
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,12,12,128,1,float16,float16,3,0.014734399318695069
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,12,12,128,1,float16,float16,15,0.014873600006103516
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,12,12,128,1,float16,float16,31,0.01494079977273941
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,12,12,128,1,float16,float16,7,0.015347200632095336
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,12,12,128,1,float16,float16,63,0.014923200011253357
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,12,12,128,1,float16,float16,127,0.018292799592018127
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,12,12,128,1,float16,float16,255,0.0225055992603302
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,12,12,128,1,float16,float16,1023,0.034539198875427245
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,12,12,128,1,float16,float16,511,0.030526399612426758
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,12,12,128,1,float16,float16,2047,0.0452320009469986
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,12,12,128,1,float16,float16,4095,0.07226880192756653
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,12,12,128,1,float16,float16,8191,0.10957119464874268
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,12,12,128,1,float16,float16,1,0.02614719867706299
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,12,12,128,1,float16,float16,3,0.025489598512649536
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,12,12,128,1,float16,float16,7,0.025843200087547303
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,12,12,128,1,float16,float16,15,0.02594879865646362
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,12,12,128,1,float16,float16,31,0.026235198974609374
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,12,12,128,1,float16,float16,63,0.026468798518180847
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,12,12,128,1,float16,float16,2047,0.7279695987701416
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,12,12,128,1,float16,float16,255,0.05788959860801697
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,12,12,128,1,float16,float16,127,0.03698720037937164
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,12,12,128,1,float16,float16,511,0.1281216025352478
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,12,12,128,1,float16,float16,1023,0.21488480567932128
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,12,12,128,1,float16,float16,2047,0.3799936056137085
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,12,12,128,1,float16,float16,1,0.015020799636840821
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,12,12,128,1,float16,float16,3,0.015028800070285796
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,12,12,128,1,float16,float16,7,0.014843200147151948
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,12,12,128,1,float16,float16,4095,0.6927872180938721
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,12,12,128,1,float16,float16,15,0.01515520066022873
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,12,12,128,1,float16,float16,31,0.01485760062932968
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,12,12,128,1,float16,float16,63,0.015203200280666351
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,12,12,128,1,float16,float16,127,0.01764799952507019
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,12,12,128,1,float16,float16,255,0.02268960028886795
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,12,12,128,1,float16,float16,511,0.034190401434898376
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,12,12,128,1,float16,float16,1023,0.04628480076789856
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,12,12,128,1,float16,float16,8191,1.3434479713439942
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,12,12,128,1,float16,float16,2047,0.07042239904403687
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,12,12,128,1,float16,float16,4095,0.11018400192260742
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,12,12,128,1,float16,float16,8191,0.18925280570983888
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,12,12,128,1,float16,float16,1,0.06918879747390747
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,12,12,128,1,float16,float16,3,0.06907039880752563
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,12,12,128,1,float16,float16,7,0.06962080001831054
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,12,12,128,1,float16,float16,15,0.06983839869499206
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,12,12,128,1,float16,float16,31,0.07174879908561707
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,12,12,128,1,float16,float16,63,0.07982879877090454
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,12,12,128,1,float16,float16,127,0.10883840322494506
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,12,12,128,1,float16,float16,255,0.1762879967689514
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,12,12,128,1,float16,float16,511,0.4331088066101074
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,12,12,128,1,float16,float16,1023,0.7634687900543213
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,12,12,128,1,float16,float16,1,0.12503360509872435
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,12,12,128,1,float16,float16,3,0.1252400040626526
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,12,12,128,1,float16,float16,3,0.04105440080165863
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,12,12,128,1,float16,float16,7,0.12539680004119874
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,12,12,128,1,float16,float16,15,0.12550079822540283
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,12,12,128,1,float16,float16,2047,1.364583969116211
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,12,12,128,1,float16,float16,31,0.13084319829940796
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,12,12,128,1,float16,float16,63,0.14418879747390748
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,12,12,128,1,float16,float16,127,0.19608800411224364
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,12,12,128,1,float16,float16,255,0.33759360313415526
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,12,12,128,1,float16,float16,4095,2.612995147705078
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,12,12,128,1,float16,float16,511,0.8162816047668457
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,12,12,128,1,float16,float16,1,0.24701600074768065
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,12,12,128,1,float16,float16,3,0.23595199584960938
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,12,12,128,1,float16,float16,7,0.2482208013534546
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,12,12,128,1,float16,float16,15,0.24435360431671144
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,12,12,128,1,float16,float16,1023,1.4910335540771484
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,12,12,128,1,float16,float16,31,0.26067519187927246
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,12,12,128,1,float16,float16,63,0.26107039451599123
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,12,12,128,1,float16,float16,127,0.3825648069381714
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,12,12,128,1,float16,float16,1,0.015238399803638458
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,12,12,128,1,float16,float16,255,0.6500448226928711
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,12,12,128,1,float16,float16,3,0.015219199657440185
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,12,12,128,1,float16,float16,15,0.015252800285816192
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,12,12,128,1,float16,float16,7,0.015532800555229187
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,12,12,128,1,float16,float16,31,0.015481600165367126
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,12,12,128,1,float16,float16,63,0.01569119989871979
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,12,12,128,1,float16,float16,127,0.01825280040502548
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,12,12,128,1,float16,float16,255,0.02354719936847687
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,12,12,128,1,float16,float16,511,1.5930944442749024
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,12,12,128,1,float16,float16,511,0.04307839870452881
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,12,12,128,1,float16,float16,1023,0.06800959706306457
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,12,12,128,1,float16,float16,2047,0.10661599636077881
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,12,12,128,1,float16,float16,4095,0.18808319568634033
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,12,12,128,1,float16,float16,7,0.041352000832557675
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,12,12,128,1,float16,float16,8191,0.3620255947113037
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,12,12,128,1,float16,float16,1,0.4523632049560547
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,12,12,128,1,float16,float16,3,0.45569601058959963
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,12,12,128,1,float16,float16,7,0.45952482223510743
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,12,12,128,1,float16,float16,15,0.4754608154296875
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,12,12,128,1,float16,float16,31,0.47643680572509767
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,12,12,128,1,float16,float16,63,0.49057278633117674
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,12,12,128,1,float16,float16,127,0.7234352111816407
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,12,12,128,1,float16,float16,3,0.8938480377197265
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,12,12,128,1,float16,float16,1,0.8937295913696289
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,12,12,128,1,float16,float16,255,1.267948818206787
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,12,12,128,1,float16,float16,7,0.9055600166320801
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,12,12,128,1,float16,float16,1,0.01703840047121048
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,12,12,128,1,float16,float16,3,0.017046399414539337
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,12,12,128,1,float16,float16,15,0.908841609954834
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,12,12,128,1,float16,float16,7,0.01717440038919449
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,12,12,128,1,float16,float16,31,0.92401123046875
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,12,12,128,1,float16,float16,31,0.017420800030231477
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,12,12,128,1,float16,float16,63,0.9559552192687988
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,12,12,128,1,float16,float16,15,0.01711200028657913
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,12,12,128,1,float16,float16,127,0.021835200488567352
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,12,12,128,1,float16,float16,63,0.017558400332927705
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,12,12,128,1,float16,float16,255,0.03130080103874207
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,12,12,128,1,float16,float16,511,0.07195360064506531
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,12,12,128,1,float16,float16,1023,0.11637760400772094
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,12,12,128,1,float16,float16,2047,0.19493919610977173
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,12,12,128,1,float16,float16,63,0.04574880003929138
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,12,12,128,1,float16,float16,127,1.4514335632324218
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,12,12,128,1,float16,float16,4095,0.35597279071807864
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,12,12,128,1,float16,float16,8191,0.678327989578247
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,8,128,1,float16,float16,1,0.03103039860725403
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,8,128,1,float16,float16,127,0.04654560089111328
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,8,128,1,float16,float16,255,0.07009440064430236
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,8,128,1,float16,float16,511,0.15707520246505738
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,12,12,128,1,float16,float16,15,0.041319999098777774
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,8,128,1,float16,float16,1023,0.267524790763855
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,8,128,1,float16,float16,4095,0.8685791969299317
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,8,128,1,float16,float16,63,0.03238880038261414
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,8,128,1,float16,float16,8191,1.7152368545532226
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,8,128,1,float16,float16,1,0.015315200388431548
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,8,128,1,float16,float16,3,0.014871999621391296
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,12,12,128,1,float16,float16,31,0.04181599915027619
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,8,128,1,float16,float16,7,0.014841599762439728
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,8,128,1,float16,float16,15,0.014697599411010741
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,8,128,1,float16,float16,63,0.01570879966020584
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,8,128,1,float16,float16,127,0.01733279973268509
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,8,128,1,float16,float16,255,0.02253440022468567
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,8,128,1,float16,float16,511,0.02364159971475601
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,8,128,1,float16,float16,1023,0.028003200888633728
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,8,128,1,float16,float16,2047,0.02900480031967163
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,8,128,1,float16,float16,4095,0.03814879953861237
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,8,128,1,float16,float16,8191,0.05556640028953552
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,8,128,1,float16,float16,16383,3.4191566467285157
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,8,128,1,float16,float16,1,0.01488800048828125
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,8,128,1,float16,float16,16383,0.08314080238342285
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,8,128,1,float16,float16,3,0.014860799908638
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,8,128,1,float16,float16,7,0.01438080072402954
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,8,128,1,float16,float16,15,0.014718399941921234
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,8,128,1,float16,float16,31,0.014435200393199921
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,8,128,1,float16,float16,63,0.015059199929237366
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,8,128,1,float16,float16,127,0.017187200486660004
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,8,128,1,float16,float16,2047,0.46521601676940916
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,8,128,1,float16,float16,511,0.0236272007226944
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,8,128,1,float16,float16,255,0.022832000255584718
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,8,128,1,float16,float16,1023,0.027752000093460082
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,8,128,1,float16,float16,2047,0.0367904007434845
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,8,128,1,float16,float16,8191,0.0815343976020813
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,8,128,1,float16,float16,4095,0.05628799796104431
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,8,128,1,float16,float16,1,0.022673599421977997
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,8,128,1,float16,float16,3,0.024107199907302857
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,8,128,1,float16,float16,16383,0.21640639305114745
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,8,128,1,float16,float16,7,0.024142399430274963
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,8,128,1,float16,float16,15,0.02330400049686432
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,8,128,1,float16,float16,31,0.024486400187015533
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,8,128,1,float16,float16,63,0.023742400109767914
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,8,128,1,float16,float16,127,0.029843199253082275
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,8,128,1,float16,float16,255,0.04636479914188385
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,8,128,1,float16,float16,1023,0.16453919410705567
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,8,128,1,float16,float16,511,0.09913920164108277
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,8,128,1,float16,float16,2047,0.28448479175567626
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,8,128,1,float16,float16,4095,0.5291279792785645
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,8,128,1,float16,float16,8191,1.0178720474243164
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,8,128,1,float16,float16,1,0.014377599954605103
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,8,128,1,float16,float16,3,0.014243200421333313
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,8,128,1,float16,float16,7,0.014363199472427368
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,8,128,1,float16,float16,31,0.014056000113487243
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,8,128,1,float16,float16,16383,1.9858543395996093
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,8,128,1,float16,float16,15,0.014327999949455262
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,8,128,1,float16,float16,31,0.014312000572681427
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,8,128,1,float16,float16,255,0.021916800737380983
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,8,128,1,float16,float16,127,0.017209599912166595
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,8,128,1,float16,float16,63,0.014472000300884247
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,8,128,1,float16,float16,511,0.027956798672676086
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,8,128,1,float16,float16,1023,0.037457600235939026
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,8,128,1,float16,float16,4095,0.08090239763259888
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,8,128,1,float16,float16,2047,0.05409600138664246
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,8,128,1,float16,float16,1,0.049804800748825075
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,8,128,1,float16,float16,8191,0.13479360342025756
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,8,128,1,float16,float16,3,0.04966399967670441
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,8,128,1,float16,float16,16383,0.35029280185699463
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,8,128,1,float16,float16,7,0.050164800882339475
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,8,128,1,float16,float16,15,0.05036640167236328
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,8,128,1,float16,float16,63,0.057948797941207886
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,8,128,1,float16,float16,31,0.05116959810256958
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,8,128,1,float16,float16,255,0.12126879692077637
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,8,128,1,float16,float16,127,0.07915359735488892
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,8,128,1,float16,float16,511,0.29379839897155763
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,8,128,1,float16,float16,1023,0.5186063766479492
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,8,128,1,float16,float16,2047,0.9167823791503906
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,8,128,1,float16,float16,1,0.08876960277557373
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,8,128,1,float16,float16,3,0.08857280015945435
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,8,128,1,float16,float16,7,0.08885120153427124
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,8,128,1,float16,float16,15,0.0893392026424408
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,8,128,1,float16,float16,4095,1.7276784896850585
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,8,128,1,float16,float16,31,0.09207040071487427
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,8,128,1,float16,float16,63,0.099755197763443
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,8,128,1,float16,float16,127,0.14237279891967775
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,8,128,1,float16,float16,255,0.22206079959869385
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,8,128,1,float16,float16,511,0.5527472019195556
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,8,128,1,float16,float16,1,0.16407840251922606
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,8,128,1,float16,float16,1023,0.9954192161560058
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,8,128,1,float16,float16,3,0.16440320014953613
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,8,128,1,float16,float16,8191,3.514089584350586
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,8,128,1,float16,float16,7,0.03128319978713989
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,8,128,1,float16,float16,15,0.16719199419021608
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,8,128,1,float16,float16,7,0.16677119731903076
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,8,128,1,float16,float16,63,0.18006720542907714
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,8,128,1,float16,float16,31,0.18992639780044557
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,8,128,1,float16,float16,127,0.25945439338684084
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,8,128,1,float16,float16,2047,1.7500272750854493
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,8,128,1,float16,float16,255,0.43293280601501466
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,8,128,1,float16,float16,1,0.014366400241851807
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,8,128,1,float16,float16,15,0.014603200554847717
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,8,128,1,float16,float16,3,0.014761599898338317
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,8,128,1,float16,float16,7,0.01456640064716339
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,8,128,1,float16,float16,31,0.01436000019311905
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,8,128,1,float16,float16,127,0.017375999689102174
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,8,128,1,float16,float16,63,0.014892800152301789
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,8,128,1,float16,float16,255,0.022145600616931917
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,8,128,1,float16,float16,511,0.04566720128059387
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,8,128,1,float16,float16,1023,0.0715071976184845
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,8,128,1,float16,float16,511,1.0755167961120606
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,8,128,1,float16,float16,2047,0.11123039722442626
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,8,128,1,float16,float16,4095,0.19270559549331664
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,8,128,1,float16,float16,1023,1.9868000030517579
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,8,128,1,float16,float16,8191,0.35809600353240967
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,8,8,128,1,float16,float16,3,0.30863680839538576
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,8,8,128,1,float16,float16,1,0.30853919982910155
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,8,128,1,float16,float16,16383,0.6763088226318359
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,8,8,128,1,float16,float16,7,0.32617599964141847
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,8,8,128,1,float16,float16,15,0.32367041110992434
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,8,8,128,1,float16,float16,31,0.3236288070678711
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,8,8,128,1,float16,float16,63,0.3337120056152344
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,8,8,128,1,float16,float16,127,0.48616800308227537
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,8,8,128,1,float16,float16,3,0.5991568088531494
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,8,8,128,1,float16,float16,1,0.5964064121246337
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,8,8,128,1,float16,float16,7,0.6132512092590332
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,8,8,128,1,float16,float16,255,0.8432751655578613
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,8,8,128,1,float16,float16,15,0.6148816108703613
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,8,8,128,1,float16,float16,31,0.6240143775939941
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,8,8,128,1,float16,float16,63,0.6439680099487305
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,8,128,1,float16,float16,1,0.016288000345230102
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,8,128,1,float16,float16,3,0.01626559942960739
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,8,128,1,float16,float16,7,0.01642400026321411
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,8,128,1,float16,float16,15,0.016392000019550323
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,8,8,128,1,float16,float16,511,2.1248096466064452
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,8,128,1,float16,float16,31,0.016515199840068818
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,8,128,1,float16,float16,63,0.01696320027112961
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,8,8,128,1,float16,float16,127,0.9871744155883789
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,8,128,1,float16,float16,255,0.029139199852943422
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,8,128,1,float16,float16,127,0.0217616006731987
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,8,128,1,float16,float16,511,0.06882399916648865
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,8,128,1,float16,float16,1023,0.11267679929733276
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,8,128,1,float16,float16,2047,0.19733599424362183
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,8,128,1,float16,float16,3,0.03115360140800476
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,8,128,1,float16,float16,4095,0.35239520072937014
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,8,8,128,1,float16,float16,255,1.627614402770996
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,8,128,1,float16,float16,8191,0.6777488231658936
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,8,128,1,float16,float16,16383,1.3422431945800781
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,4,128,1,float16,float16,1,0.023464000225067137
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,4,128,1,float16,float16,63,0.023966400325298308
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,4,128,1,float16,float16,127,0.030329599976539612
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,4,128,1,float16,float16,255,0.04547840058803558
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,4,128,1,float16,float16,511,0.09930239915847779
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,4,128,1,float16,float16,1023,0.1653615951538086
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,4,128,1,float16,float16,2047,0.28531999588012696
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,8,128,1,float16,float16,15,0.03138239979743958
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,4,128,1,float16,float16,4095,0.5273888111114502
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,4,128,1,float16,float16,16383,1.986412811279297
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,4,128,1,float16,float16,3,0.023417599499225616
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,4,128,1,float16,float16,1,0.013984000682830811
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,4,128,1,float16,float16,3,0.014027200639247894
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,4,128,1,float16,float16,7,0.014033600687980652
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,4,128,1,float16,float16,15,0.013998399674892425
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,4,128,1,float16,float16,31,0.01401440054178238
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,4,128,1,float16,float16,63,0.014083200693130493
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,4,128,1,float16,float16,127,0.016601599752902985
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,4,128,1,float16,float16,255,0.021510399878025055
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,4,128,1,float16,float16,1023,0.02778880000114441
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,4,128,1,float16,float16,2047,0.02826879918575287
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,4,128,1,float16,float16,4095,0.029071998596191407
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,4,128,1,float16,float16,8191,0.03795520067214966
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,4,128,1,float16,float16,16383,0.05545120239257813
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,4,128,1,float16,float16,32767,0.08249279856681824
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,4,128,1,float16,float16,1,0.013787199556827546
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,4,128,1,float16,float16,3,0.01403679996728897
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,4,128,1,float16,float16,7,0.014032000303268432
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,4,128,1,float16,float16,15,0.019249600172042847
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,4,128,1,float16,float16,31,0.014022399485111237
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,8,128,1,float16,float16,31,0.03158400058746338
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,4,128,1,float16,float16,63,0.014099200069904328
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,4,128,1,float16,float16,255,0.02141599953174591
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,4,128,1,float16,float16,32767,3.972119903564453
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,4,128,1,float16,float16,511,0.024004800617694853
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,4,128,1,float16,float16,2047,0.02915999889373779
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,4,128,1,float16,float16,1023,0.029084798693656922
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,4,128,1,float16,float16,4095,0.03797119855880737
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,4,128,1,float16,float16,8191,0.05670560002326965
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,4,128,1,float16,float16,16383,0.0791263997554779
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,4,128,1,float16,float16,1,0.016920000314712524
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,4,128,1,float16,float16,32767,0.1318112015724182
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,4,128,1,float16,float16,3,0.01677280068397522
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,4,128,1,float16,float16,15,0.016952000558376312
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,4,128,1,float16,float16,7,0.017367999255657195
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,4,128,1,float16,float16,31,0.01717119961977005
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,4,128,1,float16,float16,127,0.021612800657749176
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,4,128,1,float16,float16,63,0.017444799840450286
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,4,128,1,float16,float16,255,0.029838401079177856
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,4,128,1,float16,float16,511,0.07075999975204468
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,4,128,1,float16,float16,1023,0.11407999992370606
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,4,128,1,float16,float16,511,0.024099199473857878
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,4,128,1,float16,float16,2047,0.19720799922943116
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,4,128,1,float16,float16,4095,0.35381760597229006
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,4,128,1,float16,float16,8191,0.6811967849731445
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,4,128,1,float16,float16,1,0.014227199554443359
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,4,128,1,float16,float16,3,0.014300799369812012
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,4,128,1,float16,float16,7,0.01419519931077957
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,4,128,1,float16,float16,15,0.014256000518798828
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,4,128,1,float16,float16,31,0.01440960019826889
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,4,128,1,float16,float16,63,0.01435839980840683
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,4,128,1,float16,float16,127,0.016945600509643555
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,4,128,1,float16,float16,255,0.021964800357818604
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,4,128,1,float16,float16,16383,1.336571216583252
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,4,128,1,float16,float16,511,0.024711999297142028
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,4,128,1,float16,float16,2047,0.03771840035915375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,4,128,1,float16,float16,1023,0.02914400100708008
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,4,128,1,float16,float16,4095,0.05520960092544556
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,4,128,1,float16,float16,8191,0.08270559906959533
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,4,128,1,float16,float16,16383,0.21429600715637206
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,4,128,1,float16,float16,1,0.03222239911556244
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,4,128,1,float16,float16,32767,2.6184623718261717
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,4,128,1,float16,float16,3,0.03209120035171509
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,4,128,1,float16,float16,32767,0.3986128091812134
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,4,128,1,float16,float16,15,0.0325872004032135
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,4,128,1,float16,float16,31,0.03263840079307556
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,4,128,1,float16,float16,7,0.03215680122375488
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,4,128,1,float16,float16,63,0.033432000875473024
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,4,128,1,float16,float16,127,0.04948320090770721
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,4,128,1,float16,float16,255,0.07293760180473327
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,4,128,1,float16,float16,511,0.15953919887542725
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,4,128,1,float16,float16,1023,0.27009119987487795
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,4,128,1,float16,float16,2047,0.4699808120727539
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,4,128,1,float16,float16,4095,0.8701871871948242
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,4,128,1,float16,float16,7,0.023464000225067137
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,4,4,128,1,float16,float16,1,0.05198079943656921
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,4,4,128,1,float16,float16,3,0.05147839784622192
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,4,4,128,1,float16,float16,7,0.05236160159111023
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,4,4,128,1,float16,float16,15,0.05179200172424316
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,4,4,128,1,float16,float16,31,0.053092801570892335
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,4,4,128,1,float16,float16,63,0.06074879765510559
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,4,4,128,1,float16,float16,127,0.08131999969482422
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,4,128,1,float16,float16,8191,1.707756805419922
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,4,4,128,1,float16,float16,255,0.12445919513702393
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,4,4,128,1,float16,float16,511,0.30645439624786375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,4,4,128,1,float16,float16,1023,0.5208240032196045
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,4,4,128,1,float16,float16,2047,0.929535961151123
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,4,4,128,1,float16,float16,1,0.09392160177230835
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,4,128,1,float16,float16,16383,3.482024002075195
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,4,4,128,1,float16,float16,3,0.09390879869461059
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,4,4,128,1,float16,float16,15,0.09435679912567138
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,4,4,128,1,float16,float16,7,0.09941920042037963
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,4,4,128,1,float16,float16,31,0.09755520224571228
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,4,4,128,1,float16,float16,63,0.11020159721374512
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,4,4,128,1,float16,float16,127,0.1452448010444641
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,4,4,128,1,float16,float16,4095,1.728615951538086
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,4,4,128,1,float16,float16,255,0.23686881065368653
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,4,128,1,float16,float16,15,0.023553599417209626
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,4,128,1,float16,float16,1,0.014284799993038177
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,4,4,128,1,float16,float16,511,0.5594944000244141
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,4,128,1,float16,float16,3,0.01419840008020401
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,4,128,1,float16,float16,7,0.014336000382900237
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,4,128,1,float16,float16,15,0.014132800698280334
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,4,4,128,1,float16,float16,1023,0.9928336143493652
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,4,128,1,float16,float16,31,0.014214399456977844
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,4,128,1,float16,float16,63,0.014470399916172027
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,4,128,1,float16,float16,255,0.02191839963197708
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,4,128,1,float16,float16,127,0.017236800491809846
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,4,128,1,float16,float16,1023,0.03817920088768005
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,4,128,1,float16,float16,2047,0.05600960254669189
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,4,128,1,float16,float16,511,0.029179200530052185
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,4,128,1,float16,float16,8191,0.13763519525527954
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,4,128,1,float16,float16,4095,0.08293439745903015
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,4,4,128,1,float16,float16,2047,1.7574880599975586
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,4,128,1,float16,float16,16383,0.35185918807983396
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,4,4,128,1,float16,float16,1,0.1653615951538086
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,4,4,128,1,float16,float16,7,0.1656607985496521
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,4,4,128,1,float16,float16,3,0.17446719408035277
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,4,4,128,1,float16,float16,15,0.16713759899139405
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,4,128,1,float16,float16,32767,0.683787202835083
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,4,4,128,1,float16,float16,31,0.17451200485229493
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,4,4,128,1,float16,float16,63,0.1875056028366089
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,4,4,128,1,float16,float16,127,0.2598432064056396
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,4,4,128,1,float16,float16,255,0.44534077644348147
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,4,4,128,1,float16,float16,1,0.30881600379943847
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,4,4,128,1,float16,float16,3,0.31146879196166993
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,4,4,128,1,float16,float16,511,1.0820832252502441
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,4,4,128,1,float16,float16,7,0.3126960039138794
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,4,4,128,1,float16,float16,15,0.32196478843688964
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,4,4,128,1,float16,float16,31,0.32675039768218994
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,4,4,128,1,float16,float16,63,0.33942399024963377
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,4,128,1,float16,float16,31,0.023665599524974823
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,4,128,1,float16,float16,1,0.014390400052070618
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,4,4,128,1,float16,float16,127,0.486521577835083
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,4,128,1,float16,float16,7,0.014441600441932679
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,4,128,1,float16,float16,3,0.014388799667358398
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,4,128,1,float16,float16,31,0.014619199931621552
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,4,128,1,float16,float16,15,0.014431999623775482
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,4,128,1,float16,float16,63,0.014619199931621552
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,4,128,1,float16,float16,127,0.017238399386405943
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,4,128,1,float16,float16,255,0.022168000042438508
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,4,128,1,float16,float16,511,0.04743199944496155
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,4,128,1,float16,float16,1023,0.07179679870605468
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,4,4,128,1,float16,float16,1023,1.9657232284545898
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,4,128,1,float16,float16,2047,0.1121135950088501
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,4,4,128,1,float16,float16,255,0.8377936363220215
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,4,128,1,float16,float16,4095,0.19284319877624512
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,4,128,1,float16,float16,8191,0.3562608003616333
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,4,128,1,float16,float16,16383,0.6956816196441651
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,4,4,128,1,float16,float16,511,2.1327728271484374
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,4,128,1,float16,float16,32767,1.3232671737670898
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,1,0.01740960031747818
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,127,0.022196799516677856
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,255,0.03033440113067627
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,511,0.07104480266571045
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,1023,0.11561280488967896
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,4,128,1,float16,float16,8191,1.0148143768310547
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,2047,0.19534560441970825
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,8191,0.6807328224182129
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,16383,1.3280015945434571
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,15,0.017752000689506532
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,4,128,1,float16,float16,127,0.016755199432373045
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,32767,2.6231727600097656
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,3,0.013910399377346038
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,7,0.013947199285030364
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,15,0.01404000073671341
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,31,0.013910399377346038
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,63,0.014103999733924866
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,127,0.01666560024023056
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,255,0.021376000344753267
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,511,0.023680000007152556
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,3,0.017569600045681
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,1023,0.027988800406455995
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,2047,0.028014400601387025
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,4095,0.028518399596214293
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,8191,0.031374400854110716
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,16383,0.040612798929214475
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,32767,0.05668479800224304
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,1,0.013955199718475341
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,65535,0.0853712022304535
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,3,0.013873599469661713
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,7,0.013737599551677703
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,15,0.013980799913406372
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,65535,5.208358383178711
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,31,0.01383039951324463
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,127,0.01648000031709671
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,255,0.021516799926757812
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,63,0.014155200123786927
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,511,0.02396160066127777
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,2047,0.02863360047340393
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,1023,0.02820959985256195
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,4095,0.02895520031452179
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,8191,0.040110400319099425
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,16383,0.05362880229949951
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,32767,0.08259040117263794
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,65535,0.1344864010810852
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,1,0.014982399344444276
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,3,0.014945599436759948
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,7,0.015068799257278442
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,15,0.014951999485492706
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,31,0.015212799608707427
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,63,0.015246400237083435
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,7,0.01759359985589981
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,255,0.022593599557876588
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,127,0.017859199643135072
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,1023,0.07208639979362488
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,511,0.04815520048141479
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,2047,0.11395519971847534
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,4095,0.19213440418243408
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,8191,0.3578416109085083
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,16383,0.6847568035125733
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,1,0.013899199664592743
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,3,0.01387840062379837
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,7,0.01401119977235794
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,32767,1.3279791831970216
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,15,0.014004799723625182
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,31,0.01404000073671341
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,63,0.01401599943637848
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,127,0.016599999368190767
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,255,0.021521599590778352
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,1023,0.028948798775672913
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,511,0.024396799504756927
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,65535,2.621326446533203
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,2047,0.02943199872970581
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,4095,0.037880000472068784
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,8191,0.05780959725379944
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,1,0.013737599551677703
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,16383,0.08211680054664612
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,32767,0.13111040592193604
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,2,2,128,1,float16,float16,1,0.024512000381946564
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,2,2,128,1,float16,float16,3,0.024507200717926024
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,2,2,128,1,float16,float16,7,0.024500800669193266
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,2,2,128,1,float16,float16,15,0.024403199553489685
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,2,2,128,1,float16,float16,31,0.024799999594688416
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,2,2,128,1,float16,float16,63,0.025003200769424437
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,2,2,128,1,float16,float16,127,0.0312175989151001
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,65535,0.23657119274139404
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,2,2,128,1,float16,float16,255,0.047417598962783816
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,2,2,128,1,float16,float16,1023,0.1675663948059082
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,2,2,128,1,float16,float16,511,0.1015504002571106
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,2,2,128,1,float16,float16,2047,0.28573439121246336
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,2,2,128,1,float16,float16,1,0.03371039927005768
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,2,2,128,1,float16,float16,4095,0.528715181350708
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,2,2,128,1,float16,float16,3,0.033713600039482115
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,2,2,128,1,float16,float16,15,0.03390879929065704
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,2,2,128,1,float16,float16,7,0.03385440111160278
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,2,2,128,1,float16,float16,31,0.03429119884967804
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,2,2,128,1,float16,float16,63,0.035043200850486754
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,2,2,128,1,float16,float16,8191,1.0172752380371093
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,2,2,128,1,float16,float16,127,0.04922080039978027
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,2,2,128,1,float16,float16,255,0.07261279821395875
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,2,2,128,1,float16,float16,511,0.1625983953475952
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,2,2,128,1,float16,float16,1023,0.2707376003265381
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,2,2,128,1,float16,float16,2047,0.4718783855438232
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,2,2,128,1,float16,float16,16383,1.9923999786376954
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,2,2,128,1,float16,float16,1,0.05572800040245056
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,2,2,128,1,float16,float16,4095,0.875494384765625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,2,2,128,1,float16,float16,3,0.05638239979743957
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,2,2,128,1,float16,float16,7,0.05586720108985901
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,2,2,128,1,float16,float16,15,0.05617920160293579
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,2,2,128,1,float16,float16,31,0.05734879970550537
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,2,2,128,1,float16,float16,127,0.08468959927558899
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,2,2,128,1,float16,float16,63,0.06493279933929444
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,63,0.01809120029211044
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,2,2,128,1,float16,float16,255,0.12938079833984376
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,2,2,128,1,float16,float16,511,0.3088320016860962
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,2,2,128,1,float16,float16,32767,3.9600318908691405
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,2,2,128,1,float16,float16,8191,1.6941120147705078
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,1,0.014212800562381745
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,2,2,128,1,float16,float16,1023,0.5295792102813721
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,15,0.014179199934005737
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,3,0.014528000354766845
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,63,0.014233599603176116
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,31,0.014156800508499146
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,7,0.014552000164985656
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,127,0.016964800655841827
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,2,2,128,1,float16,float16,2047,0.9270319938659668
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,255,0.021879999339580535
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,1023,0.029894399642944335
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,511,0.02489120066165924
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,2047,0.038299199938774106
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,4095,0.055396801233291625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,8191,0.08488799929618836
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,16383,0.22046079635620117
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,2,2,128,1,float16,float16,1,0.09331200122833253
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,2,2,128,1,float16,float16,7,0.09352800250053406
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,2,2,128,1,float16,float16,3,0.09429280161857605
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,32767,0.403934383392334
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,2,2,128,1,float16,float16,15,0.09385600090026855
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,2,2,128,1,float16,float16,4095,1.74310245513916
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,2,2,128,1,float16,float16,31,0.09818080067634583
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,2,2,128,1,float16,float16,63,0.10629279613494873
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,65535,0.7767439842224121
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,2,2,128,1,float16,float16,127,0.14126399755477906
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,2,2,128,1,float16,float16,255,0.2346400022506714
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,2,2,128,1,float16,float16,1,0.16531519889831542
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,2,2,128,1,float16,float16,7,0.16568959951400758
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,2,2,128,1,float16,float16,3,0.16920959949493408
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,2,2,128,1,float16,float16,511,0.5715919971466065
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,2,2,128,1,float16,float16,15,0.16640640497207643
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,2,2,128,1,float16,float16,31,0.17886719703674317
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,2,2,128,1,float16,float16,63,0.1860767960548401
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,2,2,128,1,float16,float16,1023,0.9964896202087402
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,2,2,128,1,float16,float16,127,0.2565887928009033
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,1,0.014086399972438813
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,2,2,128,1,float16,float16,255,0.4396240234375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,3,0.014079999923706055
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,7,0.01440960019826889
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,15,0.01430719941854477
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,2,2,128,1,float16,float16,2047,1.7442960739135742
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,31,0.014539200067520141
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,63,0.014404800534248353
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,2,2,128,1,float16,float16,511,1.107372760772705
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,255,0.022233599424362184
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,511,0.02961440086364746
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,127,0.01741439998149872
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,2047,0.05591520071029663
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,1023,0.03861440122127533
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,4095,0.08288159966468811
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,8191,0.14146560430526733
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,16383,0.3562191963195801
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,31,0.01780800074338913
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,32767,0.6763199806213379
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,2,2,128,1,float16,float16,1023,1.974977684020996
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,65535,1.3232992172241211
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,1,0.015520000457763672
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,255,0.023369599878787995
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,511,0.04862880110740662
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,4095,0.3558063983917236
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,1023,0.07354720234870911
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,4095,0.19328960180282592
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,8191,0.3583631992340088
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,16383,0.6871695995330811
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,31,0.015732799470424653
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,32767,1.3329808235168457
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,65535,2.6329599380493165
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,1,0.013864000141620637
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,3,0.013916799426078796
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,7,0.013923199474811554
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,15,0.013799999654293061
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,31,0.014000000059604644
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,2047,0.11471840143203735
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,63,0.014084799587726593
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,127,0.016756799817085267
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,255,0.021376000344753267
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,511,0.02383680045604706
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,1023,0.027745598554611207
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,2047,0.02818399965763092
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,4095,0.028142398595809935
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,8191,0.031179198622703554
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,32767,0.04537599980831146
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,16383,0.03609279990196228
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,65535,0.061924797296524045
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,1,0.014035199582576752
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,131071,0.09078559875488282
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,3,0.014023999869823455
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,7,0.013969600200653076
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,15,0.014116799831390381
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,63,0.01414719969034195
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,31,0.013948799669742584
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,127,0.01679680049419403
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,511,0.024089600145816802
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,255,0.021372799575328828
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,1023,0.028271999955177308
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,2047,0.028465598821640015
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,4095,0.02889919877052307
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,131071,5.210684967041016
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,8191,0.0314191997051239
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,16383,0.041257598996162416
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,32767,0.05929279923439026
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,131071,0.13964320421218873
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,65535,0.08637760281562805
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,1,0.014640000462532044
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,3,0.014921599626541137
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,15,0.01496960073709488
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,7,0.014878399670124054
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,31,0.015003199875354766
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,63,0.01483680009841919
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,255,0.02255360037088394
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,127,0.017627200484275816
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,1023,0.039073601365089417
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,511,0.030716800689697267
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,2047,0.05624160170555115
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,4095,0.08391039967536926
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,8191,0.13906559944152833
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,63,0.015932799875736238
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,16383,0.3610912084579468
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,32767,0.6800640106201172
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,1,0.014102399349212646
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,3,0.014056000113487243
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,7,0.013984000682830811
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,15,0.014113600552082061
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,65535,1.3332752227783202
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,31,0.014256000518798828
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,127,0.016791999340057373
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,63,0.014120000600814819
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,255,0.02179519981145859
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,511,0.02417600005865097
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,1023,0.028446400165557863
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,4095,0.02933279871940613
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,2047,0.02871519923210144
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,8191,0.04019840061664581
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,16383,0.05948479771614075
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,32767,0.08536959886550903
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,131071,0.24238080978393556
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,65535,0.1410752058029175
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,1,0.018559999763965607
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,131071,2.629960060119629
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,7,0.01863040030002594
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,3,0.018535999953746794
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,31,0.01882079988718033
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,63,0.019023999571800232
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,15,0.018673600256443025
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,127,0.023337599635124207
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,255,0.03127039968967438
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,511,0.07327839732170105
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,1023,0.11657119989395141
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,2047,0.19608479738235474
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,4095,0.35719358921051025
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,8191,0.6828559875488281
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,3,0.015732799470424653
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,16383,1.3281696319580079
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,1,1,128,1,float16,float16,1,0.026092800498008727
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,1,1,128,1,float16,float16,7,0.026364800333976746
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,1,1,128,1,float16,float16,3,0.026148799061775207
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,1,1,128,1,float16,float16,15,0.026372799277305604
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,1,1,128,1,float16,float16,31,0.0263264000415802
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,1,1,128,1,float16,float16,63,0.026804798841476442
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,1,1,128,1,float16,float16,127,0.03307999968528748
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,1,1,128,1,float16,float16,255,0.048712000250816345
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,1,1,128,1,float16,float16,1023,0.17059040069580078
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,1,1,128,1,float16,float16,511,0.10513119697570801
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,1,1,128,1,float16,float16,2047,0.29000320434570315
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,1,1,128,1,float16,float16,4095,0.5314032077789307
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,32767,2.623161506652832
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,1,1,128,1,float16,float16,1,0.038001599907875064
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,1,1,128,1,float16,float16,3,0.03804959952831268
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,1,1,128,1,float16,float16,7,0.038155201077461245
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,1,1,128,1,float16,float16,15,0.03838399946689606
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,1,1,128,1,float16,float16,31,0.038683199882507326
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,1,1,128,1,float16,float16,63,0.039478400349617006
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,1,1,128,1,float16,float16,8191,1.0223088264465332
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,1,1,128,1,float16,float16,127,0.054499202966690065
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,1,1,128,1,float16,float16,255,0.07725279927253723
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,1,1,128,1,float16,float16,511,0.17136640548706056
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,1,1,128,1,float16,float16,1023,0.2794784069061279
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,1,1,128,1,float16,float16,16383,1.9956720352172852
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,1,1,128,1,float16,float16,2047,0.4799071788787842
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,65535,5.212030410766602
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,3,0.014022399485111237
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,1,0.014292800426483154
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,7,0.014259199798107147
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,15,0.014123199880123139
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,31,0.014185599982738495
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,63,0.014406399428844452
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,127,0.016809600591659545
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,1,1,128,1,float16,float16,4095,0.8811936378479004
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,255,0.02175839990377426
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,1023,0.02871200144290924
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,511,0.024347199499607085
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,2047,0.029168000817298888
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,4095,0.03858560025691986
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,8191,0.057817602157592775
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,16383,0.08694239854812622
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,32767,0.13661439418792726
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,1,1,128,1,float16,float16,1,0.0562175989151001
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,1,1,128,1,float16,float16,8191,1.7013984680175782
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,1,1,128,1,float16,float16,3,0.05617759823799133
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,65535,0.2391279935836792
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,1,1,128,1,float16,float16,7,0.05637440085411072
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,1,1,128,1,float16,float16,31,0.05711039900779724
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,131071,0.4446864128112793
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,1,1,128,1,float16,float16,15,0.05644479990005493
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,1,1,128,1,float16,float16,63,0.06509600281715393
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,1,1,128,1,float16,float16,255,0.127510404586792
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,1,1,128,1,float16,float16,127,0.08480319976806641
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,1,1,128,1,float16,float16,511,0.3174207925796509
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,1,1,128,1,float16,float16,1,0.0936735987663269
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,1,1,128,1,float16,float16,7,0.09377279877662659
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,1,1,128,1,float16,float16,3,0.09338719844818115
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,1,1,128,1,float16,float16,15,0.09429919719696045
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,7,0.01582240015268326
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,1,1,128,1,float16,float16,1023,0.5379487991333007
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,1,1,128,1,float16,float16,63,0.10507040023803711
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,1,1,128,1,float16,float16,31,0.0969103991985321
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,1,1,128,1,float16,float16,2047,0.9364336013793946
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,1,1,128,1,float16,float16,127,0.14245280027389526
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,1,1,128,1,float16,float16,255,0.2254256010055542
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,1,0.014248000085353851
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,3,0.014193600416183472
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,7,0.014227199554443359
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,1,1,128,1,float16,float16,4095,1.7602096557617188
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,1,1,128,1,float16,float16,511,0.5885263919830322
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,15,0.014232000708580017
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,127,0.01703200042247772
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,31,0.01478080004453659
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,63,0.014425599575042724
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,255,0.021862399578094483
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,1,1,128,1,float16,float16,1023,1.000928020477295
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,511,0.02601439952850342
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,1023,0.029500800371170043
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,2047,0.03816800117492676
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,4095,0.056062400341033936
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,8191,0.08785120248794556
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,16383,0.22116799354553224
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,32767,0.40230717658996584
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,1,1,128,1,float16,float16,2047,1.7791183471679688
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,1,128,1,float16,float16,1,0.01703519970178604
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,1,128,1,float16,float16,3,0.01661120057106018
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,1,128,1,float16,float16,15,0.016993600130081176
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,1,128,1,float16,float16,7,0.016771200299263
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,1,128,1,float16,float16,31,0.016816000640392303
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,1,128,1,float16,float16,63,0.01701119989156723
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,1,128,1,float16,float16,127,0.019524799287319185
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,65535,0.7799903869628906
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,1,128,1,float16,float16,255,0.024447999894618988
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,2,128,1,float16,float16,1,0.018086400628089905
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,1,128,1,float16,float16,511,0.06518080234527587
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,1,128,1,float16,float16,1023,0.08960000276565552
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,2,128,1,float16,float16,3,0.018313600122928618
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,1,128,1,float16,float16,2047,0.13072160482406617
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,2,128,1,float16,float16,7,0.018251200020313264
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,2,128,1,float16,float16,31,0.01828320026397705
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,2,128,1,float16,float16,15,0.018454399704933167
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,2,128,1,float16,float16,63,0.018636800348758698
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,2,128,1,float16,float16,127,0.022742399573326112
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,131071,1.5244447708129882
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,2,128,1,float16,float16,255,0.03139680027961731
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,2,128,1,float16,float16,511,0.07473919987678528
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,4,128,1,float16,float16,3,0.02393440008163452
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,4,128,1,float16,float16,1,0.024222399294376373
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,4,128,1,float16,float16,7,0.024260799586772918
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,2,128,1,float16,float16,1023,0.11802400350570678
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,4,128,1,float16,float16,15,0.02399040013551712
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,4,128,1,float16,float16,31,0.024238400161266327
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,2,128,1,float16,float16,2047,0.1977679967880249
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,4,128,1,float16,float16,63,0.024619199335575104
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,4,128,1,float16,float16,255,0.04950560033321381
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,4,128,1,float16,float16,127,0.03116160035133362
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,127,0.01849119961261749
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,8,128,1,float16,float16,1,0.03145120143890381
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,8,128,1,float16,float16,3,0.03117760121822357
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,4,128,1,float16,float16,1023,0.16809920072555543
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,4,128,1,float16,float16,511,0.10384000539779663
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,8,128,1,float16,float16,15,0.03165439963340759
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,8,128,1,float16,float16,7,0.03130879998207092
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,8,128,1,float16,float16,31,0.031839999556541446
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,4,128,1,float16,float16,2047,0.29125759601593015
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,8,128,1,float16,float16,127,0.0502623975276947
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,8,128,1,float16,float16,255,0.07173600196838378
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,8,128,1,float16,float16,63,0.033267199993133545
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,1,128,1,float16,float16,1,0.014454400539398194
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,1,128,1,float16,float16,3,0.014774399995803832
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,1,128,1,float16,float16,7,0.014643199741840363
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,8,128,1,float16,float16,511,0.15824480056762696
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,1,128,1,float16,float16,15,0.014451199769973755
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,8,128,1,float16,float16,1023,0.2676559925079346
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,1,128,1,float16,float16,31,0.014713600277900696
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,1,128,1,float16,float16,63,0.014801600575447082
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,1,128,1,float16,float16,127,0.01711200028657913
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,1,128,1,float16,float16,1023,0.02941280007362366
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,1,128,1,float16,float16,255,0.021934400498867034
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,1,128,1,float16,float16,511,0.024942399561405183
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,2,128,1,float16,float16,1,0.014204800128936768
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,2,128,1,float16,float16,3,0.014505599439144135
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,1,128,1,float16,float16,2047,0.030001598596572875
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,2,128,1,float16,float16,15,0.01414719969034195
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,2,128,1,float16,float16,7,0.014153599739074707
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,2,128,1,float16,float16,31,0.014151999354362487
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,2,128,1,float16,float16,63,0.014424000680446625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,8,128,1,float16,float16,2047,0.46831521987915037
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,2,128,1,float16,float16,127,0.017182399332523347
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,2,128,1,float16,float16,255,0.021695999801158904
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,2,128,1,float16,float16,1023,0.027849599719047546
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,2,128,1,float16,float16,511,0.023470400273799895
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,2,128,1,float16,float16,2047,0.028585600852966308
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,4,128,1,float16,float16,3,0.014241600036621093
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,4,128,1,float16,float16,7,0.014035199582576752
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,4,128,1,float16,float16,15,0.014217600226402283
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,4,128,1,float16,float16,1,0.014131200313568116
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,4,128,1,float16,float16,31,0.014132800698280334
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,4,128,1,float16,float16,63,0.014392000436782838
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,4,128,1,float16,float16,127,0.016991999745368958
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,4,128,1,float16,float16,255,0.021635200083255767
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,4,128,1,float16,float16,511,0.023345600068569183
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,8,128,1,float16,float16,1,0.014257599413394929
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,8,128,1,float16,float16,3,0.013974399864673614
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,8,128,1,float16,float16,7,0.014095999300479889
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,4,128,1,float16,float16,2047,0.028352001309394838
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,4,128,1,float16,float16,1023,0.02738879919052124
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,8,128,1,float16,float16,15,0.014291200041770934
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,8,128,1,float16,float16,31,0.014068800210952758
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,8,128,1,float16,float16,63,0.014083200693130493
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,8,128,1,float16,float16,127,0.016923199594020843
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,8,128,1,float16,float16,511,0.023284800350666046
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,8,128,1,float16,float16,255,0.021665599942207337
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,8,128,1,float16,float16,1023,0.027662399411201476
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,8,128,1,float16,float16,2047,0.02852639853954315
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,1,128,1,float16,float16,1,0.014584000408649444
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,1,128,1,float16,float16,3,0.014776000380516052
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,1,128,1,float16,float16,15,0.014582400023937226
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,1,128,1,float16,float16,7,0.014404800534248353
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,1,128,1,float16,float16,31,0.014532800018787383
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,1,128,1,float16,float16,63,0.014825600385665893
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,1,128,1,float16,float16,127,0.01743520051240921
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,1,128,1,float16,float16,255,0.02202560007572174
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,2,128,1,float16,float16,1,0.014496000111103058
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,1,128,1,float16,float16,511,0.02513119876384735
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,2,128,1,float16,float16,3,0.014238399267196656
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,1,128,1,float16,float16,2047,0.03012160062789917
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,1,128,1,float16,float16,1023,0.029337599873542786
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,2,128,1,float16,float16,7,0.014577600359916686
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,2,128,1,float16,float16,15,0.014120000600814819
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,2,128,1,float16,float16,31,0.01422400027513504
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,2,128,1,float16,float16,127,0.016846400499343873
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,2,128,1,float16,float16,63,0.014724799990653991
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,2,128,1,float16,float16,255,0.02202560007572174
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,2,128,1,float16,float16,511,0.02398719936609268
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,2,128,1,float16,float16,1023,0.0280239999294281
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,4,128,1,float16,float16,3,0.014289599657058717
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,4,128,1,float16,float16,1,0.014124800264835358
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,2,128,1,float16,float16,2047,0.028921601176261903
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,4,128,1,float16,float16,7,0.014168000221252442
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,4,128,1,float16,float16,31,0.014431999623775482
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,4,128,1,float16,float16,63,0.014468799531459808
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,4,128,1,float16,float16,15,0.014201599359512328
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,4,128,1,float16,float16,127,0.016969600319862367
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,4,128,1,float16,float16,511,0.023528000712394713
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,4,128,1,float16,float16,255,0.02181279957294464
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,4,128,1,float16,float16,1023,0.028174400329589844
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,4,128,1,float16,float16,2047,0.029142400622367857
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,8,128,1,float16,float16,1,0.014078399538993836
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,8,128,1,float16,float16,3,0.014102399349212646
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,8,128,1,float16,float16,7,0.014097599685192109
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,8,128,1,float16,float16,15,0.014209599792957306
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,8,128,1,float16,float16,63,0.01438080072402954
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,8,128,1,float16,float16,127,0.01693760007619858
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,8,128,1,float16,float16,31,0.0143327996134758
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,8,128,1,float16,float16,255,0.021831999719142913
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,8,128,1,float16,float16,511,0.02398560047149658
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,8,128,1,float16,float16,1023,0.028139200806617738
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,1,128,1,float16,float16,1,0.01560479998588562
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,8,128,1,float16,float16,2047,0.03742879927158356
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,1,128,1,float16,float16,3,0.015678399801254274
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,1,128,1,float16,float16,7,0.015497599542140961
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,1,128,1,float16,float16,15,0.015547199547290802
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,1,128,1,float16,float16,31,0.01565600037574768
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,1,128,1,float16,float16,63,0.015668800473213194
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,1,128,1,float16,float16,127,0.018335999548435213
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,1,128,1,float16,float16,255,0.023209600150585173
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,2,128,1,float16,float16,1,0.015158399939537048
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,1,128,1,float16,float16,511,0.035364800691604616
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,2,128,1,float16,float16,3,0.015468800067901611
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,1,128,1,float16,float16,1023,0.04626240134239197
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,1,128,1,float16,float16,2047,0.06242079734802246
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,2,128,1,float16,float16,7,0.015388800203800202
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,2,128,1,float16,float16,15,0.015121600031852723
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,2,128,1,float16,float16,31,0.015440000593662262
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,2,128,1,float16,float16,127,0.018190400302410127
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,2,128,1,float16,float16,63,0.015591999888420105
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,2,128,1,float16,float16,255,0.022809599339962006
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,2,128,1,float16,float16,511,0.05242400169372559
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,2,128,1,float16,float16,1023,0.07775520086288452
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,4,128,1,float16,float16,1,0.01700959950685501
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,4,128,1,float16,float16,3,0.01727360039949417
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,4,128,1,float16,float16,7,0.017047999799251555
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,2,128,1,float16,float16,2047,0.11762720346450806
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,4,128,1,float16,float16,63,0.01767520010471344
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,4,128,1,float16,float16,15,0.01727519929409027
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,4,128,1,float16,float16,31,0.01730400025844574
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,4,128,1,float16,float16,127,0.021976000070571898
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,4,128,1,float16,float16,255,0.030001598596572875
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,4,128,1,float16,float16,511,0.07168319821357727
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,8,128,1,float16,float16,3,0.023052799701690673
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,8,128,1,float16,float16,1,0.022756800055503845
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,4,128,1,float16,float16,1023,0.11503679752349853
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,8,128,1,float16,float16,7,0.02306240051984787
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,4,128,1,float16,float16,2047,0.19455360174179076
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,8,128,1,float16,float16,31,0.023416000604629516
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,8,128,1,float16,float16,15,0.02316640019416809
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,8,128,1,float16,float16,63,0.023520000278949738
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,8,128,1,float16,float16,127,0.029787200689315795
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,8,128,1,float16,float16,255,0.046412798762321475
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,8,128,1,float16,float16,511,0.09849119782447815
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,1,128,1,float16,float16,1,0.014497600495815277
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,8,128,1,float16,float16,1023,0.16544320583343505
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,1,128,1,float16,float16,3,0.014686399698257446
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,1,128,1,float16,float16,15,0.014510400593280792
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,1,128,1,float16,float16,7,0.014679999649524688
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,1,128,1,float16,float16,31,0.014908799529075622
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,1,128,1,float16,float16,63,0.0147599995136261
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,1,128,1,float16,float16,127,0.017206400632858276
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,1,128,1,float16,float16,255,0.022327999770641326
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,8,128,1,float16,float16,2047,0.28315999507904055
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,1,128,1,float16,float16,1023,0.0295087993144989
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,2,128,1,float16,float16,1,0.014555199444293976
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,1,128,1,float16,float16,2047,0.03059520125389099
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,1,128,1,float16,float16,511,0.0255840003490448
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,2,128,1,float16,float16,15,0.014584000408649444
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,2,128,1,float16,float16,7,0.014375999569892883
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,2,128,1,float16,float16,31,0.014232000708580017
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,2,128,1,float16,float16,3,0.01419679969549179
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,2,128,1,float16,float16,63,0.014603200554847717
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,2,128,1,float16,float16,127,0.017220799624919892
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,2,128,1,float16,float16,255,0.021756799519062044
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,4,128,1,float16,float16,1,0.014420799911022186
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,2,128,1,float16,float16,511,0.02425439953804016
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,4,128,1,float16,float16,3,0.014113600552082061
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,2,128,1,float16,float16,2047,0.029905599355697633
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,2,128,1,float16,float16,1023,0.028619199991226196
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,4,128,1,float16,float16,7,0.014271999895572662
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,4,128,1,float16,float16,15,0.014289599657058717
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,4,128,1,float16,float16,31,0.014292800426483154
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,4,128,1,float16,float16,63,0.014494399726390838
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,4,128,1,float16,float16,255,0.022014400362968443
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,4,128,1,float16,float16,127,0.016791999340057373
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,4,128,1,float16,float16,511,0.024158400297164918
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,4,128,1,float16,float16,1023,0.02845599949359894
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,8,128,1,float16,float16,1,0.014368000626564025
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,8,128,1,float16,float16,3,0.014374400675296783
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,4,128,1,float16,float16,2047,0.03781760036945343
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,8,128,1,float16,float16,7,0.014407999813556671
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,8,128,1,float16,float16,31,0.014457599818706512
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,8,128,1,float16,float16,15,0.014348800480365752
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,8,128,1,float16,float16,63,0.014388799667358398
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,8,128,1,float16,float16,255,0.021971200406551362
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,8,128,1,float16,float16,127,0.017044800519943237
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,8,128,1,float16,float16,511,0.028681600093841554
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,1,128,1,float16,float16,1,0.020265600085258482
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,8,128,1,float16,float16,1023,0.0378928005695343
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,1,128,1,float16,float16,3,0.020608000457286835
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,8,128,1,float16,float16,2047,0.05390239953994751
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,1,128,1,float16,float16,7,0.0207056000828743
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,1,128,1,float16,float16,15,0.020452800393104553
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,1,128,1,float16,float16,63,0.020812800526618956
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,1,128,1,float16,float16,31,0.020502400398254395
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,1,128,1,float16,float16,127,0.02486239969730377
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,1,128,1,float16,float16,255,0.03365280032157898
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,2,128,1,float16,float16,1,0.026081600785255434
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,2,128,1,float16,float16,3,0.02590239942073822
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,1,128,1,float16,float16,511,0.07971519827842713
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,2,128,1,float16,float16,7,0.026067200303077697
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,2,128,1,float16,float16,15,0.02630240023136139
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,1,128,1,float16,float16,1023,0.12231520414352418
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,2,128,1,float16,float16,31,0.026131200790405273
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,2,128,1,float16,float16,63,0.026635199785232544
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,2,128,1,float16,float16,255,0.05797759890556335
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,2,128,1,float16,float16,127,0.03396640121936798
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,4,128,1,float16,float16,1,0.0328031986951828
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,4,128,1,float16,float16,3,0.032902398705482484
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,2,128,1,float16,float16,511,0.10672960281372071
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,15,0.015625600516796113
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,4,128,1,float16,float16,15,0.03323360085487366
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,4,128,1,float16,float16,7,0.032767999172210696
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,2,128,1,float16,float16,1023,0.17084159851074218
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,4,128,1,float16,float16,31,0.03351520001888275
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,4,128,1,float16,float16,63,0.03528960049152374
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,4,128,1,float16,float16,127,0.05244960188865662
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,8,128,1,float16,float16,3,0.05023999810218811
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,4,128,1,float16,float16,255,0.07959679961204529
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,8,128,1,float16,float16,7,0.0503711998462677
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,4,128,1,float16,float16,511,0.16244319677352906
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,8,128,1,float16,float16,1,0.05007359981536865
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,8,128,1,float16,float16,15,0.05089920163154602
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,8,128,1,float16,float16,31,0.052112001180648806
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,8,128,1,float16,float16,63,0.06097440123558044
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,4,128,1,float16,float16,1023,0.2711440086364746
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,8,128,1,float16,float16,255,0.12396960258483887
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,8,128,1,float16,float16,127,0.08106880187988282
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,128,1,128,1,float16,float16,1,0.03012160062789917
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,128,1,128,1,float16,float16,3,0.030622398853302
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,128,1,128,1,float16,float16,63,0.03205440044403076
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,128,1,128,1,float16,float16,31,0.03149600028991699
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,128,1,128,1,float16,float16,7,0.030472001433372496
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,8,128,1,float16,float16,511,0.29719839096069334
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,128,1,128,1,float16,float16,15,0.03089120090007782
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,128,1,128,1,float16,float16,127,0.04613119959831238
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,128,1,128,1,float16,float16,255,0.0658735990524292
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,128,2,128,1,float16,float16,1,0.03596160113811493
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,128,2,128,1,float16,float16,3,0.03601279854774475
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,128,2,128,1,float16,float16,31,0.037006399035453795
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,8,128,1,float16,float16,1023,0.5222256183624268
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,128,2,128,1,float16,float16,15,0.037676799297332766
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,128,2,128,1,float16,float16,7,0.036001598834991454
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,128,2,128,1,float16,float16,63,0.04720959961414337
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,128,2,128,1,float16,float16,127,0.05830079913139343
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,128,4,128,1,float16,float16,3,0.053014397621154785
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,128,4,128,1,float16,float16,15,0.053671997785568235
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,128,2,128,1,float16,float16,255,0.08455039858818054
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,128,4,128,1,float16,float16,7,0.05307040214538574
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,128,4,128,1,float16,float16,31,0.05726240277290344
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,128,4,128,1,float16,float16,63,0.06570240259170532
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,128,4,128,1,float16,float16,1,0.05351999998092651
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,128,4,128,1,float16,float16,127,0.08675680160522461
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,128,8,128,1,float16,float16,1,0.08946239948272705
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,128,8,128,1,float16,float16,3,0.08990079760551453
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,128,4,128,1,float16,float16,255,0.13069920539855956
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,128,8,128,1,float16,float16,7,0.09035040140151977
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,128,8,128,1,float16,float16,31,0.09679520130157471
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,128,8,128,1,float16,float16,15,0.0937936007976532
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,128,8,128,1,float16,float16,63,0.10283360481262208
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,128,1,128,1,float16,float16,1,0.05526880025863647
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,128,8,128,1,float16,float16,127,0.14346400499343873
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,128,1,128,1,float16,float16,3,0.055030399560928346
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,128,1,128,1,float16,float16,7,0.05602080225944519
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,128,1,128,1,float16,float16,15,0.05635679960250854
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,128,8,128,1,float16,float16,255,0.22728478908538818
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,128,1,128,1,float16,float16,31,0.05796639919281006
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,128,1,128,1,float16,float16,63,0.06178399920463562
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,128,1,128,1,float16,float16,127,0.07320160269737244
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,128,2,128,1,float16,float16,1,0.06778720021247864
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,128,2,128,1,float16,float16,3,0.06777920126914978
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,128,2,128,1,float16,float16,31,0.07019039988517761
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,128,2,128,1,float16,float16,15,0.06938400268554687
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,128,2,128,1,float16,float16,63,0.07549279928207397
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,128,2,128,1,float16,float16,7,0.06905279755592346
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,128,4,128,1,float16,float16,1,0.10145280361175538
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,128,4,128,1,float16,float16,3,0.10162400007247925
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,128,4,128,1,float16,float16,7,0.10353440046310425
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,128,2,128,1,float16,float16,127,0.09752960205078125
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,128,4,128,1,float16,float16,15,0.10530400276184082
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,128,4,128,1,float16,float16,31,0.10605920553207397
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,128,4,128,1,float16,float16,63,0.11208000183105468
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,128,8,128,1,float16,float16,1,0.17155200242996216
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,128,8,128,1,float16,float16,3,0.17274080514907836
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,128,4,128,1,float16,float16,127,0.15470720529556276
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,128,8,128,1,float16,float16,7,0.17412320375442505
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,128,8,128,1,float16,float16,15,0.17661919593811035
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,128,8,128,1,float16,float16,31,0.17901599407196045
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,128,8,128,1,float16,float16,63,0.18825919628143312
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,1,128,1,float16,float16,1,0.014500799775123595
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,1,128,1,float16,float16,3,0.014798399806022645
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,1,128,1,float16,float16,127,0.01711200028657913
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,1,128,1,float16,float16,7,0.014619199931621552
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,1,128,1,float16,float16,255,0.022310400009155275
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,1,128,1,float16,float16,15,0.014851200580596923
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,1,128,1,float16,float16,31,0.014547200500965118
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,1,128,1,float16,float16,63,0.014686399698257446
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,128,8,128,1,float16,float16,127,0.2701103925704956
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,1,128,1,float16,float16,1023,0.030931198596954347
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,1,128,1,float16,float16,511,0.026526400446891786
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,2,128,1,float16,float16,1,0.01451680064201355
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,1,128,1,float16,float16,2047,0.03257279992103577
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,2,128,1,float16,float16,3,0.014243200421333313
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,2,128,1,float16,float16,7,0.014345599710941315
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,2,128,1,float16,float16,31,0.014467200636863709
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,2,128,1,float16,float16,127,0.017246399819850922
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,2,128,1,float16,float16,63,0.014510400593280792
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,2,128,1,float16,float16,255,0.022124800086021423
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,2,128,1,float16,float16,1023,0.029969599843025208
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,2,128,1,float16,float16,15,0.014246399700641631
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,2,128,1,float16,float16,511,0.025775998830795288
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,4,128,1,float16,float16,1,0.014401599764823914
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,4,128,1,float16,float16,3,0.014363199472427368
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,2,128,1,float16,float16,2047,0.038913598656654357
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,4,128,1,float16,float16,7,0.014366400241851807
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,4,128,1,float16,float16,31,0.01440960019826889
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,4,128,1,float16,float16,15,0.014164799451828003
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,4,128,1,float16,float16,127,0.017203199863433837
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,4,128,1,float16,float16,63,0.014444799721240997
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,4,128,1,float16,float16,255,0.021932800114154816
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,4,128,1,float16,float16,511,0.029604798555374144
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,8,128,1,float16,float16,1,0.014419199526309967
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,4,128,1,float16,float16,1023,0.038980799913406375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,8,128,1,float16,float16,15,0.014448000490665436
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,8,128,1,float16,float16,3,0.014560000598430633
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,8,128,1,float16,float16,7,0.01433439999818802
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,4,128,1,float16,float16,2047,0.055486398935317996
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,8,128,1,float16,float16,31,0.014617599546909332
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,8,128,1,float16,float16,63,0.0147024005651474
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,8,128,1,float16,float16,255,0.022124800086021423
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,8,128,1,float16,float16,127,0.01738879978656769
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,8,128,1,float16,float16,511,0.04697119891643524
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,128,1,128,1,float16,float16,1,0.080622398853302
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,8,128,1,float16,float16,1023,0.07158560156822205
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,128,1,128,1,float16,float16,7,0.08402079939842225
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,128,1,128,1,float16,float16,3,0.0828320026397705
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,128,1,128,1,float16,float16,15,0.08472639918327332
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,8,128,1,float16,float16,2047,0.1118783950805664
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,128,1,128,1,float16,float16,31,0.08738240003585815
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,128,1,128,1,float16,float16,63,0.0941760003566742
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,128,2,128,1,float16,float16,7,0.11261440515518188
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,128,2,128,1,float16,float16,1,0.11072319746017456
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,128,2,128,1,float16,float16,3,0.1121664047241211
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,128,2,128,1,float16,float16,31,0.11545920372009277
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,128,2,128,1,float16,float16,15,0.11505279541015626
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,128,2,128,1,float16,float16,63,0.12189600467681885
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,128,4,128,1,float16,float16,1,0.1797984004020691
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,128,4,128,1,float16,float16,3,0.18002560138702392
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,128,4,128,1,float16,float16,15,0.18421759605407714
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,128,4,128,1,float16,float16,7,0.18275200128555297
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,128,4,128,1,float16,float16,31,0.19273760318756103
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,128,4,128,1,float16,float16,63,0.1973855972290039
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,128,8,128,1,float16,float16,3,0.3220223903656006
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,128,8,128,1,float16,float16,1,0.3211136102676392
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,128,8,128,1,float16,float16,7,0.3253024101257324
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,128,8,128,1,float16,float16,15,0.3298304080963135
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,128,8,128,1,float16,float16,31,0.3361263990402222
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,128,1,128,1,float16,float16,3,0.13148000240325927
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,128,1,128,1,float16,float16,1,0.13266079425811766
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,128,1,128,1,float16,float16,7,0.13412959575653077
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,128,8,128,1,float16,float16,63,0.35150721073150637
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,128,1,128,1,float16,float16,31,0.14139039516448976
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,128,1,128,1,float16,float16,15,0.13605600595474243
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,128,2,128,1,float16,float16,3,0.19612799882888793
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,128,2,128,1,float16,float16,1,0.19250080585479737
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,128,2,128,1,float16,float16,7,0.19793280363082885
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,128,2,128,1,float16,float16,15,0.20018401145935058
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,128,2,128,1,float16,float16,31,0.20287680625915527
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,128,4,128,1,float16,float16,1,0.3340575933456421
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,128,4,128,1,float16,float16,3,0.33509440422058107
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,128,4,128,1,float16,float16,7,0.340449595451355
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,128,4,128,1,float16,float16,15,0.3432192087173462
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,128,4,128,1,float16,float16,31,0.348524808883667
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,1,128,1,float16,float16,1,0.014864000678062438
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,1,128,1,float16,float16,3,0.014720000326633453
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,1,128,1,float16,float16,7,0.014694400131702423
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,128,8,128,1,float16,float16,1,0.6232336044311524
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,1,128,1,float16,float16,31,0.014766399562358857
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,1,128,1,float16,float16,15,0.014881600439548493
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,1,128,1,float16,float16,127,0.017556799948215483
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,1,128,1,float16,float16,63,0.014996799826622009
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,1,128,1,float16,float16,255,0.022339199483394623
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,128,8,128,1,float16,float16,3,0.6227471828460693
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,128,8,128,1,float16,float16,7,0.627507209777832
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,1,128,1,float16,float16,511,0.027977600693702698
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,128,8,128,1,float16,float16,15,0.63405442237854
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,1,128,1,float16,float16,1023,0.03254719972610474
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,1,128,1,float16,float16,2047,0.04284160137176514
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,2,128,1,float16,float16,7,0.014608000218868256
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,128,8,128,1,float16,float16,31,0.6466879844665527
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,2,128,1,float16,float16,1,0.014582400023937226
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,2,128,1,float16,float16,3,0.014483200013637542
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,2,128,1,float16,float16,31,0.014668799936771393
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,2,128,1,float16,float16,15,0.014575999975204468
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,2,128,1,float16,float16,127,0.01733119934797287
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,2,128,1,float16,float16,63,0.01465120017528534
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,2,128,1,float16,float16,255,0.022169600427150726
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,4,128,1,float16,float16,1,0.014604799449443817
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,2,128,1,float16,float16,1023,0.040822398662567136
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,2,128,1,float16,float16,511,0.03129119873046875
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,4,128,1,float16,float16,7,0.014606399834156037
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,4,128,1,float16,float16,3,0.014443199336528777
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,2,128,1,float16,float16,2047,0.05723839998245239
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,4,128,1,float16,float16,31,0.014553600549697876
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,4,128,1,float16,float16,63,0.014646400511264802
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,4,128,1,float16,float16,15,0.014372800290584565
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,4,128,1,float16,float16,127,0.0174687996506691
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,4,128,1,float16,float16,255,0.022233599424362184
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,4,128,1,float16,float16,511,0.048824000358581546
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,8,128,1,float16,float16,3,0.01642560064792633
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,8,128,1,float16,float16,1,0.017057600617408752
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,4,128,1,float16,float16,1023,0.0796015977859497
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,8,128,1,float16,float16,15,0.016519999504089354
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,8,128,1,float16,float16,7,0.0164015993475914
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,8,128,1,float16,float16,63,0.016889600455760954
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,8,128,1,float16,float16,31,0.016412800550460814
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,4,128,1,float16,float16,2047,0.11229280233383179
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,8,128,1,float16,float16,127,0.02104160040616989
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,8,128,1,float16,float16,255,0.029283198714256286
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,1,128,1,float16,float16,1,0.016195200383663177
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,1,128,1,float16,float16,7,0.016161599755287172
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,8,128,1,float16,float16,1023,0.1129871964454651
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,1,128,1,float16,float16,3,0.016323199868202208
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,1,128,1,float16,float16,15,0.01648160070180893
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,1,128,1,float16,float16,31,0.016355200111865996
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,8,128,1,float16,float16,511,0.06964160203933716
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,1,128,1,float16,float16,63,0.016161599755287172
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,8,128,1,float16,float16,2047,0.1934991955757141
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,1,128,1,float16,float16,255,0.023665599524974823
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,1,128,1,float16,float16,127,0.018911999464035035
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,1,128,1,float16,float16,511,0.05971360206604004
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,2,128,1,float16,float16,7,0.017950400710105896
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,2,128,1,float16,float16,1,0.017684799432754517
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,2,128,1,float16,float16,3,0.0178384006023407
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,1,128,1,float16,float16,1023,0.08376320004463196
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,2,128,1,float16,float16,127,0.022380800545215608
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,2,128,1,float16,float16,63,0.018320000171661376
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,2,128,1,float16,float16,255,0.03056640028953552
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,2,128,1,float16,float16,15,0.018011200428009033
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,2,128,1,float16,float16,31,0.018216000497341157
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,4,128,1,float16,float16,1,0.0236735999584198
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,1,128,1,float16,float16,2047,0.12455519437789916
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,2,128,1,float16,float16,511,0.07360479831695557
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,4,128,1,float16,float16,3,0.02375040054321289
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,2,128,1,float16,float16,1023,0.11576639413833618
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,4,128,1,float16,float16,7,0.023683199286460878
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,4,128,1,float16,float16,15,0.023793600499629974
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,4,128,1,float16,float16,63,0.024327999353408812
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,2,128,1,float16,float16,2047,0.19554239511489868
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,4,128,1,float16,float16,31,0.024038399755954742
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,4,128,1,float16,float16,255,0.049112001061439516
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,4,128,1,float16,float16,127,0.03054080009460449
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,8,128,1,float16,float16,1,0.03137120008468628
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,4,128,1,float16,float16,511,0.10094560384750366
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,8,128,1,float16,float16,3,0.03118079900741577
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,8,128,1,float16,float16,7,0.03115839958190918
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,4,128,1,float16,float16,1023,0.16795040369033815
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,8,128,1,float16,float16,31,0.031974399089813234
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,8,128,1,float16,float16,15,0.03143840134143829
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,8,128,1,float16,float16,63,0.03250559866428375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,8,128,1,float16,float16,127,0.0483711987733841
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,4,128,1,float16,float16,2047,0.28579039573669435
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,8,128,1,float16,float16,255,0.07103679776191711
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,8,128,1,float16,float16,511,0.15756160020828247
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,1,128,1,float16,float16,1,0.014257599413394929
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,8,128,1,float16,float16,1023,0.26773600578308104
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,1,128,1,float16,float16,3,0.014019200205802917
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,1,128,1,float16,float16,31,0.014256000518798828
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,1,128,1,float16,float16,7,0.014475199580192565
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,1,128,1,float16,float16,15,0.014300799369812012
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,1,128,1,float16,float16,127,0.016777600347995757
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,1,128,1,float16,float16,255,0.02176479995250702
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,1,128,1,float16,float16,511,0.02465440034866333
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,1,128,1,float16,float16,1023,0.02866879999637604
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,1,128,1,float16,float16,63,0.014321599900722504
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,2,128,1,float16,float16,1,0.013942399621009826
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,2,128,1,float16,float16,3,0.014420799911022186
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,1,128,1,float16,float16,2047,0.029003199934959412
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,2,128,1,float16,float16,7,0.01401119977235794
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,2,128,1,float16,float16,31,0.01401119977235794
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,2,128,1,float16,float16,15,0.014289599657058717
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,2,128,1,float16,float16,63,0.014654399454593658
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,2,128,1,float16,float16,127,0.016728000342845918
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,8,128,1,float16,float16,2047,0.46691360473632815
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,2,128,1,float16,float16,255,0.021697600185871125
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,2,128,1,float16,float16,511,0.023175999522209167
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,4,128,1,float16,float16,1,0.014035199582576752
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,4,128,1,float16,float16,3,0.014124800264835358
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,4,128,1,float16,float16,15,0.014099200069904328
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,2,128,1,float16,float16,1023,0.027483201026916503
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,2,128,1,float16,float16,2047,0.028513601422309874
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,4,128,1,float16,float16,31,0.014123199880123139
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,4,128,1,float16,float16,7,0.013990400731563568
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,4,128,1,float16,float16,63,0.0141744002699852
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,4,128,1,float16,float16,127,0.016579200327396394
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,4,128,1,float16,float16,255,0.021726399660110474
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,4,128,1,float16,float16,511,0.023255999386310577
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,4,128,1,float16,float16,1023,0.02712160050868988
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,8,128,1,float16,float16,1,0.01401119977235794
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,8,128,1,float16,float16,3,0.014075200259685516
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,4,128,1,float16,float16,2047,0.02800639867782593
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,8,128,1,float16,float16,15,0.014182400703430176
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,8,128,1,float16,float16,7,0.014168000221252442
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,8,128,1,float16,float16,127,0.01659200042486191
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,8,128,1,float16,float16,63,0.014207999408245086
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,8,128,1,float16,float16,31,0.014091199636459351
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,8,128,1,float16,float16,511,0.023263999819755556
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,8,128,1,float16,float16,2047,0.02855679988861084
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,1,128,1,float16,float16,1,0.014084799587726593
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,8,128,1,float16,float16,255,0.021492800116539
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,1,128,1,float16,float16,3,0.014204800128936768
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,8,128,1,float16,float16,1023,0.027718400955200194
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,1,128,1,float16,float16,7,0.014537599682807923
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,1,128,1,float16,float16,15,0.014340800046920777
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,1,128,1,float16,float16,31,0.0143327996134758
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,1,128,1,float16,float16,127,0.016908800601959227
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,1,128,1,float16,float16,63,0.014427199959754944
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,1,128,1,float16,float16,511,0.02439039945602417
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,1,128,1,float16,float16,255,0.02176959961652756
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,1,128,1,float16,float16,1023,0.028545600175857545
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,2,128,1,float16,float16,1,0.013980799913406372
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,1,128,1,float16,float16,2047,0.029204800724983215
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,2,128,1,float16,float16,7,0.014324800670146942
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,2,128,1,float16,float16,3,0.014116799831390381
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,2,128,1,float16,float16,15,0.014126400649547576
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,2,128,1,float16,float16,63,0.014134399592876434
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,2,128,1,float16,float16,31,0.014129599928855896
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,2,128,1,float16,float16,127,0.01682399958372116
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,2,128,1,float16,float16,255,0.02208479940891266
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,2,128,1,float16,float16,511,0.023612800240516662
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,2,128,1,float16,float16,2047,0.02855840027332306
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,2,128,1,float16,float16,1023,0.027777600288391113
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,4,128,1,float16,float16,1,0.014051200449466705
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,4,128,1,float16,float16,7,0.014151999354362487
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,4,128,1,float16,float16,3,0.014177599549293518
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,4,128,1,float16,float16,15,0.014179199934005737
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,4,128,1,float16,float16,31,0.013977600634098053
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,4,128,1,float16,float16,63,0.014163200557231904
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,4,128,1,float16,float16,127,0.016811199486255646
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,4,128,1,float16,float16,255,0.021964800357818604
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,4,128,1,float16,float16,511,0.023499199748039247
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,4,128,1,float16,float16,1023,0.0278656005859375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,8,128,1,float16,float16,1,0.014129599928855896
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,4,128,1,float16,float16,2047,0.028774398565292358
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,8,128,1,float16,float16,3,0.014326399564743042
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,8,128,1,float16,float16,7,0.01395840048789978
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,8,128,1,float16,float16,15,0.014142400026321411
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,8,128,1,float16,float16,31,0.014180800318717957
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,8,128,1,float16,float16,63,0.014201599359512328
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,8,128,1,float16,float16,127,0.01695840060710907
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,8,128,1,float16,float16,255,0.021753600239753722
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,8,128,1,float16,float16,511,0.0238864004611969
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,8,128,1,float16,float16,1023,0.028224000334739686
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,1,128,1,float16,float16,1,0.01541920006275177
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,8,128,1,float16,float16,2047,0.037161600589752194
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,1,128,1,float16,float16,3,0.015304000675678253
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,1,128,1,float16,float16,7,0.01530880033969879
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,1,128,1,float16,float16,31,0.01536639928817749
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,1,128,1,float16,float16,15,0.015145599842071533
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,1,128,1,float16,float16,63,0.015406399965286255
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,1,128,1,float16,float16,127,0.018105599284172057
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,1,128,1,float16,float16,255,0.022724799811840057
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,1,128,1,float16,float16,511,0.03367359936237335
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,2,128,1,float16,float16,1,0.015241600573062897
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,1,128,1,float16,float16,1023,0.05264639854431152
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,2,128,1,float16,float16,3,0.015228800475597382
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,1,128,1,float16,float16,2047,0.05989919900894165
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,2,128,1,float16,float16,15,0.01515360027551651
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,2,128,1,float16,float16,7,0.015268799662590028
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,2,128,1,float16,float16,63,0.015371200442314149
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,2,128,1,float16,float16,31,0.015508800745010376
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,2,128,1,float16,float16,127,0.01791200041770935
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,2,128,1,float16,float16,255,0.022862400114536285
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,2,128,1,float16,float16,511,0.051046401262283325
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,4,128,1,float16,float16,1,0.017254400253295898
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,4,128,1,float16,float16,3,0.016940799355506898
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,4,128,1,float16,float16,7,0.017190399765968322
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,2,128,1,float16,float16,1023,0.07613120079040528
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,2,128,1,float16,float16,2047,0.11533440351486206
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,4,128,1,float16,float16,15,0.017059199512004852
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,4,128,1,float16,float16,63,0.01759839951992035
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,4,128,1,float16,float16,127,0.021700799465179443
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,4,128,1,float16,float16,31,0.017425599694252013
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,4,128,1,float16,float16,255,0.030067199468612672
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,4,128,1,float16,float16,511,0.07624319791793824
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,8,128,1,float16,float16,1,0.022868800163269042
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,4,128,1,float16,float16,1023,0.11446880102157593
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,8,128,1,float16,float16,3,0.023097600042819976
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,8,128,1,float16,float16,7,0.023028799891471864
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,8,128,1,float16,float16,31,0.023161600530147552
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,8,128,1,float16,float16,15,0.022918400168418885
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,4,128,1,float16,float16,2047,0.19389439821243287
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,8,128,1,float16,float16,63,0.023499199748039247
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,8,128,1,float16,float16,127,0.02971999943256378
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,8,128,1,float16,float16,255,0.04711039960384369
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,1,128,1,float16,float16,1,0.014345599710941315
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,8,128,1,float16,float16,511,0.09782559871673584
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,8,128,1,float16,float16,1023,0.16438720226287842
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,1,128,1,float16,float16,3,0.014537599682807923
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,1,128,1,float16,float16,15,0.014486399292945863
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,1,128,1,float16,float16,7,0.014552000164985656
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,1,128,1,float16,float16,127,0.017183999717235564
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,1,128,1,float16,float16,31,0.014327999949455262
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,1,128,1,float16,float16,255,0.021857599914073943
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,1,128,1,float16,float16,63,0.014382399618625641
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,8,128,1,float16,float16,2047,0.2840399980545044
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,1,128,1,float16,float16,511,0.024899199604988098
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,2,128,1,float16,float16,1,0.014348800480365752
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,2,128,1,float16,float16,3,0.014192000031471252
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,1,128,1,float16,float16,2047,0.029788801074028017
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,1,128,1,float16,float16,1023,0.029123198986053467
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,2,128,1,float16,float16,31,0.014417600631713868
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,2,128,1,float16,float16,15,0.014529600739479065
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,2,128,1,float16,float16,63,0.014318400621414184
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,2,128,1,float16,float16,7,0.014105600118637086
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,2,128,1,float16,float16,255,0.022086399793624877
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,2,128,1,float16,float16,127,0.01686079949140549
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,2,128,1,float16,float16,511,0.023790399730205535
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,2,128,1,float16,float16,2047,0.029180800914764403
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,4,128,1,float16,float16,7,0.014177599549293518
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,4,128,1,float16,float16,1,0.014280000329017639
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,2,128,1,float16,float16,1023,0.02851040065288544
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,4,128,1,float16,float16,3,0.014251199364662171
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,4,128,1,float16,float16,15,0.01422239989042282
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,4,128,1,float16,float16,31,0.014283199608325959
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,4,128,1,float16,float16,63,0.01443839967250824
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,4,128,1,float16,float16,127,0.016991999745368958
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,4,128,1,float16,float16,255,0.021916800737380983
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,4,128,1,float16,float16,1023,0.02851040065288544
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,8,128,1,float16,float16,1,0.014209599792957306
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,8,128,1,float16,float16,3,0.014398400485515595
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,4,128,1,float16,float16,511,0.024166400730609893
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,8,128,1,float16,float16,7,0.014377599954605103
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,4,128,1,float16,float16,2047,0.037268799543380735
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,8,128,1,float16,float16,15,0.014235199987888336
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,8,128,1,float16,float16,31,0.014398400485515595
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,8,128,1,float16,float16,63,0.014377599954605103
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,8,128,1,float16,float16,127,0.01698399931192398
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,8,128,1,float16,float16,511,0.02863680124282837
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,8,128,1,float16,float16,255,0.02210240066051483
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,8,128,1,float16,float16,1023,0.03795199990272522
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,1,128,1,float16,float16,1,0.019644799828529357
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,1,128,1,float16,float16,3,0.01958400011062622
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,1,128,1,float16,float16,7,0.019678400456905366
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,8,128,1,float16,float16,2047,0.05389760136604309
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,1,128,1,float16,float16,15,0.019860799610614776
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,1,128,1,float16,float16,31,0.01992799937725067
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,1,128,1,float16,float16,63,0.020291200280189513
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,1,128,1,float16,float16,127,0.024243199825286867
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,1,128,1,float16,float16,255,0.03280799984931946
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,2,128,1,float16,float16,1,0.025220799446105956
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,1,128,1,float16,float16,511,0.0772607982158661
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,2,128,1,float16,float16,3,0.025436800718307496
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,2,128,1,float16,float16,7,0.025231999158859254
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,2,128,1,float16,float16,15,0.025281599164009093
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,2,128,1,float16,float16,31,0.025740799307823182
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,1,128,1,float16,float16,1023,0.12071199417114258
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,2,128,1,float16,float16,127,0.03299840092658997
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,2,128,1,float16,float16,63,0.02616479992866516
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,2,128,1,float16,float16,255,0.05480319857597351
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,2,128,1,float16,float16,511,0.10471999645233154
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,4,128,1,float16,float16,3,0.03277760148048401
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,4,128,1,float16,float16,7,0.03266400098800659
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,4,128,1,float16,float16,1,0.03275519907474518
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,4,128,1,float16,float16,15,0.032892799377441405
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,2,128,1,float16,float16,1023,0.1696768045425415
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,4,128,1,float16,float16,63,0.03462719917297363
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,4,128,1,float16,float16,31,0.03322719931602478
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,4,128,1,float16,float16,127,0.05158720016479492
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,4,128,1,float16,float16,255,0.0866320013999939
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,4,128,1,float16,float16,511,0.1613055944442749
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,8,128,1,float16,float16,1,0.050113600492477414
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,8,128,1,float16,float16,3,0.05030080080032349
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,8,128,1,float16,float16,7,0.050323200225830075
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,8,128,1,float16,float16,15,0.05061280131340027
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,4,128,1,float16,float16,1023,0.2699295997619629
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,8,128,1,float16,float16,31,0.051716798543930055
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,8,128,1,float16,float16,63,0.06055359840393067
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,8,128,1,float16,float16,127,0.080622398853302
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,8,128,1,float16,float16,255,0.12310880422592163
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,96,1,128,1,float16,float16,1,0.02889919877052307
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,96,1,128,1,float16,float16,3,0.028683200478553772
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,96,1,128,1,float16,float16,7,0.02901279926300049
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,8,128,1,float16,float16,511,0.2991071939468384
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,96,1,128,1,float16,float16,31,0.02916319966316223
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,96,1,128,1,float16,float16,15,0.02921440005302429
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,96,1,128,1,float16,float16,63,0.03012000024318695
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,96,1,128,1,float16,float16,127,0.038068801164627075
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,96,2,128,1,float16,float16,3,0.0349263995885849
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,96,2,128,1,float16,float16,1,0.03545120060443878
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,96,1,128,1,float16,float16,255,0.06344000101089478
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,96,2,128,1,float16,float16,7,0.03574559986591339
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,96,2,128,1,float16,float16,15,0.03521760106086731
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,96,2,128,1,float16,float16,31,0.03607040047645569
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,8,128,1,float16,float16,1023,0.5198847770690918
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,96,2,128,1,float16,float16,63,0.04209280014038086
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,96,2,128,1,float16,float16,127,0.05704960227012634
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,96,4,128,1,float16,float16,1,0.05264319777488709
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,96,2,128,1,float16,float16,255,0.08027999997138976
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,96,4,128,1,float16,float16,3,0.052393597364425656
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,96,4,128,1,float16,float16,15,0.05324959754943848
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,96,4,128,1,float16,float16,31,0.05482239723205566
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,96,4,128,1,float16,float16,7,0.05273759961128235
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,96,4,128,1,float16,float16,63,0.06489440202713012
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,96,8,128,1,float16,float16,1,0.08940160274505615
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,96,4,128,1,float16,float16,127,0.08491680026054382
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,96,8,128,1,float16,float16,3,0.0895151972770691
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,96,8,128,1,float16,float16,7,0.08958879709243775
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,96,4,128,1,float16,float16,255,0.12833280563354493
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,96,8,128,1,float16,float16,31,0.09597280025482177
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,96,8,128,1,float16,float16,15,0.09181280136108398
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,96,1,128,1,float16,float16,1,0.04397920072078705
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,96,8,128,1,float16,float16,63,0.10171040296554565
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,96,8,128,1,float16,float16,127,0.14150079488754272
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,96,1,128,1,float16,float16,7,0.044940799474716187
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,96,8,128,1,float16,float16,255,0.226310396194458
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,96,1,128,1,float16,float16,3,0.044014400243759154
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,96,1,128,1,float16,float16,15,0.04555839896202087
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,96,1,128,1,float16,float16,31,0.04992319941520691
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,96,1,128,1,float16,float16,63,0.0564736008644104
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,96,2,128,1,float16,float16,1,0.06043040156364441
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,96,1,128,1,float16,float16,127,0.06756640076637269
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,96,2,128,1,float16,float16,15,0.06425759792327881
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,96,2,128,1,float16,float16,7,0.06179680228233338
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,96,2,128,1,float16,float16,3,0.06265919804573059
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,96,2,128,1,float16,float16,31,0.07578240036964416
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,96,2,128,1,float16,float16,63,0.07859839797019959
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,96,4,128,1,float16,float16,1,0.09764320254325867
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,96,2,128,1,float16,float16,127,0.09340320229530334
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,96,4,128,1,float16,float16,3,0.09786239862442017
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,96,4,128,1,float16,float16,7,0.09974719882011414
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,96,4,128,1,float16,float16,31,0.10421760082244873
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,96,4,128,1,float16,float16,15,0.10268959999084473
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,96,4,128,1,float16,float16,63,0.11005120277404785
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,96,4,128,1,float16,float16,127,0.15150400400161743
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,96,8,128,1,float16,float16,3,0.1695647954940796
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,96,8,128,1,float16,float16,1,0.16930400133132933
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,96,8,128,1,float16,float16,7,0.17199039459228516
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,96,8,128,1,float16,float16,15,0.17475520372390746
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,96,8,128,1,float16,float16,31,0.17768160104751587
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,1,128,1,float16,float16,1,0.014323200285434722
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,1,128,1,float16,float16,3,0.01462399959564209
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,96,8,128,1,float16,float16,63,0.18620959520339966
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,1,128,1,float16,float16,7,0.01446560025215149
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,1,128,1,float16,float16,31,0.014321599900722504
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,1,128,1,float16,float16,63,0.014745600521564484
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,1,128,1,float16,float16,15,0.01451680064201355
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,96,8,128,1,float16,float16,127,0.2668447971343994
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,1,128,1,float16,float16,255,0.022177599370479584
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,1,128,1,float16,float16,511,0.02499680072069168
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,1,128,1,float16,float16,127,0.017023999989032746
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,1,128,1,float16,float16,1023,0.029627200961112977
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,2,128,1,float16,float16,3,0.0144896000623703
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,1,128,1,float16,float16,2047,0.031043198704719544
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,2,128,1,float16,float16,15,0.014299200475215912
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,2,128,1,float16,float16,1,0.014312000572681427
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,2,128,1,float16,float16,31,0.01456640064716339
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,2,128,1,float16,float16,63,0.014528000354766845
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,2,128,1,float16,float16,7,0.014185599982738495
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,2,128,1,float16,float16,127,0.017113600671291352
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,2,128,1,float16,float16,255,0.021907199919223786
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,4,128,1,float16,float16,1,0.01435679942369461
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,2,128,1,float16,float16,1023,0.02897599935531616
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,4,128,1,float16,float16,3,0.014295999705791474
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,2,128,1,float16,float16,511,0.024828800559043886
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,4,128,1,float16,float16,7,0.014259199798107147
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,2,128,1,float16,float16,2047,0.038192000985145566
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,4,128,1,float16,float16,15,0.01435679942369461
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,4,128,1,float16,float16,31,0.014556799829006196
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,4,128,1,float16,float16,63,0.014552000164985656
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,4,128,1,float16,float16,255,0.021876800060272216
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,4,128,1,float16,float16,127,0.01706880033016205
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,4,128,1,float16,float16,511,0.02933279871940613
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,4,128,1,float16,float16,1023,0.03784480094909668
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,8,128,1,float16,float16,1,0.014502400159835815
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,4,128,1,float16,float16,2047,0.05573279857635498
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,8,128,1,float16,float16,3,0.01430400013923645
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,8,128,1,float16,float16,7,0.014390400052070618
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,8,128,1,float16,float16,15,0.014606399834156037
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,8,128,1,float16,float16,63,0.014601600170135499
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,8,128,1,float16,float16,31,0.014584000408649444
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,8,128,1,float16,float16,255,0.0220880001783371
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,8,128,1,float16,float16,127,0.017396800220012665
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,8,128,1,float16,float16,511,0.04623039960861206
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,96,1,128,1,float16,float16,1,0.07546079754829407
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,8,128,1,float16,float16,1023,0.07093120217323304
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,96,1,128,1,float16,float16,7,0.07606239914894104
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,8,128,1,float16,float16,2047,0.11161919832229614
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,96,1,128,1,float16,float16,3,0.07513759732246399
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,96,1,128,1,float16,float16,15,0.07679679989814758
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,96,1,128,1,float16,float16,31,0.07821760177612305
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,96,1,128,1,float16,float16,63,0.08503199815750122
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,96,2,128,1,float16,float16,1,0.10724480152130127
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,96,2,128,1,float16,float16,3,0.10819679498672485
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,96,2,128,1,float16,float16,7,0.10927519798278809
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,96,2,128,1,float16,float16,15,0.10980639457702637
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,96,2,128,1,float16,float16,31,0.11179840564727783
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,96,2,128,1,float16,float16,63,0.11720800399780273
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,96,4,128,1,float16,float16,1,0.1770815968513489
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,96,4,128,1,float16,float16,15,0.18196799755096435
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,96,4,128,1,float16,float16,3,0.1783552050590515
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,96,4,128,1,float16,float16,7,0.1799855947494507
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,96,4,128,1,float16,float16,31,0.18326719999313354
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,96,4,128,1,float16,float16,63,0.19477920532226561
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,96,8,128,1,float16,float16,1,0.3200464010238647
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,96,8,128,1,float16,float16,3,0.32012479305267333
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,96,8,128,1,float16,float16,7,0.3235055923461914
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,96,8,128,1,float16,float16,15,0.32665119171142576
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,96,8,128,1,float16,float16,31,0.3312319993972778
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,96,1,128,1,float16,float16,1,0.12124799489974976
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,96,8,128,1,float16,float16,63,0.34936161041259767
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,96,1,128,1,float16,float16,7,0.1232367992401123
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,96,1,128,1,float16,float16,3,0.12216000556945801
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,96,1,128,1,float16,float16,15,0.1244320034980774
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,96,1,128,1,float16,float16,31,0.12709759473800658
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,96,2,128,1,float16,float16,1,0.1873792052268982
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,96,2,128,1,float16,float16,3,0.19183839559555055
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,96,2,128,1,float16,float16,7,0.1917871952056885
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,96,2,128,1,float16,float16,15,0.19196000099182128
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,96,2,128,1,float16,float16,31,0.19540640115737914
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,96,4,128,1,float16,float16,1,0.32880799770355223
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,96,4,128,1,float16,float16,3,0.33173599243164065
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,96,4,128,1,float16,float16,7,0.3365200042724609
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,96,4,128,1,float16,float16,15,0.33904640674591063
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,96,4,128,1,float16,float16,31,0.34305601119995116
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,1,128,1,float16,float16,1,0.014507199823856353
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,96,8,128,1,float16,float16,1,0.6158143997192382
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,1,128,1,float16,float16,3,0.014582400023937226
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,1,128,1,float16,float16,7,0.01441120058298111
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,96,8,128,1,float16,float16,3,0.616156816482544
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,1,128,1,float16,float16,31,0.014539200067520141
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,1,128,1,float16,float16,15,0.014419199526309967
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,1,128,1,float16,float16,63,0.014644800126552582
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,96,8,128,1,float16,float16,7,0.6237311840057373
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,1,128,1,float16,float16,255,0.02210559993982315
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,1,128,1,float16,float16,127,0.017348800599575043
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,1,128,1,float16,float16,511,0.026713600754737853
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,96,8,128,1,float16,float16,15,0.6327936172485351
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,1,128,1,float16,float16,1023,0.03097119927406311
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,2,128,1,float16,float16,1,0.014377599954605103
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,2,128,1,float16,float16,3,0.014401599764823914
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,96,8,128,1,float16,float16,31,0.6419456005096436
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,2,128,1,float16,float16,7,0.014417600631713868
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,1,128,1,float16,float16,2047,0.04074400067329407
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,2,128,1,float16,float16,31,0.014481599628925323
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,2,128,1,float16,float16,63,0.014584000408649444
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,2,128,1,float16,float16,15,0.01451839953660965
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,2,128,1,float16,float16,255,0.0219200000166893
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,2,128,1,float16,float16,511,0.03018240034580231
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,4,128,1,float16,float16,1,0.014643199741840363
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,4,128,1,float16,float16,3,0.01446399986743927
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,2,128,1,float16,float16,1023,0.03946720063686371
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,2,128,1,float16,float16,127,0.01722719967365265
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,4,128,1,float16,float16,7,0.014710399508476257
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,4,128,1,float16,float16,15,0.014547200500965118
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,2,128,1,float16,float16,2047,0.05610399842262268
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,4,128,1,float16,float16,31,0.014601600170135499
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,4,128,1,float16,float16,127,0.01737920045852661
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,4,128,1,float16,float16,63,0.014720000326633453
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,4,128,1,float16,float16,255,0.022358399629592896
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,4,128,1,float16,float16,511,0.047654399275779726
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,8,128,1,float16,float16,1,0.016438399255275727
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,4,128,1,float16,float16,1023,0.07230880260467529
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,8,128,1,float16,float16,3,0.016340799629688263
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,8,128,1,float16,float16,7,0.016574400663375854
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,8,128,1,float16,float16,15,0.016627199947834015
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,4,128,1,float16,float16,2047,0.11202239990234375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,8,128,1,float16,float16,31,0.0166703999042511
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,8,128,1,float16,float16,63,0.016899199783802034
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,8,128,1,float16,float16,127,0.02114879935979843
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,8,128,1,float16,float16,255,0.029348799586296083
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,8,128,1,float16,float16,1023,0.1129871964454651
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,8,128,1,float16,float16,511,0.0698303997516632
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,1,128,1,float16,float16,3,0.01594880074262619
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,1,128,1,float16,float16,1,0.015919999778270723
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,1,128,1,float16,float16,7,0.016097599267959596
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,8,128,1,float16,float16,2047,0.19306880235671997
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,1,128,1,float16,float16,127,0.018654400110244752
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,1,128,1,float16,float16,15,0.01584160029888153
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,1,128,1,float16,float16,255,0.02380640059709549
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,1,128,1,float16,float16,63,0.015987199544906617
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,1,128,1,float16,float16,31,0.01611520051956177
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,2,128,1,float16,float16,1,0.017716799676418305
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,1,128,1,float16,float16,511,0.05354560017585754
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,2,128,1,float16,float16,3,0.017833599448204042
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,1,128,1,float16,float16,1023,0.07904000282287597
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,2,128,1,float16,float16,7,0.01764959990978241
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,1,128,1,float16,float16,2047,0.1181264042854309
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,2,128,1,float16,float16,31,0.01801760047674179
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,2,128,1,float16,float16,15,0.017803199589252472
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,1,128,1,float16,float16,4095,0.19876159429550172
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,2,128,1,float16,float16,127,0.022280000150203705
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,2,128,1,float16,float16,63,0.018145599961280824
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,2,128,1,float16,float16,255,0.03047040104866028
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,4,128,1,float16,float16,1,0.023550400137901308
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,4,128,1,float16,float16,3,0.023523199558258056
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,2,128,1,float16,float16,511,0.0722432017326355
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,2,128,1,float16,float16,2047,0.1954527974128723
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,4,128,1,float16,float16,7,0.023647999763488768
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,2,128,1,float16,float16,1023,0.11609280109405518
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,4,128,1,float16,float16,15,0.023740799725055696
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,4,128,1,float16,float16,31,0.023852799832820893
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,4,128,1,float16,float16,255,0.04779999852180481
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,4,128,1,float16,float16,63,0.02415519952774048
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,4,128,1,float16,float16,127,0.030529600381851197
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,4,128,1,float16,float16,511,0.10079200267791748
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,2,128,1,float16,float16,4095,0.3559056043624878
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,4,128,1,float16,float16,1023,0.16610080003738403
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,8,128,1,float16,float16,1,0.031124800443649292
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,8,128,1,float16,float16,3,0.031224000453948974
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,8,128,1,float16,float16,7,0.03105440139770508
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,4,128,1,float16,float16,2047,0.2852400064468384
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,8,128,1,float16,float16,63,0.03239839971065521
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,8,128,1,float16,float16,15,0.03129279911518097
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,8,128,1,float16,float16,31,0.031700798869133
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,8,128,1,float16,float16,127,0.0475951999425888
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,8,128,1,float16,float16,255,0.07038080096244811
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,1,128,1,float16,float16,1,0.01433439999818802
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,1,128,1,float16,float16,3,0.014008000493049622
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,4,128,1,float16,float16,4095,0.5258895874023437
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,1,128,1,float16,float16,7,0.014256000518798828
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,8,128,1,float16,float16,511,0.15746239423751832
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,1,128,1,float16,float16,15,0.014164799451828003
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,8,128,1,float16,float16,1023,0.2672096014022827
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,1,128,1,float16,float16,63,0.014455999433994293
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,1,128,1,float16,float16,255,0.02168159931898117
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,1,128,1,float16,float16,31,0.014211200177669525
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,1,128,1,float16,float16,127,0.016867199540138246
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,1,128,1,float16,float16,1023,0.027663999795913698
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,1,128,1,float16,float16,2047,0.02824159860610962
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,1,128,1,float16,float16,511,0.02346560060977936
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,2,128,1,float16,float16,1,0.013955199718475341
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,8,128,1,float16,float16,2047,0.46617441177368163
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,2,128,1,float16,float16,3,0.014340800046920777
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,2,128,1,float16,float16,7,0.014017599821090698
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,1,128,1,float16,float16,4095,0.029161599278450013
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,2,128,1,float16,float16,31,0.01408960074186325
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,2,128,1,float16,float16,15,0.014232000708580017
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,2,128,1,float16,float16,127,0.01701759994029999
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,2,128,1,float16,float16,255,0.02163680046796799
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,2,128,1,float16,float16,1023,0.027292799949645997
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,2,128,1,float16,float16,63,0.014193600416183472
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,2,128,1,float16,float16,511,0.022860799729824067
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,2,128,1,float16,float16,2047,0.02783840000629425
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,2,128,1,float16,float16,4095,0.029313600063323973
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,8,128,1,float16,float16,4095,0.8651472091674804
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,4,128,1,float16,float16,1,0.01401440054178238
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,4,128,1,float16,float16,3,0.013924799859523773
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,4,128,1,float16,float16,7,0.014095999300479889
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,4,128,1,float16,float16,15,0.014124800264835358
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,4,128,1,float16,float16,127,0.016574400663375854
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,4,128,1,float16,float16,31,0.014163200557231904
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,4,128,1,float16,float16,1023,0.027024000883102417
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,4,128,1,float16,float16,255,0.021585600078105928
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,4,128,1,float16,float16,511,0.022867199778556824
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,4,128,1,float16,float16,2047,0.02778880000114441
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,4,128,1,float16,float16,4095,0.02932800054550171
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,4,128,1,float16,float16,63,0.014295999705791474
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,8,128,1,float16,float16,1,0.014131200313568116
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,8,128,1,float16,float16,7,0.014009599387645722
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,8,128,1,float16,float16,3,0.01401280015707016
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,8,128,1,float16,float16,15,0.014108799397945404
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,8,128,1,float16,float16,63,0.014140799641609192
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,8,128,1,float16,float16,31,0.014236800372600555
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,8,128,1,float16,float16,127,0.016681599617004394
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,8,128,1,float16,float16,255,0.0215488001704216
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,8,128,1,float16,float16,1023,0.027169600129127502
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,8,128,1,float16,float16,511,0.02279199957847595
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,1,128,1,float16,float16,1,0.01419840008020401
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,1,128,1,float16,float16,3,0.014070400595664978
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,8,128,1,float16,float16,2047,0.028193598985671996
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,1,128,1,float16,float16,7,0.014132800698280334
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,8,128,1,float16,float16,4095,0.03720960021018982
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,1,128,1,float16,float16,31,0.014283199608325959
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,1,128,1,float16,float16,15,0.014151999354362487
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,1,128,1,float16,float16,63,0.01419519931077957
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,1,128,1,float16,float16,255,0.021492800116539
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,1,128,1,float16,float16,127,0.01675039976835251
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,1,128,1,float16,float16,1023,0.02786720097064972
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,1,128,1,float16,float16,511,0.02351360023021698
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,1,128,1,float16,float16,2047,0.02851040065288544
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,2,128,1,float16,float16,1,0.014180800318717957
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,1,128,1,float16,float16,4095,0.02982560098171234
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,2,128,1,float16,float16,3,0.013984000682830811
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,2,128,1,float16,float16,7,0.01408960074186325
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,2,128,1,float16,float16,63,0.014267200231552124
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,2,128,1,float16,float16,15,0.014091199636459351
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,2,128,1,float16,float16,31,0.014176000654697419
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,2,128,1,float16,float16,127,0.016760000586509706
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,2,128,1,float16,float16,511,0.02330559939146042
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,2,128,1,float16,float16,255,0.02162880003452301
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,2,128,1,float16,float16,1023,0.027444800734519957
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,2,128,1,float16,float16,4095,0.029598399996757507
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,4,128,1,float16,float16,1,0.013911999762058258
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,2,128,1,float16,float16,2047,0.028433600068092348
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,4,128,1,float16,float16,3,0.014193600416183472
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,4,128,1,float16,float16,7,0.014131200313568116
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,4,128,1,float16,float16,15,0.01406400054693222
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,4,128,1,float16,float16,63,0.014241600036621093
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,4,128,1,float16,float16,31,0.014127999544143677
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,4,128,1,float16,float16,255,0.02178719937801361
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,4,128,1,float16,float16,1023,0.0276528000831604
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,4,128,1,float16,float16,127,0.016916799545288085
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,4,128,1,float16,float16,511,0.023203200101852416
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,4,128,1,float16,float16,2047,0.029039999842643736
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,8,128,1,float16,float16,1,0.014289599657058717
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,4,128,1,float16,float16,4095,0.03761279881000519
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,8,128,1,float16,float16,3,0.014214399456977844
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,8,128,1,float16,float16,7,0.0142752006649971
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,8,128,1,float16,float16,15,0.014168000221252442
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,8,128,1,float16,float16,31,0.014207999408245086
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,8,128,1,float16,float16,63,0.01438560038805008
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,8,128,1,float16,float16,255,0.02156960070133209
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,8,128,1,float16,float16,127,0.016947199404239655
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,8,128,1,float16,float16,511,0.023680000007152556
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,8,128,1,float16,float16,1023,0.027806401252746582
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,8,128,1,float16,float16,2047,0.0364656001329422
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,1,128,1,float16,float16,1,0.015110400319099427
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,1,128,1,float16,float16,3,0.015171200037002563
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,8,128,1,float16,float16,4095,0.054308801889419556
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,1,128,1,float16,float16,7,0.01507679969072342
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,1,128,1,float16,float16,15,0.015134400129318238
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,1,128,1,float16,float16,63,0.015289600193500518
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,1,128,1,float16,float16,127,0.017763200402259826
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,1,128,1,float16,float16,31,0.015057599544525147
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,1,128,1,float16,float16,255,0.02281759977340698
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,2,128,1,float16,float16,1,0.015041600167751312
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,1,128,1,float16,float16,511,0.031830400228500366
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,2,128,1,float16,float16,3,0.015324799716472626
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,1,128,1,float16,float16,1023,0.04174239933490753
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,1,128,1,float16,float16,2047,0.05817599892616272
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,2,128,1,float16,float16,7,0.015145599842071533
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,1,128,1,float16,float16,4095,0.08520320057868958
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,2,128,1,float16,float16,31,0.015352000296115876
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,2,128,1,float16,float16,63,0.015292799472808838
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,2,128,1,float16,float16,15,0.015209600329399109
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,2,128,1,float16,float16,255,0.02263679951429367
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,2,128,1,float16,float16,127,0.017924800515174866
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,2,128,1,float16,float16,511,0.04891839921474457
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,4,128,1,float16,float16,1,0.01681919991970062
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,4,128,1,float16,float16,3,0.01696320027112961
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,2,128,1,float16,float16,1023,0.07400799989700317
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,2,128,1,float16,float16,2047,0.11379040479660034
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,4,128,1,float16,float16,7,0.017123199999332428
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,4,128,1,float16,float16,15,0.01698399931192398
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,2,128,1,float16,float16,4095,0.19408479928970337
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,4,128,1,float16,float16,127,0.021624000370502473
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,4,128,1,float16,float16,63,0.017478400468826295
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,4,128,1,float16,float16,31,0.01729599982500076
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,4,128,1,float16,float16,255,0.029736000299453735
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,8,128,1,float16,float16,1,0.022993600368499754
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,8,128,1,float16,float16,3,0.022886399924755097
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,4,128,1,float16,float16,1023,0.11454720497131347
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,4,128,1,float16,float16,511,0.07098079919815063
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,8,128,1,float16,float16,7,0.02298399955034256
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,8,128,1,float16,float16,15,0.022972799837589264
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,4,128,1,float16,float16,2047,0.19326720237731934
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,8,128,1,float16,float16,127,0.0297791987657547
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,8,128,1,float16,float16,31,0.023212799429893495
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,8,128,1,float16,float16,63,0.023606400191783904
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,4,128,1,float16,float16,4095,0.35366239547729494
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,8,128,1,float16,float16,255,0.0465391993522644
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,8,128,1,float16,float16,511,0.09808160066604614
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,1,128,1,float16,float16,1,0.014108799397945404
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,1,128,1,float16,float16,3,0.014256000518798828
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,8,128,1,float16,float16,1023,0.16488959789276122
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,1,128,1,float16,float16,7,0.014120000600814819
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,1,128,1,float16,float16,15,0.014060799777507783
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,1,128,1,float16,float16,31,0.014180800318717957
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,8,128,1,float16,float16,2047,0.2840224027633667
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,1,128,1,float16,float16,127,0.016791999340057373
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,1,128,1,float16,float16,63,0.014556799829006196
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,1,128,1,float16,float16,511,0.02386080026626587
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,1,128,1,float16,float16,255,0.021755200624465943
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,1,128,1,float16,float16,2047,0.028828799724578857
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,2,128,1,float16,float16,1,0.014188799262046813
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,2,128,1,float16,float16,3,0.014108799397945404
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,1,128,1,float16,float16,4095,0.03044799864292145
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,1,128,1,float16,float16,1023,0.028014400601387025
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,8,128,1,float16,float16,4095,0.5244192123413086
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,2,128,1,float16,float16,7,0.013972799479961395
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,2,128,1,float16,float16,15,0.014217600226402283
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,2,128,1,float16,float16,31,0.013942399621009826
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,2,128,1,float16,float16,63,0.01446560025215149
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,2,128,1,float16,float16,511,0.02346719950437546
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,2,128,1,float16,float16,2047,0.028996801376342772
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,2,128,1,float16,float16,1023,0.02788960039615631
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,2,128,1,float16,float16,127,0.016771200299263
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,2,128,1,float16,float16,255,0.021756799519062044
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,2,128,1,float16,float16,4095,0.03814240097999573
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,4,128,1,float16,float16,1,0.014179199934005737
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,4,128,1,float16,float16,3,0.014220799505710601
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,4,128,1,float16,float16,15,0.014105600118637086
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,4,128,1,float16,float16,7,0.013998399674892425
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,4,128,1,float16,float16,127,0.016795200109481812
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,4,128,1,float16,float16,63,0.014451199769973755
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,4,128,1,float16,float16,31,0.014342400431632995
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,4,128,1,float16,float16,255,0.021729600429534913
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,4,128,1,float16,float16,511,0.023963199555873872
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,4,128,1,float16,float16,1023,0.028347200155258177
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,4,128,1,float16,float16,2047,0.03738879859447479
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,8,128,1,float16,float16,3,0.014270399510860444
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,8,128,1,float16,float16,7,0.01417279988527298
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,8,128,1,float16,float16,15,0.0142752006649971
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,4,128,1,float16,float16,4095,0.05482400059700012
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,8,128,1,float16,float16,1,0.0142752006649971
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,8,128,1,float16,float16,31,0.014310400187969207
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,8,128,1,float16,float16,63,0.014379200339317322
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,8,128,1,float16,float16,127,0.017028799653053282
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,8,128,1,float16,float16,255,0.021840000152587892
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,8,128,1,float16,float16,511,0.028300800919532777
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,8,128,1,float16,float16,2047,0.05386719703674316
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,1,128,1,float16,float16,1,0.019032000005245207
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,1,128,1,float16,float16,3,0.019139200448989868
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,8,128,1,float16,float16,1023,0.03774079978466034
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,8,128,1,float16,float16,4095,0.08075839877128602
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,1,128,1,float16,float16,7,0.019307200610637665
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,1,128,1,float16,float16,63,0.0196383997797966
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,1,128,1,float16,float16,15,0.019211199879646302
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,1,128,1,float16,float16,31,0.019324800372123717
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,1,128,1,float16,float16,127,0.02385440021753311
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,1,128,1,float16,float16,255,0.0318336009979248
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,1,128,1,float16,float16,511,0.07635679841041565
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,2,128,1,float16,float16,1,0.024963200092315674
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,2,128,1,float16,float16,3,0.0248879998922348
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,2,128,1,float16,float16,7,0.02491360008716583
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,2,128,1,float16,float16,31,0.02507199943065643
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,2,128,1,float16,float16,15,0.025119999051094057
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,1,128,1,float16,float16,1023,0.11894559860229492
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,1,128,1,float16,float16,2047,0.1979423999786377
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,2,128,1,float16,float16,127,0.03204480111598969
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,2,128,1,float16,float16,63,0.025625601410865784
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,2,128,1,float16,float16,255,0.04984320104122162
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,4,128,1,float16,float16,3,0.03256799876689911
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,2,128,1,float16,float16,511,0.10415040254592896
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,4,128,1,float16,float16,1,0.03239200115203857
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,4,128,1,float16,float16,7,0.03222239911556244
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,2,128,1,float16,float16,1023,0.16919360160827637
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,4,128,1,float16,float16,15,0.03250080049037933
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,4,128,1,float16,float16,63,0.03425759971141815
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,4,128,1,float16,float16,31,0.0330159991979599
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,4,128,1,float16,float16,127,0.05006719827651977
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,2,128,1,float16,float16,2047,0.28821918964385984
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,4,128,1,float16,float16,255,0.07309439778327942
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,4,128,1,float16,float16,511,0.16070560216903687
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,4,128,1,float16,float16,1023,0.2776704072952271
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,8,128,1,float16,float16,1,0.049772799015045166
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,8,128,1,float16,float16,3,0.05007359981536865
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,8,128,1,float16,float16,7,0.05014240145683289
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,8,128,1,float16,float16,63,0.059438401460647584
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,8,128,1,float16,float16,15,0.05055360198020935
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,8,128,1,float16,float16,31,0.051236802339553834
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,4,128,1,float16,float16,2047,0.4693552017211914
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,8,128,1,float16,float16,127,0.07956799864768982
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,8,128,1,float16,float16,255,0.12223199605941773
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,1,128,1,float16,float16,1,0.027532801032066345
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,1,128,1,float16,float16,3,0.02744320034980774
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,1,128,1,float16,float16,7,0.02787039875984192
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,8,128,1,float16,float16,511,0.29555840492248536
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,1,128,1,float16,float16,31,0.027926400303840637
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,1,128,1,float16,float16,15,0.027934399247169495
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,1,128,1,float16,float16,63,0.028431999683380126
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,1,128,1,float16,float16,127,0.035743999481201175
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,1,128,1,float16,float16,255,0.05877439975738526
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,2,128,1,float16,float16,1,0.03494080007076263
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,8,128,1,float16,float16,1023,0.5202832221984863
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,2,128,1,float16,float16,15,0.03508319854736328
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,2,128,1,float16,float16,7,0.03495360016822815
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,1,128,1,float16,float16,511,0.10994080305099488
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,2,128,1,float16,float16,3,0.03450399935245514
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,2,128,1,float16,float16,31,0.03555519878864288
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,2,128,1,float16,float16,63,0.037699198722839354
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,8,128,1,float16,float16,2047,0.9127568244934082
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,2,128,1,float16,float16,127,0.053495997190475465
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,2,128,1,float16,float16,255,0.07542080283164979
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,4,128,1,float16,float16,3,0.05224480032920838
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,4,128,1,float16,float16,1,0.052142399549484256
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,2,128,1,float16,float16,511,0.16604959964752197
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,4,128,1,float16,float16,15,0.05246400237083435
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,4,128,1,float16,float16,31,0.05310720205307007
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,4,128,1,float16,float16,7,0.052116799354553225
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,4,128,1,float16,float16,63,0.06276000142097474
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,4,128,1,float16,float16,255,0.12561919689178466
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,4,128,1,float16,float16,127,0.08432160019874572
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,8,128,1,float16,float16,1,0.08904799818992615
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,8,128,1,float16,float16,3,0.08903520107269287
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,8,128,1,float16,float16,7,0.08936960101127625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,8,128,1,float16,float16,15,0.0898912012577057
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,4,128,1,float16,float16,511,0.30119359493255615
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,8,128,1,float16,float16,31,0.09437919855117798
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,8,128,1,float16,float16,63,0.10089279413223266
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,8,128,1,float16,float16,127,0.14124959707260132
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,64,1,128,1,float16,float16,3,0.040012800693511964
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,8,128,1,float16,float16,255,0.22470240592956542
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,64,1,128,1,float16,float16,7,0.04035199880599975
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,64,1,128,1,float16,float16,1,0.04070079922676086
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,64,1,128,1,float16,float16,31,0.04190399944782257
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,64,1,128,1,float16,float16,15,0.040750399231910706
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,64,1,128,1,float16,float16,63,0.05116159915924072
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,64,1,128,1,float16,float16,255,0.08822879791259766
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,64,2,128,1,float16,float16,1,0.057231998443603514
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,64,1,128,1,float16,float16,127,0.06344000101089478
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,8,128,1,float16,float16,511,0.5533360004425049
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,64,2,128,1,float16,float16,3,0.05776320099830627
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,64,2,128,1,float16,float16,15,0.058350402116775515
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,64,2,128,1,float16,float16,7,0.05745120048522949
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,64,2,128,1,float16,float16,127,0.09027040004730225
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,64,2,128,1,float16,float16,31,0.06155359745025635
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,64,2,128,1,float16,float16,63,0.06971200108528137
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,64,4,128,1,float16,float16,3,0.09455999732017517
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,64,4,128,1,float16,float16,7,0.09431359767913819
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,64,4,128,1,float16,float16,15,0.09649279713630676
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,64,2,128,1,float16,float16,255,0.13737599849700927
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,64,4,128,1,float16,float16,31,0.10234400033950805
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,64,4,128,1,float16,float16,1,0.0943615972995758
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,64,4,128,1,float16,float16,63,0.10750399827957154
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,64,4,128,1,float16,float16,127,0.1478976011276245
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,64,8,128,1,float16,float16,3,0.16618720293045045
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,64,8,128,1,float16,float16,7,0.16654080152511597
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,64,8,128,1,float16,float16,1,0.16560640335083007
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,64,4,128,1,float16,float16,255,0.23405919075012208
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,64,8,128,1,float16,float16,15,0.17224160432815552
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,64,8,128,1,float16,float16,31,0.17634880542755127
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,1,128,1,float16,float16,1,0.01438560038805008
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,64,8,128,1,float16,float16,63,0.18433120250701904
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,1,128,1,float16,float16,3,0.01422400027513504
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,1,128,1,float16,float16,15,0.014273600280284881
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,1,128,1,float16,float16,31,0.014399999380111694
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,1,128,1,float16,float16,7,0.014158399403095245
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,64,8,128,1,float16,float16,127,0.2619663953781128
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,1,128,1,float16,float16,127,0.0169295996427536
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,1,128,1,float16,float16,255,0.021878400444984437
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,1,128,1,float16,float16,63,0.014455999433994293
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,1,128,1,float16,float16,511,0.024153600633144378
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,1,128,1,float16,float16,1023,0.02845599949359894
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,2,128,1,float16,float16,1,0.014230400323867798
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,1,128,1,float16,float16,2047,0.02990880012512207
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,64,8,128,1,float16,float16,255,0.4299168109893799
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,2,128,1,float16,float16,3,0.014316800236701965
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,1,128,1,float16,float16,4095,0.03893440067768097
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,2,128,1,float16,float16,15,0.014271999895572662
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,2,128,1,float16,float16,7,0.014875200390815736
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,2,128,1,float16,float16,31,0.014263999462127686
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,2,128,1,float16,float16,63,0.014342400431632995
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,2,128,1,float16,float16,511,0.02414720058441162
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,2,128,1,float16,float16,1023,0.0285504013299942
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,2,128,1,float16,float16,255,0.021844799816608428
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,2,128,1,float16,float16,127,0.017001600563526155
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,2,128,1,float16,float16,2047,0.03709439933300018
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,4,128,1,float16,float16,1,0.014294399321079254
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,4,128,1,float16,float16,3,0.014246399700641631
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,4,128,1,float16,float16,7,0.0143327996134758
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,2,128,1,float16,float16,4095,0.05536159873008728
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,4,128,1,float16,float16,63,0.014419199526309967
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,4,128,1,float16,float16,31,0.014284799993038177
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,4,128,1,float16,float16,15,0.014203199744224548
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,4,128,1,float16,float16,255,0.021886399388313292
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,4,128,1,float16,float16,127,0.017131200432777403
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,4,128,1,float16,float16,511,0.028896000981330872
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,4,128,1,float16,float16,1023,0.037723198533058167
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,8,128,1,float16,float16,3,0.014454400539398194
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,8,128,1,float16,float16,1,0.014372800290584565
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,4,128,1,float16,float16,2047,0.05585600137710571
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,8,128,1,float16,float16,7,0.014401599764823914
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,8,128,1,float16,float16,15,0.014350399374961853
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,8,128,1,float16,float16,31,0.01464959979057312
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,8,128,1,float16,float16,63,0.014644800126552582
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,8,128,1,float16,float16,127,0.01724800020456314
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,4,128,1,float16,float16,4095,0.081387197971344
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,8,128,1,float16,float16,255,0.02208479940891266
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,8,128,1,float16,float16,511,0.04536960124969482
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,64,1,128,1,float16,float16,1,0.06777759790420532
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,8,128,1,float16,float16,1023,0.07000799775123596
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,64,1,128,1,float16,float16,3,0.06821600198745728
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,64,1,128,1,float16,float16,7,0.06901760101318359
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,8,128,1,float16,float16,2047,0.11037280559539794
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,8,128,1,float16,float16,4095,0.1904911994934082
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,64,1,128,1,float16,float16,31,0.07078400254249573
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,64,1,128,1,float16,float16,15,0.06958720088005066
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,64,1,128,1,float16,float16,127,0.09644479751586914
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,64,2,128,1,float16,float16,1,0.10190240144729615
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,64,1,128,1,float16,float16,63,0.07582560181617737
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,64,2,128,1,float16,float16,3,0.10368160009384156
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,64,2,128,1,float16,float16,7,0.10476160049438477
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,64,2,128,1,float16,float16,15,0.10570240020751953
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,64,2,128,1,float16,float16,31,0.10622400045394897
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,64,2,128,1,float16,float16,63,0.11225119829177857
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,64,2,128,1,float16,float16,127,0.15385600328445434
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,64,4,128,1,float16,float16,1,0.17296639680862427
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,64,4,128,1,float16,float16,7,0.17650560140609742
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,64,4,128,1,float16,float16,3,0.17375359535217286
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,64,4,128,1,float16,float16,15,0.1788815975189209
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,64,4,128,1,float16,float16,31,0.1799407958984375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,64,4,128,1,float16,float16,63,0.188811194896698
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,64,8,128,1,float16,float16,1,0.31665279865264895
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,64,8,128,1,float16,float16,3,0.3185168027877808
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,64,4,128,1,float16,float16,127,0.27096478939056395
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,64,8,128,1,float16,float16,7,0.3197887897491455
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,64,8,128,1,float16,float16,31,0.3287152051925659
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,64,8,128,1,float16,float16,15,0.32435040473937987
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,64,1,128,1,float16,float16,1,0.1117408037185669
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,64,8,128,1,float16,float16,63,0.34281439781188966
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,64,1,128,1,float16,float16,3,0.11324000358581543
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,64,1,128,1,float16,float16,7,0.11375999450683594
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,64,1,128,1,float16,float16,31,0.11594879627227783
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,64,1,128,1,float16,float16,15,0.11508799791336059
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,64,1,128,1,float16,float16,63,0.12187360525131226
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,64,8,128,1,float16,float16,127,0.5006847858428956
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,64,2,128,1,float16,float16,3,0.18310400247573852
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,64,2,128,1,float16,float16,1,0.18262879848480223
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,64,2,128,1,float16,float16,7,0.18438080549240113
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,64,2,128,1,float16,float16,15,0.18592959642410278
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,64,2,128,1,float16,float16,31,0.18907359838485718
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,64,2,128,1,float16,float16,63,0.19942879676818848
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,64,4,128,1,float16,float16,3,0.32545440196990966
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,64,4,128,1,float16,float16,1,0.3240528106689453
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,64,4,128,1,float16,float16,7,0.3292608022689819
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,64,4,128,1,float16,float16,15,0.3320352077484131
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,64,4,128,1,float16,float16,31,0.3369999885559082
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,64,4,128,1,float16,float16,63,0.35236639976501466
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,1,128,1,float16,float16,1,0.014300799369812012
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,64,8,128,1,float16,float16,1,0.6126880168914794
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,64,8,128,1,float16,float16,3,0.6115071773529053
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,1,128,1,float16,float16,3,0.0142752006649971
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,1,128,1,float16,float16,7,0.014428800344467163
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,64,8,128,1,float16,float16,7,0.6188511848449707
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,64,8,128,1,float16,float16,15,0.6230031967163085
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,1,128,1,float16,float16,63,0.014496000111103058
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,1,128,1,float16,float16,31,0.014433600008487701
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,1,128,1,float16,float16,15,0.014343999326229095
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,1,128,1,float16,float16,127,0.01711679995059967
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,64,8,128,1,float16,float16,31,0.6357135772705078
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,1,128,1,float16,float16,511,0.025772801041603087
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,1,128,1,float16,float16,1023,0.02985279858112335
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,1,128,1,float16,float16,2047,0.03920480012893677
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,1,128,1,float16,float16,255,0.021881599724292756
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,64,8,128,1,float16,float16,63,0.6616687774658203
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,2,128,1,float16,float16,3,0.014326399564743042
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,2,128,1,float16,float16,1,0.014396800100803376
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,2,128,1,float16,float16,7,0.0143327996134758
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,1,128,1,float16,float16,4095,0.057227200269699095
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,2,128,1,float16,float16,15,0.014326399564743042
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,2,128,1,float16,float16,31,0.014524799585342408
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,2,128,1,float16,float16,127,0.016979199647903443
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,2,128,1,float16,float16,63,0.014679999649524688
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,2,128,1,float16,float16,255,0.02195200026035309
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,2,128,1,float16,float16,511,0.02941280007362366
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,2,128,1,float16,float16,2047,0.055511999130249026
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,2,128,1,float16,float16,1023,0.038843199610710144
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,4,128,1,float16,float16,3,0.014371199905872345
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,4,128,1,float16,float16,1,0.014550399780273438
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,4,128,1,float16,float16,7,0.014368000626564025
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,2,128,1,float16,float16,4095,0.08280320167541504
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,4,128,1,float16,float16,31,0.014545600116252898
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,4,128,1,float16,float16,15,0.01446399986743927
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,4,128,1,float16,float16,63,0.014595200121402741
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,4,128,1,float16,float16,255,0.02210240066051483
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,4,128,1,float16,float16,127,0.017484800517559053
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,4,128,1,float16,float16,511,0.047092801332473753
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,4,128,1,float16,float16,1023,0.07143999934196472
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,8,128,1,float16,float16,1,0.016355200111865996
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,8,128,1,float16,float16,3,0.01653279960155487
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,4,128,1,float16,float16,2047,0.11205439567565918
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,8,128,1,float16,float16,7,0.016284799575805663
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,4,128,1,float16,float16,4095,0.1915120005607605
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,8,128,1,float16,float16,31,0.016582399606704712
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,8,128,1,float16,float16,15,0.016521599888801575
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,8,128,1,float16,float16,63,0.016935999691486358
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,8,128,1,float16,float16,255,0.029238399863243104
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,8,128,1,float16,float16,127,0.02099040001630783
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,1,128,1,float16,float16,3,0.015756799280643462
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,8,128,1,float16,float16,511,0.06938719749450684
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,1,128,1,float16,float16,1,0.015809600055217744
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,8,128,1,float16,float16,1023,0.11267839670181275
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,1,128,1,float16,float16,7,0.015772800147533416
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,1,128,1,float16,float16,31,0.015969599783420562
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,1,128,1,float16,float16,15,0.015838399529457092
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,8,128,1,float16,float16,2047,0.19196799993515015
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,1,128,1,float16,float16,127,0.018545599281787874
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,8,128,1,float16,float16,4095,0.35253760814666746
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,1,128,1,float16,float16,63,0.01592479944229126
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,1,128,1,float16,float16,255,0.023470400273799895
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,1,128,1,float16,float16,511,0.05002239942550659
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,2,128,1,float16,float16,1,0.01754239946603775
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,1,128,1,float16,float16,1023,0.07521600127220154
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,2,128,1,float16,float16,3,0.01757279932498932
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,1,128,1,float16,float16,2047,0.11453280448913575
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,2,128,1,float16,float16,7,0.017683200538158417
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,2,128,1,float16,float16,15,0.01761920005083084
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,1,128,1,float16,float16,4095,0.19560480117797852
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,2,128,1,float16,float16,31,0.017798399925231932
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,2,128,1,float16,float16,127,0.022254399955272675
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,2,128,1,float16,float16,63,0.0179407998919487
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,2,128,1,float16,float16,255,0.030294400453567506
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,1,128,1,float16,float16,8191,0.3584367990493774
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,2,128,1,float16,float16,511,0.0718176007270813
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,2,128,1,float16,float16,1023,0.11531679630279541
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,4,128,1,float16,float16,1,0.023396800458431243
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,2,128,1,float16,float16,2047,0.19515039920806884
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,4,128,1,float16,float16,3,0.023563200235366823
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,4,128,1,float16,float16,7,0.0233487993478775
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,2,128,1,float16,float16,4095,0.3543936014175415
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,4,128,1,float16,float16,31,0.023712000250816344
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,4,128,1,float16,float16,63,0.024241599440574645
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,4,128,1,float16,float16,15,0.023611199855804444
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,4,128,1,float16,float16,127,0.03020319938659668
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,4,128,1,float16,float16,255,0.04877119958400726
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,4,128,1,float16,float16,511,0.10054559707641601
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,2,128,1,float16,float16,8191,0.6801536083221436
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,4,128,1,float16,float16,1023,0.165937602519989
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,4,128,1,float16,float16,2047,0.28531360626220703
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,8,128,1,float16,float16,1,0.03129439949989319
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,8,128,1,float16,float16,7,0.031350401043891904
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,8,128,1,float16,float16,3,0.030844798684120177
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,8,128,1,float16,float16,15,0.031462401151657104
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,8,128,1,float16,float16,31,0.031611201167106626
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,8,128,1,float16,float16,63,0.03234399855136871
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,8,128,1,float16,float16,127,0.04679679870605469
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,4,128,1,float16,float16,4095,0.5250383853912354
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,8,128,1,float16,float16,255,0.07279999852180481
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,8,128,1,float16,float16,511,0.15751359462738038
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,8,128,1,float16,float16,1023,0.26621758937835693
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,1,128,1,float16,float16,1,0.013969600200653076
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,1,128,1,float16,float16,3,0.01422560065984726
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,1,128,1,float16,float16,31,0.014192000031471252
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,1,128,1,float16,float16,7,0.013991999626159667
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,8,128,1,float16,float16,2047,0.46582717895507814
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,1,128,1,float16,float16,15,0.013944000005722046
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,4,128,1,float16,float16,8191,1.0142607688903809
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,1,128,1,float16,float16,63,0.014230400323867798
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,1,128,1,float16,float16,127,0.016553600132465363
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,1,128,1,float16,float16,511,0.02285120040178299
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,1,128,1,float16,float16,2047,0.027724799513816834
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,1,128,1,float16,float16,4095,0.0284496009349823
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,1,128,1,float16,float16,8191,0.030460798740386964
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,1,128,1,float16,float16,255,0.02157599925994873
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,8,128,1,float16,float16,4095,0.8672191619873046
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,2,128,1,float16,float16,1,0.014316800236701965
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,1,128,1,float16,float16,1023,0.0272816002368927
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,2,128,1,float16,float16,3,0.013924799859523773
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,2,128,1,float16,float16,7,0.013811199367046357
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,2,128,1,float16,float16,63,0.014449599385261535
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,2,128,1,float16,float16,15,0.01419840008020401
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,2,128,1,float16,float16,127,0.016577599942684172
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,2,128,1,float16,float16,255,0.02146400064229965
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,2,128,1,float16,float16,511,0.022812800109386445
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,2,128,1,float16,float16,31,0.014620800316333771
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,2,128,1,float16,float16,2047,0.027564799785614012
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,2,128,1,float16,float16,1023,0.02712959945201874
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,2,128,1,float16,float16,4095,0.02880159914493561
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,2,128,1,float16,float16,8191,0.03078719973564148
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,4,128,1,float16,float16,1,0.014230400323867798
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,4,128,1,float16,float16,3,0.014006400108337402
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,4,128,1,float16,float16,7,0.014076800644397735
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,4,128,1,float16,float16,15,0.014044800400733947
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,4,128,1,float16,float16,31,0.014155200123786927
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,4,128,1,float16,float16,63,0.014579200744628906
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,4,128,1,float16,float16,127,0.016862399876117706
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,4,128,1,float16,float16,255,0.02150239944458008
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,4,128,1,float16,float16,511,0.022382399439811705
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,4,128,1,float16,float16,1023,0.026868799328804018
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,4,128,1,float16,float16,4095,0.028830400109291075
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,8,128,1,float16,float16,1,0.013944000005722046
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,4,128,1,float16,float16,8191,0.0375791996717453
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,4,128,1,float16,float16,2047,0.028091201186180116
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,8,128,1,float16,float16,3,0.014078399538993836
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,8,128,1,float16,float16,7,0.014025600254535675
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,8,128,1,float16,float16,15,0.013996799290180207
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,8,128,1,float16,float16,31,0.014107200503349304
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,8,128,1,float16,float16,63,0.014441600441932679
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,8,128,1,float16,float16,127,0.01687999963760376
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,8,128,1,float16,float16,255,0.021615999937057494
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,8,128,1,float16,float16,511,0.022737599909305573
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,8,128,1,float16,float16,1023,0.02731040120124817
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,8,128,1,float16,float16,4095,0.03698239922523498
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,1,128,1,float16,float16,1,0.013799999654293061
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,8,128,1,float16,float16,2047,0.02855679988861084
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,8,128,1,float16,float16,8191,1.6911231994628906
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,8,128,1,float16,float16,8191,0.05371999740600586
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,1,128,1,float16,float16,3,0.013952000439167023
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,1,128,1,float16,float16,7,0.01419679969549179
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,1,128,1,float16,float16,15,0.01393119990825653
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,1,128,1,float16,float16,255,0.02157920002937317
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,1,128,1,float16,float16,31,0.01446239948272705
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,1,128,1,float16,float16,511,0.023080000281333925
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,1,128,1,float16,float16,127,0.01671999990940094
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,1,128,1,float16,float16,1023,0.02717759907245636
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,1,128,1,float16,float16,63,0.01507679969072342
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,1,128,1,float16,float16,2047,0.028016000986099243
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,2,128,1,float16,float16,1,0.013831999897956849
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,1,128,1,float16,float16,8191,0.03107360005378723
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,1,128,1,float16,float16,4095,0.029635199904441835
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,2,128,1,float16,float16,3,0.013991999626159667
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,2,128,1,float16,float16,7,0.014745600521564484
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,2,128,1,float16,float16,15,0.014001600444316864
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,2,128,1,float16,float16,31,0.013806399703025819
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,2,128,1,float16,float16,255,0.02143840044736862
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,2,128,1,float16,float16,63,0.014361600577831268
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,2,128,1,float16,float16,127,0.01701440066099167
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,2,128,1,float16,float16,1023,0.027167999744415285
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,2,128,1,float16,float16,511,0.024076800048351287
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,2,128,1,float16,float16,2047,0.028007999062538147
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,2,128,1,float16,float16,4095,0.02885279953479767
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,4,128,1,float16,float16,1,0.014188799262046813
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,2,128,1,float16,float16,8191,0.03882080018520355
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,4,128,1,float16,float16,3,0.013929599523544311
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,4,128,1,float16,float16,15,0.013988800346851349
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,4,128,1,float16,float16,7,0.014497600495815277
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,4,128,1,float16,float16,63,0.014481599628925323
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,4,128,1,float16,float16,31,0.0141744002699852
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,4,128,1,float16,float16,255,0.021595199406147004
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,4,128,1,float16,float16,511,0.022835199534893037
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,4,128,1,float16,float16,127,0.016867199540138246
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,4,128,1,float16,float16,2047,0.028675198554992676
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,4,128,1,float16,float16,4095,0.03690080046653747
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,4,128,1,float16,float16,1023,0.028040000796318056
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,8,128,1,float16,float16,1,0.014220799505710601
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,8,128,1,float16,float16,7,0.014215999841690063
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,8,128,1,float16,float16,3,0.014100800454616546
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,4,128,1,float16,float16,8191,0.05443840026855469
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,8,128,1,float16,float16,15,0.01419840008020401
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,8,128,1,float16,float16,255,0.021931199729442595
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,8,128,1,float16,float16,127,0.016910399496555328
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,8,128,1,float16,float16,31,0.014180800318717957
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,8,128,1,float16,float16,63,0.014519999921321868
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,8,128,1,float16,float16,511,0.023452800512313843
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,8,128,1,float16,float16,1023,0.02776159942150116
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,8,128,1,float16,float16,2047,0.03683040142059326
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,1,128,1,float16,float16,1,0.014804799854755402
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,1,128,1,float16,float16,3,0.015174399316310882
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,1,128,1,float16,float16,7,0.01496639996767044
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,1,128,1,float16,float16,15,0.014998400211334228
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,8,128,1,float16,float16,4095,0.05330560207366943
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,8,128,1,float16,float16,8191,0.08137919902801513
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,1,128,1,float16,float16,31,0.015001599490642548
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,1,128,1,float16,float16,63,0.015086400508880615
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,1,128,1,float16,float16,255,0.0225600004196167
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,1,128,1,float16,float16,511,0.030107200145721436
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,1,128,1,float16,float16,127,0.017819200456142426
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,1,128,1,float16,float16,1023,0.039419201016426084
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,1,128,1,float16,float16,2047,0.05645120143890381
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,2,128,1,float16,float16,1,0.014979200065135955
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,2,128,1,float16,float16,3,0.015031999349594117
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,1,128,1,float16,float16,4095,0.08325279951095581
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,2,128,1,float16,float16,31,0.01515360027551651
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,2,128,1,float16,float16,15,0.015192000567913056
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,2,128,1,float16,float16,7,0.015193599462509155
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,1,128,1,float16,float16,8191,0.13628319501876832
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,2,128,1,float16,float16,63,0.015228800475597382
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,2,128,1,float16,float16,255,0.02274879962205887
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,2,128,1,float16,float16,127,0.017983999848365784
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,2,128,1,float16,float16,511,0.04776639938354492
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,4,128,1,float16,float16,1,0.016993600130081176
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,2,128,1,float16,float16,1023,0.07232800126075745
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,2,128,1,float16,float16,2047,0.11219359636306762
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,4,128,1,float16,float16,3,0.016908800601959227
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,4,128,1,float16,float16,7,0.017073599994182585
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,2,128,1,float16,float16,4095,0.19288480281829834
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,4,128,1,float16,float16,63,0.01733119934797287
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,4,128,1,float16,float16,15,0.01717280000448227
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,4,128,1,float16,float16,31,0.017159999907016756
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,4,128,1,float16,float16,127,0.021796800196170807
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,4,128,1,float16,float16,255,0.03003840148448944
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,2,128,1,float16,float16,8191,0.3560271978378296
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,4,128,1,float16,float16,511,0.07082239985466003
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,4,128,1,float16,float16,1023,0.11436480283737183
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,8,128,1,float16,float16,3,0.022894400358200073
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,8,128,1,float16,float16,1,0.022734400629997254
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,4,128,1,float16,float16,2047,0.19347200393676758
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,8,128,1,float16,float16,15,0.022912000119686127
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,8,128,1,float16,float16,7,0.02306720018386841
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,4,128,1,float16,float16,4095,0.3537456035614014
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,8,128,1,float16,float16,63,0.023473599553108217
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,8,128,1,float16,float16,127,0.029734399914741517
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,8,128,1,float16,float16,31,0.023203200101852416
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,8,128,1,float16,float16,255,0.04564639925956726
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,4,128,1,float16,float16,8191,0.6804800033569336
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,8,128,1,float16,float16,511,0.09880160093307495
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,8,128,1,float16,float16,1023,0.16340320110321044
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,1,128,1,float16,float16,1,0.01403679996728897
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,8,128,1,float16,float16,2047,0.28364479541778564
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,1,128,1,float16,float16,3,0.01417279988527298
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,1,128,1,float16,float16,7,0.014020800590515137
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,1,128,1,float16,float16,15,0.014006400108337402
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,1,128,1,float16,float16,63,0.014435200393199921
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,1,128,1,float16,float16,31,0.014139199256896972
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,8,128,1,float16,float16,4095,0.52325439453125
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,1,128,1,float16,float16,127,0.01669120043516159
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,1,128,1,float16,float16,255,0.021660800278186797
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,1,128,1,float16,float16,511,0.023395200073719025
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,1,128,1,float16,float16,4095,0.029300799965858458
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,2,128,1,float16,float16,1,0.01417119950056076
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,1,128,1,float16,float16,1023,0.027432000637054442
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,1,128,1,float16,float16,2047,0.028302401304244995
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,1,128,1,float16,float16,8191,0.038756799697875974
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,2,128,1,float16,float16,7,0.014268800616264343
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,2,128,1,float16,float16,3,0.014068800210952758
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,2,128,1,float16,float16,15,0.014220799505710601
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,2,128,1,float16,float16,31,0.014052799344062806
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,2,128,1,float16,float16,127,0.017115199565887453
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,2,128,1,float16,float16,255,0.021550400555133818
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,2,128,1,float16,float16,63,0.01419840008020401
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,2,128,1,float16,float16,511,0.023240000009536743
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,8,128,1,float16,float16,8191,1.0149696350097657
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,2,128,1,float16,float16,2047,0.02879520058631897
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,2,128,1,float16,float16,1023,0.02771199941635132
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,4,128,1,float16,float16,1,0.014124800264835358
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,2,128,1,float16,float16,4095,0.037161600589752194
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,4,128,1,float16,float16,7,0.014244799315929414
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,4,128,1,float16,float16,31,0.014169600605964661
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,4,128,1,float16,float16,15,0.014103999733924866
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,2,128,1,float16,float16,8191,0.054980802536010745
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,4,128,1,float16,float16,63,0.014238399267196656
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,4,128,1,float16,float16,3,0.014289599657058717
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,4,128,1,float16,float16,127,0.016924799978733064
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,4,128,1,float16,float16,511,0.0237296000123024
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,4,128,1,float16,float16,255,0.02173919975757599
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,4,128,1,float16,float16,1023,0.02786239981651306
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,8,128,1,float16,float16,1,0.014435200393199921
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,4,128,1,float16,float16,2047,0.03651039898395538
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,8,128,1,float16,float16,3,0.014201599359512328
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,8,128,1,float16,float16,7,0.014268800616264343
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,8,128,1,float16,float16,15,0.014315199851989747
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,4,128,1,float16,float16,8191,0.08143680095672608
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,8,128,1,float16,float16,31,0.014390400052070618
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,4,128,1,float16,float16,4095,0.054897600412368776
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,8,128,1,float16,float16,127,0.017000000178813934
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,8,128,1,float16,float16,63,0.014430400729179383
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,8,128,1,float16,float16,255,0.021857599914073943
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,8,128,1,float16,float16,511,0.028169599175453187
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,8,128,1,float16,float16,1023,0.03723520040512085
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,1,128,1,float16,float16,1,0.018654400110244752
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,8,128,1,float16,float16,2047,0.05362880229949951
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,1,128,1,float16,float16,3,0.018811200559139252
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,1,128,1,float16,float16,7,0.01881439983844757
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,8,128,1,float16,float16,8191,0.13382400274276735
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,1,128,1,float16,float16,15,0.0189423993229866
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,1,128,1,float16,float16,31,0.018915200233459474
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,1,128,1,float16,float16,127,0.02340639978647232
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,8,128,1,float16,float16,4095,0.08149600028991699
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,1,128,1,float16,float16,255,0.031488001346588135
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,1,128,1,float16,float16,63,0.01942719966173172
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,2,128,1,float16,float16,3,0.024830399453639983
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,1,128,1,float16,float16,511,0.07390720248222352
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,2,128,1,float16,float16,1,0.024641600251197816
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,1,128,1,float16,float16,1023,0.1178976058959961
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,2,128,1,float16,float16,7,0.02476319968700409
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,1,128,1,float16,float16,2047,0.19690719842910767
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,2,128,1,float16,float16,15,0.02470880001783371
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,2,128,1,float16,float16,31,0.024833600223064422
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,2,128,1,float16,float16,63,0.02510559856891632
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,2,128,1,float16,float16,127,0.03167040050029755
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,2,128,1,float16,float16,255,0.04848800003528595
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,2,128,1,float16,float16,511,0.10215200185775757
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,2,128,1,float16,float16,1023,0.16784800291061402
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,1,128,1,float16,float16,4095,0.3567359924316406
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,4,128,1,float16,float16,1,0.03193280100822449
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,2,128,1,float16,float16,2047,0.2865920066833496
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,4,128,1,float16,float16,3,0.032227200269699094
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,4,128,1,float16,float16,31,0.032760000228881835
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,4,128,1,float16,float16,7,0.03221920132637024
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,4,128,1,float16,float16,15,0.03273760080337525
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,4,128,1,float16,float16,63,0.03380799889564514
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,4,128,1,float16,float16,127,0.04955039918422699
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,2,128,1,float16,float16,4095,0.5291600227355957
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,4,128,1,float16,float16,255,0.07301279902458191
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,4,128,1,float16,float16,511,0.15879039764404296
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,8,128,1,float16,float16,3,0.04976319968700409
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,8,128,1,float16,float16,1,0.050097602605819705
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,8,128,1,float16,float16,7,0.04997439980506897
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,8,128,1,float16,float16,15,0.05014560222625732
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,4,128,1,float16,float16,1023,0.2692528009414673
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,8,128,1,float16,float16,63,0.05815520286560059
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,8,128,1,float16,float16,31,0.051571202278137204
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,8,128,1,float16,float16,127,0.07878559827804565
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,4,128,1,float16,float16,2047,0.46713762283325194
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,8,128,1,float16,float16,255,0.12239359617233277
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,4,128,1,float16,float16,4095,0.8669887542724609
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,8,128,1,float16,float16,511,0.29453439712524415
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,1,128,1,float16,float16,1,0.026791998744010927
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,1,128,1,float16,float16,3,0.02656799852848053
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,1,128,1,float16,float16,7,0.02670240104198456
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,8,128,1,float16,float16,1023,0.5210415840148925
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,1,128,1,float16,float16,15,0.026927998661994933
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,1,128,1,float16,float16,31,0.026924800872802735
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,1,128,1,float16,float16,127,0.03365119993686676
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,1,128,1,float16,float16,63,0.02729920148849487
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,2,128,1,float16,float16,1,0.03401919901371002
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,1,128,1,float16,float16,255,0.05502560138702393
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,8,128,1,float16,float16,2047,0.9133343696594238
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,1,128,1,float16,float16,511,0.1066480040550232
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,2,128,1,float16,float16,3,0.03405919969081879
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,1,128,1,float16,float16,1023,0.17209919691085815
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,2,128,1,float16,float16,7,0.034092798829078674
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,2,128,1,float16,float16,15,0.0342960000038147
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,2,128,1,float16,float16,31,0.034414398670196536
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,2,128,1,float16,float16,63,0.03595679998397827
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,2,128,1,float16,float16,255,0.07330080270767211
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,2,128,1,float16,float16,127,0.05281599760055542
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,2,128,1,float16,float16,511,0.16396960020065307
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,4,128,1,float16,float16,1,0.05175039768218994
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,4,128,1,float16,float16,3,0.051937597990036014
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,4,128,1,float16,float16,7,0.05174559950828552
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,4,128,1,float16,float16,15,0.052193599939346316
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,2,128,1,float16,float16,1023,0.2728303909301758
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,4,128,1,float16,float16,31,0.05272799730300903
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,4,128,1,float16,float16,63,0.06192319989204407
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,8,128,1,float16,float16,4095,1.712081527709961
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,4,128,1,float16,float16,127,0.08133919835090637
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,4,128,1,float16,float16,255,0.12478879690170289
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,8,128,1,float16,float16,1,0.08889279961585998
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,8,128,1,float16,float16,3,0.0885807991027832
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,8,128,1,float16,float16,15,0.08994879722595214
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,8,128,1,float16,float16,63,0.09965440034866332
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,8,128,1,float16,float16,31,0.0927839994430542
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,8,128,1,float16,float16,7,0.09277120232582092
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,4,128,1,float16,float16,1023,0.522267198562622
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,8,128,1,float16,float16,127,0.1386304020881653
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,4,128,1,float16,float16,511,0.29847040176391604
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,1,128,1,float16,float16,1,0.03868640065193176
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,1,128,1,float16,float16,7,0.03886080086231232
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,1,128,1,float16,float16,3,0.03889119923114777
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,1,128,1,float16,float16,31,0.03928000032901764
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,1,128,1,float16,float16,15,0.039017599821090695
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,1,128,1,float16,float16,63,0.04143680036067963
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,8,128,1,float16,float16,255,0.22302560806274413
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,1,128,1,float16,float16,127,0.059564799070358276
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,2,128,1,float16,float16,1,0.05623999834060669
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,1,128,1,float16,float16,255,0.0793615996837616
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,8,128,1,float16,float16,511,0.5519296169281006
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,2,128,1,float16,float16,7,0.05651519894599914
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,2,128,1,float16,float16,15,0.05649120211601257
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,2,128,1,float16,float16,3,0.056599998474121095
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,2,128,1,float16,float16,31,0.05733280181884766
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,1,128,1,float16,float16,511,0.1741312026977539
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,2,128,1,float16,float16,63,0.06702399849891663
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,2,128,1,float16,float16,127,0.09640799760818482
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,2,128,1,float16,float16,255,0.1355072021484375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,4,128,1,float16,float16,1,0.09342399835586548
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,4,128,1,float16,float16,3,0.09340479969978333
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,4,128,1,float16,float16,15,0.09402719736099244
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,8,128,1,float16,float16,1023,0.9892271995544434
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,4,128,1,float16,float16,7,0.0935151994228363
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,4,128,1,float16,float16,31,0.09933599829673767
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,4,128,1,float16,float16,63,0.10543040037155152
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,2,128,1,float16,float16,511,0.3095072031021118
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,4,128,1,float16,float16,127,0.14451999664306642
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,8,128,1,float16,float16,1,0.16396960020065307
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,8,128,1,float16,float16,3,0.16457120180130005
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,8,128,1,float16,float16,15,0.1684015989303589
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,4,128,1,float16,float16,255,0.23715360164642335
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,8,128,1,float16,float16,7,0.16542400121688844
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,8,128,1,float16,float16,31,0.17422879934310914
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,1,128,1,float16,float16,1,0.014230400323867798
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,4,128,1,float16,float16,511,0.5598303794860839
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,1,128,1,float16,float16,3,0.014126400649547576
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,8,128,1,float16,float16,63,0.18152320384979248
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,1,128,1,float16,float16,7,0.014259199798107147
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,1,128,1,float16,float16,127,0.01677920073270798
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,1,128,1,float16,float16,31,0.01420000046491623
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,8,128,1,float16,float16,127,0.25996320247650145
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,1,128,1,float16,float16,15,0.014020800590515137
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,1,128,1,float16,float16,63,0.014319999516010285
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,1,128,1,float16,float16,255,0.02181600034236908
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,8,128,1,float16,float16,255,0.43371357917785647
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,1,128,1,float16,float16,511,0.02341119945049286
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,1,128,1,float16,float16,1023,0.02794399857521057
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,1,128,1,float16,float16,4095,0.03761279881000519
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,1,128,1,float16,float16,2047,0.028832000494003297
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,8,128,1,float16,float16,511,1.0780511856079102
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,2,128,1,float16,float16,3,0.014135999977588654
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,1,128,1,float16,float16,8191,0.05512319803237915
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,2,128,1,float16,float16,31,0.014227199554443359
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,2,128,1,float16,float16,7,0.014353600144386292
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,2,128,1,float16,float16,1,0.014140799641609192
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,2,128,1,float16,float16,15,0.014094400405883788
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,2,128,1,float16,float16,511,0.023708799481391908
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,2,128,1,float16,float16,255,0.021748800575733186
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,2,128,1,float16,float16,1023,0.028391999006271363
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,2,128,1,float16,float16,127,0.016892799735069276
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,2,128,1,float16,float16,63,0.01451680064201355
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,2,128,1,float16,float16,2047,0.03731679916381836
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,2,128,1,float16,float16,4095,0.05450720191001892
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,4,128,1,float16,float16,1,0.014348800480365752
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,4,128,1,float16,float16,7,0.014190399646759033
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,4,128,1,float16,float16,3,0.014291200041770934
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,4,128,1,float16,float16,15,0.014267200231552124
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,2,128,1,float16,float16,8191,0.08184000253677368
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,4,128,1,float16,float16,255,0.02171359956264496
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,4,128,1,float16,float16,127,0.0169855996966362
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,4,128,1,float16,float16,31,0.014374400675296783
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,4,128,1,float16,float16,511,0.02826080024242401
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,4,128,1,float16,float16,63,0.014431999623775482
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,4,128,1,float16,float16,2047,0.0542464017868042
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,4,128,1,float16,float16,1023,0.03753120005130768
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,8,128,1,float16,float16,1,0.014396800100803376
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,8,128,1,float16,float16,7,0.014414399862289429
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,8,128,1,float16,float16,3,0.014508800208568573
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,8,128,1,float16,float16,15,0.014460800588130951
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,4,128,1,float16,float16,4095,0.08113759756088257
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,8,128,1,float16,float16,127,0.017294399440288544
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,8,128,1,float16,float16,255,0.022283199429512023
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,4,128,1,float16,float16,8191,0.13441760540008546
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,8,128,1,float16,float16,63,0.014689600467681885
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,8,128,1,float16,float16,31,0.014545600116252898
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,8,128,1,float16,float16,511,0.045337599515914914
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,8,128,1,float16,float16,1023,0.06986719965934754
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,8,128,1,float16,float16,2047,0.11009600162506103
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,32,1,128,1,float16,float16,1,0.05758240222930908
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,32,1,128,1,float16,float16,3,0.058284801244735715
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,32,1,128,1,float16,float16,7,0.057848000526428224
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,8,128,1,float16,float16,4095,0.1907039999961853
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,32,1,128,1,float16,float16,15,0.058483201265335086
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,32,1,128,1,float16,float16,31,0.061617600917816165
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,32,1,128,1,float16,float16,63,0.06964480280876159
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,8,128,1,float16,float16,8191,0.35410399436950685
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,32,2,128,1,float16,float16,3,0.09408159852027893
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,32,2,128,1,float16,float16,7,0.09483199715614318
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,32,2,128,1,float16,float16,1,0.0947055995464325
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,32,1,128,1,float16,float16,255,0.13248640298843384
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,32,1,128,1,float16,float16,127,0.08969119787216187
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,32,2,128,1,float16,float16,15,0.09728000164031983
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,32,2,128,1,float16,float16,31,0.10402239561080932
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,32,2,128,1,float16,float16,63,0.10801279544830322
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,32,2,128,1,float16,float16,127,0.14744160175323487
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,32,4,128,1,float16,float16,3,0.1666144013404846
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,32,4,128,1,float16,float16,7,0.16782399415969848
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,32,4,128,1,float16,float16,15,0.1745296001434326
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,32,2,128,1,float16,float16,255,0.2360703945159912
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,32,4,128,1,float16,float16,1,0.16619679927825928
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,32,4,128,1,float16,float16,63,0.1848464012145996
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,32,4,128,1,float16,float16,31,0.1775279998779297
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,32,8,128,1,float16,float16,1,0.3112544059753418
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,32,4,128,1,float16,float16,127,0.26367199420928955
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,32,8,128,1,float16,float16,7,0.31558239459991455
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,32,8,128,1,float16,float16,3,0.3161999940872192
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,32,4,128,1,float16,float16,255,0.4361855983734131
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,32,8,128,1,float16,float16,15,0.3208735942840576
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,32,8,128,1,float16,float16,31,0.32510879039764407
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,32,1,128,1,float16,float16,1,0.10397919416427612
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,32,8,128,1,float16,float16,63,0.3380239963531494
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,32,1,128,1,float16,float16,7,0.10460000038146973
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,32,1,128,1,float16,float16,3,0.1056447982788086
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,32,1,128,1,float16,float16,15,0.10639840364456177
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,32,8,128,1,float16,float16,127,0.4968463897705078
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,32,1,128,1,float16,float16,63,0.11290559768676758
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,32,1,128,1,float16,float16,31,0.10784800052642822
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,32,2,128,1,float16,float16,1,0.17493120431900025
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,32,1,128,1,float16,float16,127,0.1538032054901123
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,32,2,128,1,float16,float16,15,0.1794800043106079
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,32,2,128,1,float16,float16,3,0.1785871982574463
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,32,8,128,1,float16,float16,255,0.8258383750915528
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,32,2,128,1,float16,float16,7,0.17754240036010743
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,32,2,128,1,float16,float16,31,0.18160799741744996
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,32,2,128,1,float16,float16,63,0.1907152056694031
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,32,4,128,1,float16,float16,1,0.3187119960784912
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,32,2,128,1,float16,float16,127,0.2702159881591797
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,32,4,128,1,float16,float16,3,0.32057440280914307
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,32,4,128,1,float16,float16,15,0.32665600776672366
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,32,4,128,1,float16,float16,7,0.32819840908050535
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,32,4,128,1,float16,float16,63,0.3427520036697388
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,32,4,128,1,float16,float16,31,0.33080639839172366
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,32,4,128,1,float16,float16,127,0.502675199508667
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,32,8,128,1,float16,float16,3,0.6084752082824707
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,32,8,128,1,float16,float16,1,0.6071375846862793
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,32,8,128,1,float16,float16,7,0.6127952098846435
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,1,128,1,float16,float16,1,0.014399999380111694
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,32,8,128,1,float16,float16,31,0.628326416015625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,32,8,128,1,float16,float16,15,0.6308976173400879
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,1,128,1,float16,float16,3,0.014230400323867798
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,1,128,1,float16,float16,7,0.01422239989042282
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,1,128,1,float16,float16,15,0.014256000518798828
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,32,8,128,1,float16,float16,63,0.6507247924804688
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,1,128,1,float16,float16,31,0.01427839994430542
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,1,128,1,float16,float16,63,0.014561599493026734
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,1,128,1,float16,float16,127,0.01700640022754669
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,1,128,1,float16,float16,511,0.024243199825286867
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,1,128,1,float16,float16,255,0.02175839990377426
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,1,128,1,float16,float16,1023,0.028703999519348145
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,2,128,1,float16,float16,1,0.014431999623775482
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,2,128,1,float16,float16,3,0.01419519931077957
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,1,128,1,float16,float16,2047,0.03762080073356629
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,1,128,1,float16,float16,8191,0.08196160197257996
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,2,128,1,float16,float16,7,0.01435520052909851
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,2,128,1,float16,float16,15,0.014276799559593201
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,1,128,1,float16,float16,4095,0.05568479895591736
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,2,128,1,float16,float16,31,0.01446399986743927
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,2,128,1,float16,float16,63,0.014505599439144135
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,2,128,1,float16,float16,127,0.017051200568675994
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,2,128,1,float16,float16,255,0.021907199919223786
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,2,128,1,float16,float16,1023,0.038524800539016725
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,2,128,1,float16,float16,511,0.028859201073646545
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,2,128,1,float16,float16,2047,0.05424000024795532
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,32,8,128,1,float16,float16,127,0.971560001373291
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,4,128,1,float16,float16,1,0.014486399292945863
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,2,128,1,float16,float16,4095,0.08220800161361694
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,4,128,1,float16,float16,7,0.01446560025215149
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,4,128,1,float16,float16,3,0.014633600413799287
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,2,128,1,float16,float16,8191,0.1347872018814087
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,4,128,1,float16,float16,63,0.014630399644374847
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,4,128,1,float16,float16,255,0.02221599966287613
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,4,128,1,float16,float16,15,0.014560000598430633
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,4,128,1,float16,float16,127,0.017316800355911256
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,4,128,1,float16,float16,31,0.014951999485492706
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,4,128,1,float16,float16,511,0.04671039879322052
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,4,128,1,float16,float16,1023,0.07007039785385132
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,8,128,1,float16,float16,1,0.01627040058374405
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,4,128,1,float16,float16,2047,0.1100816011428833
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,8,128,1,float16,float16,3,0.016859200596809388
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,4,128,1,float16,float16,4095,0.19032959938049315
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,8,128,1,float16,float16,7,0.016380800306797026
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,8,128,1,float16,float16,31,0.01656640022993088
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,8,128,1,float16,float16,15,0.016302399337291718
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,8,128,1,float16,float16,63,0.016926400363445282
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,8,128,1,float16,float16,127,0.021566399931907655
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,4,128,1,float16,float16,8191,0.3536815881729126
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,8,128,1,float16,float16,255,0.029420799016952513
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,8,128,1,float16,float16,511,0.06868799924850463
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,8,128,1,float16,float16,1023,0.11272000074386597
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,1,128,1,float16,float16,3,0.01586720049381256
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,8,128,1,float16,float16,2047,0.19238239526748657
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,1,128,1,float16,float16,1,0.015643200278282164
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,1,128,1,float16,float16,7,0.015662400424480437
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,1,128,1,float16,float16,15,0.01581120043992996
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,1,128,1,float16,float16,31,0.015897600352764128
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,8,128,1,float16,float16,4095,0.3524111986160278
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,1,128,1,float16,float16,255,0.02340639978647232
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,1,128,1,float16,float16,127,0.018512000143527985
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,1,128,1,float16,float16,63,0.01579039990901947
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,1,128,1,float16,float16,511,0.048256000876426695
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,1,128,1,float16,float16,1023,0.0742799997329712
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,2,128,1,float16,float16,1,0.017433600127696992
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,1,128,1,float16,float16,2047,0.11454399824142455
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,2,128,1,float16,float16,3,0.017665599286556245
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,2,128,1,float16,float16,7,0.017348800599575043
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,1,128,1,float16,float16,4095,0.1936959981918335
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,8,128,1,float16,float16,8191,0.6879680156707764
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,2,128,1,float16,float16,15,0.017761600017547608
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,1,128,1,float16,float16,8191,0.35738561153411863
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,2,128,1,float16,float16,31,0.0178384006023407
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,2,128,1,float16,float16,63,0.01775680035352707
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,2,128,1,float16,float16,127,0.02218559980392456
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,2,128,1,float16,float16,255,0.03086079955101013
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,1,128,1,float16,float16,16383,0.6796688079833985
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,2,128,1,float16,float16,1023,0.11493279933929443
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,2,128,1,float16,float16,511,0.07099040150642395
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,2,128,1,float16,float16,2047,0.19481279850006103
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,4,128,1,float16,float16,1,0.02339999973773956
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,4,128,1,float16,float16,3,0.02343039959669113
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,4,128,1,float16,float16,7,0.02339999973773956
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,4,128,1,float16,float16,15,0.023742400109767914
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,4,128,1,float16,float16,31,0.02378239929676056
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,2,128,1,float16,float16,4095,0.35545599460601807
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,4,128,1,float16,float16,127,0.03049600124359131
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,4,128,1,float16,float16,63,0.02401120066642761
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,4,128,1,float16,float16,255,0.046902400255203244
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,4,128,1,float16,float16,511,0.10037599802017212
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,2,128,1,float16,float16,8191,0.6892848014831543
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,4,128,1,float16,float16,1023,0.16546080112457276
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,4,128,1,float16,float16,2047,0.2846528053283691
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,8,128,1,float16,float16,1,0.03105599880218506
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,8,128,1,float16,float16,3,0.03096800148487091
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,4,128,1,float16,float16,4095,0.5274943828582763
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,8,128,1,float16,float16,7,0.03129920065402984
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,8,128,1,float16,float16,15,0.03138239979743958
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,8,128,1,float16,float16,63,0.032359999418258664
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,2,128,1,float16,float16,16383,1.3297167778015138
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,8,128,1,float16,float16,31,0.03172479867935181
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,8,128,1,float16,float16,127,0.04676640033721924
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,8,128,1,float16,float16,255,0.07009279727935791
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,8,128,1,float16,float16,511,0.15774240493774414
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,4,128,1,float16,float16,8191,1.0156800270080566
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,8,128,1,float16,float16,1023,0.2679296016693115
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,8,128,1,float16,float16,2047,0.4643424034118652
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,1,128,1,float16,float16,1,0.01385599970817566
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,1,128,1,float16,float16,3,0.014134399592876434
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,1,128,1,float16,float16,15,0.013928000628948212
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,1,128,1,float16,float16,7,0.013788799941539764
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,1,128,1,float16,float16,31,0.014151999354362487
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,1,128,1,float16,float16,63,0.013947199285030364
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,1,128,1,float16,float16,255,0.021768000721931458
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,1,128,1,float16,float16,127,0.016497600078582763
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,1,128,1,float16,float16,1023,0.027132800221443175
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,1,128,1,float16,float16,511,0.022577600181102754
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,8,128,1,float16,float16,4095,0.8630784034729004
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,1,128,1,float16,float16,2047,0.02715199887752533
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,1,128,1,float16,float16,4095,0.02850399911403656
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,1,128,1,float16,float16,8191,0.030324798822402955
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,4,128,1,float16,float16,16383,1.9947919845581055
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,2,128,1,float16,float16,3,0.014238399267196656
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,2,128,1,float16,float16,1,0.01383039951324463
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,1,128,1,float16,float16,16383,0.0339711993932724
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,2,128,1,float16,float16,7,0.013926400244235993
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,2,128,1,float16,float16,31,0.014310400187969207
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,2,128,1,float16,float16,63,0.013896000385284425
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,2,128,1,float16,float16,127,0.016631999611854555
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,2,128,1,float16,float16,15,0.013967999815940857
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,2,128,1,float16,float16,255,0.021459199488162994
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,2,128,1,float16,float16,511,0.022644799947738648
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,2,128,1,float16,float16,1023,0.026625600457191468
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,2,128,1,float16,float16,2047,0.027219200134277345
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,4,128,1,float16,float16,1,0.013945600390434265
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,2,128,1,float16,float16,16383,0.039540800452232364
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,2,128,1,float16,float16,8191,0.030399999022483824
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,2,128,1,float16,float16,4095,0.02850399911403656
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,8,128,1,float16,float16,8191,1.6942752838134765
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,4,128,1,float16,float16,7,0.014000000059604644
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,4,128,1,float16,float16,15,0.01401440054178238
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,4,128,1,float16,float16,31,0.013980799913406372
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,4,128,1,float16,float16,63,0.014353600144386292
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,4,128,1,float16,float16,3,0.0141184002161026
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,4,128,1,float16,float16,255,0.02160159945487976
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,4,128,1,float16,float16,511,0.022806400060653688
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,4,128,1,float16,float16,127,0.017507199943065644
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,4,128,1,float16,float16,1023,0.026700800657272337
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,4,128,1,float16,float16,2047,0.027880001068115234
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,4,128,1,float16,float16,4095,0.028867200016975403
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,4,128,1,float16,float16,8191,0.03749920129776001
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,4,128,1,float16,float16,16383,0.055404800176620486
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,8,128,1,float16,float16,1,0.01467359960079193
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,8,128,1,float16,float16,7,0.014187200367450714
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,8,128,1,float16,float16,15,0.014103999733924866
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,8,128,1,float16,float16,3,0.013851200044155122
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,8,128,1,float16,float16,31,0.01408800035715103
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,8,128,1,float16,float16,63,0.014366400241851807
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,8,128,1,float16,float16,127,0.017475199699401856
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,8,128,1,float16,float16,255,0.021663999557495116
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,8,128,1,float16,float16,1023,0.027003198862075806
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,8,128,1,float16,float16,511,0.022575999796390533
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,8,128,1,float16,float16,2047,0.02802239954471588
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,1,128,1,float16,float16,1,0.013964800536632538
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,8,128,1,float16,float16,4095,0.03720000088214874
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,1,128,1,float16,float16,3,0.013990400731563568
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,1,128,1,float16,float16,7,0.013984000682830811
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,8,128,1,float16,float16,8191,0.05496799945831299
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,1,128,1,float16,float16,15,0.013971200585365296
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,1,128,1,float16,float16,63,0.014134399592876434
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,1,128,1,float16,float16,31,0.014182400703430176
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,1,128,1,float16,float16,127,0.016630400717258454
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,8,128,1,float16,float16,16383,0.08320639729499817
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,1,128,1,float16,float16,511,0.02264160066843033
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,1,128,1,float16,float16,255,0.02216159999370575
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,1,128,1,float16,float16,1023,0.027000001072883605
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,1,128,1,float16,float16,2047,0.027779200673103334
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,1,128,1,float16,float16,4095,0.028998398780822755
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,1,128,1,float16,float16,8191,0.030747199058532716
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,2,128,1,float16,float16,1,0.014071999490261078
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,8,128,1,float16,float16,16383,3.359521484375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,2,128,1,float16,float16,3,0.014075200259685516
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,1,128,1,float16,float16,16383,0.03773120045661926
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,2,128,1,float16,float16,31,0.014209599792957306
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,2,128,1,float16,float16,15,0.01390880048274994
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,2,128,1,float16,float16,7,0.013942399621009826
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,2,128,1,float16,float16,127,0.016841599345207216
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,2,128,1,float16,float16,511,0.022631999850273133
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,2,128,1,float16,float16,255,0.02212799936532974
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,2,128,1,float16,float16,2047,0.02771199941635132
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,2,128,1,float16,float16,4095,0.02900800108909607
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,2,128,1,float16,float16,1023,0.026919999718666078
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,2,128,1,float16,float16,63,0.014468799531459808
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,4,128,1,float16,float16,1,0.01393119990825653
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,2,128,1,float16,float16,8191,0.037571200728416444
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,4,128,1,float16,float16,3,0.014059199392795563
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,4,128,1,float16,float16,7,0.013980799913406372
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,4,128,1,float16,float16,15,0.014059199392795563
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,4,128,1,float16,float16,31,0.014483200013637542
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,4,128,1,float16,float16,63,0.014057600498199463
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,2,128,1,float16,float16,16383,0.05278880000114441
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,4,128,1,float16,float16,127,0.016857600212097167
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,4,128,1,float16,float16,255,0.021646399796009064
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,4,128,1,float16,float16,1023,0.027294400334358215
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,4,128,1,float16,float16,511,0.02274720072746277
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,4,128,1,float16,float16,4095,0.03710080087184906
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,8,128,1,float16,float16,1,0.014299200475215912
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,4,128,1,float16,float16,2047,0.028905600309371948
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,4,128,1,float16,float16,8191,0.054995197057724
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,8,128,1,float16,float16,3,0.014168000221252442
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,8,128,1,float16,float16,7,0.014211200177669525
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,4,128,1,float16,float16,16383,0.07872480154037476
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,8,128,1,float16,float16,31,0.014268800616264343
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,8,128,1,float16,float16,15,0.014105600118637086
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,8,128,1,float16,float16,63,0.01451839953660965
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,8,128,1,float16,float16,511,0.023491199314594268
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,8,128,1,float16,float16,255,0.021782399713993074
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,8,128,1,float16,float16,1023,0.02776640057563782
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,8,128,1,float16,float16,127,0.017088000476360322
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,8,128,1,float16,float16,2047,0.036478400230407715
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,1,128,1,float16,float16,1,0.014828799664974213
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,8,128,1,float16,float16,4095,0.05406879782676697
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,1,128,1,float16,float16,3,0.014795200526714325
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,1,128,1,float16,float16,7,0.015006400644779205
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,1,128,1,float16,float16,31,0.015011200308799743
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,1,128,1,float16,float16,15,0.01491519957780838
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,8,128,1,float16,float16,16383,0.21297600269317626
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,1,128,1,float16,float16,63,0.01510079950094223
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,1,128,1,float16,float16,127,0.017652800679206847
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,8,128,1,float16,float16,8191,0.08230239748954774
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,1,128,1,float16,float16,255,0.022622400522232057
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,1,128,1,float16,float16,511,0.029304000735282897
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,1,128,1,float16,float16,1023,0.03864479959011078
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,2,128,1,float16,float16,1,0.01520639955997467
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,1,128,1,float16,float16,2047,0.05533440113067627
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,2,128,1,float16,float16,3,0.015059199929237366
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,2,128,1,float16,float16,15,0.015195199847221374
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,1,128,1,float16,float16,4095,0.08286399841308593
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,2,128,1,float16,float16,7,0.0151296004652977
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,1,128,1,float16,float16,8191,0.13583680391311645
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,2,128,1,float16,float16,63,0.015217599272727967
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,2,128,1,float16,float16,31,0.015118399262428283
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,2,128,1,float16,float16,127,0.018027199804782866
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,2,128,1,float16,float16,255,0.02274720072746277
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,2,128,1,float16,float16,511,0.04626719951629639
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,2,128,1,float16,float16,1023,0.07189760208129883
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,1,128,1,float16,float16,16383,0.3588464021682739
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,2,128,1,float16,float16,2047,0.11134719848632812
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,2,128,1,float16,float16,4095,0.1912351965904236
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,4,128,1,float16,float16,3,0.016771200299263
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,4,128,1,float16,float16,1,0.016996799409389494
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,4,128,1,float16,float16,7,0.0172447994351387
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,4,128,1,float16,float16,15,0.017136000096797943
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,2,128,1,float16,float16,8191,0.3550623893737793
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,4,128,1,float16,float16,63,0.01727519929409027
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,4,128,1,float16,float16,127,0.021595199406147004
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,4,128,1,float16,float16,31,0.01723040044307709
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,4,128,1,float16,float16,511,0.06962720155715943
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,4,128,1,float16,float16,255,0.03001759946346283
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,2,128,1,float16,float16,16383,0.6787983894348144
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,4,128,1,float16,float16,1023,0.11450879573822022
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,4,128,1,float16,float16,2047,0.1937567949295044
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,8,128,1,float16,float16,1,0.022977599501609804
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,8,128,1,float16,float16,3,0.022884799540042876
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,8,128,1,float16,float16,7,0.02301599979400635
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,4,128,1,float16,float16,4095,0.3549168109893799
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,8,128,1,float16,float16,15,0.023027199506759643
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,8,128,1,float16,float16,31,0.023342399299144743
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,8,128,1,float16,float16,63,0.02343679964542389
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,8,128,1,float16,float16,127,0.02961280047893524
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,8,128,1,float16,float16,255,0.04694559872150421
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,8,128,1,float16,float16,511,0.09778879880905152
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,4,128,1,float16,float16,8191,0.6811567783355713
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,8,128,1,float16,float16,1023,0.1649616003036499
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,1,128,1,float16,float16,1,0.013967999815940857
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,1,128,1,float16,float16,3,0.013944000005722046
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,8,128,1,float16,float16,2047,0.28422400951385496
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,8,128,1,float16,float16,4095,0.5229087829589844
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,1,128,1,float16,float16,7,0.013977600634098053
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,1,128,1,float16,float16,15,0.014159999787807465
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,1,128,1,float16,float16,63,0.014071999490261078
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,1,128,1,float16,float16,31,0.01430719941854477
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,4,128,1,float16,float16,16383,1.323801612854004
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,1,128,1,float16,float16,127,0.016785599291324615
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,1,128,1,float16,float16,255,0.02155359983444214
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,1,128,1,float16,float16,511,0.023171199858188628
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,1,128,1,float16,float16,2047,0.027995198965072632
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,1,128,1,float16,float16,1023,0.027113598585128785
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,1,128,1,float16,float16,4095,0.02906399965286255
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,8,128,1,float16,float16,8191,1.0172831535339355
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,2,128,1,float16,float16,1,0.013996799290180207
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,1,128,1,float16,float16,8191,0.038431999087333676
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,1,128,1,float16,float16,16383,0.0529744029045105
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,2,128,1,float16,float16,3,0.013972799479961395
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,2,128,1,float16,float16,15,0.01406559944152832
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,2,128,1,float16,float16,127,0.016641600430011748
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,2,128,1,float16,float16,63,0.014169600605964661
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,2,128,1,float16,float16,31,0.014315199851989747
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,2,128,1,float16,float16,7,0.01427839994430542
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,2,128,1,float16,float16,255,0.02152799963951111
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,2,128,1,float16,float16,1023,0.0272271990776062
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,2,128,1,float16,float16,511,0.022916799783706664
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,2,128,1,float16,float16,2047,0.028611201047897338
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,2,128,1,float16,float16,4095,0.03740479946136475
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,4,128,1,float16,float16,3,0.014382399618625641
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,2,128,1,float16,float16,16383,0.0782751977443695
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,4,128,1,float16,float16,1,0.014220799505710601
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,4,128,1,float16,float16,7,0.014327999949455262
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,4,128,1,float16,float16,15,0.014286400377750396
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,2,128,1,float16,float16,8191,0.055009597539901735
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,4,128,1,float16,float16,31,0.014283199608325959
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,4,128,1,float16,float16,63,0.014051200449466705
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,4,128,1,float16,float16,255,0.0219200000166893
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,4,128,1,float16,float16,127,0.017027199268341064
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,4,128,1,float16,float16,511,0.02345760017633438
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,4,128,1,float16,float16,1023,0.02781440019607544
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,8,128,1,float16,float16,16383,1.9926959991455078
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,4,128,1,float16,float16,4095,0.05398079752922058
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,4,128,1,float16,float16,2047,0.036520001292228696
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,8,128,1,float16,float16,1,0.014248000085353851
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,8,128,1,float16,float16,31,0.014343999326229095
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,4,128,1,float16,float16,8191,0.08199040293693542
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,8,128,1,float16,float16,7,0.014451199769973755
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,8,128,1,float16,float16,63,0.014441600441932679
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,8,128,1,float16,float16,15,0.014265599846839904
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,8,128,1,float16,float16,3,0.014424000680446625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,8,128,1,float16,float16,127,0.016923199594020843
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,4,128,1,float16,float16,16383,0.21447999477386476
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,8,128,1,float16,float16,255,0.021859200298786165
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,8,128,1,float16,float16,511,0.028177601099014283
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,8,128,1,float16,float16,1023,0.03733600080013275
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,1,128,1,float16,float16,1,0.018641600012779237
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,1,128,1,float16,float16,3,0.018779200315475465
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,8,128,1,float16,float16,2047,0.053598397970199586
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,1,128,1,float16,float16,7,0.018580800294876097
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,8,128,1,float16,float16,4095,0.08240640163421631
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,8,128,1,float16,float16,8191,0.13339040279388428
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,1,128,1,float16,float16,15,0.01894560009241104
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,1,128,1,float16,float16,31,0.018952000141143798
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,1,128,1,float16,float16,63,0.018939200043678283
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,1,128,1,float16,float16,127,0.023299199342727662
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,8,128,1,float16,float16,16383,0.35168321132659913
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,1,128,1,float16,float16,255,0.03167999982833862
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,1,128,1,float16,float16,511,0.07301440238952636
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,1,128,1,float16,float16,1023,0.11772639751434326
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,1,128,1,float16,float16,2047,0.19654560089111328
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,2,128,1,float16,float16,1,0.024414399266242982
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,2,128,1,float16,float16,3,0.024817599356174468
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,2,128,1,float16,float16,7,0.024414399266242982
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,2,128,1,float16,float16,15,0.024779200553894043
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,2,128,1,float16,float16,63,0.025196799635887147
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,2,128,1,float16,float16,127,0.032014399766922
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,2,128,1,float16,float16,31,0.02496480047702789
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,2,128,1,float16,float16,255,0.048332801461219786
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,1,128,1,float16,float16,4095,0.3553567886352539
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,2,128,1,float16,float16,511,0.10242880582809448
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,1,128,1,float16,float16,8191,0.6833407878875732
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,2,128,1,float16,float16,1023,0.16751199960708618
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,4,128,1,float16,float16,1,0.03181599974632263
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,4,128,1,float16,float16,3,0.03220320045948029
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,4,128,1,float16,float16,7,0.032092800736427306
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,4,128,1,float16,float16,63,0.03336159884929657
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,4,128,1,float16,float16,31,0.03238399922847748
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,4,128,1,float16,float16,15,0.032358399033546446
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,2,128,1,float16,float16,2047,0.2898207902908325
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,2,128,1,float16,float16,4095,0.527566385269165
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,4,128,1,float16,float16,127,0.050070399045944215
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,4,128,1,float16,float16,255,0.07153599858283996
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,4,128,1,float16,float16,511,0.15846719741821289
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,8,128,1,float16,float16,1,0.05007839798927307
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,4,128,1,float16,float16,1023,0.2678096055984497
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,8,128,1,float16,float16,3,0.049644801020622256
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,8,128,1,float16,float16,7,0.05029919743537903
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,2,128,1,float16,float16,8191,1.015841579437256
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,8,128,1,float16,float16,15,0.050195199251174924
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,8,128,1,float16,float16,31,0.050836801528930664
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,4,128,1,float16,float16,2047,0.4694608211517334
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,8,128,1,float16,float16,63,0.058689600229263304
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,8,128,1,float16,float16,127,0.07929440140724182
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,8,128,1,float16,float16,255,0.12184799909591675
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,4,128,1,float16,float16,4095,0.8720527648925781
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,8,128,1,float16,float16,511,0.29598240852355956
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,1,128,1,float16,float16,1,0.02645919919013977
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,8,128,1,float16,float16,1023,0.5168191909790039
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,1,128,1,float16,float16,3,0.02630400061607361
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,1,128,1,float16,float16,7,0.02632960081100464
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,1,128,1,float16,float16,15,0.026491200923919676
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,8,128,1,float16,float16,2047,0.9144271850585938
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,1,128,1,float16,float16,63,0.026897600293159483
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,1,128,1,float16,float16,31,0.0265855997800827
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,4,128,1,float16,float16,8191,1.6901664733886719
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,1,128,1,float16,float16,127,0.03372800052165985
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,1,128,1,float16,float16,255,0.05224639773368835
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,1,128,1,float16,float16,511,0.10582560300827026
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,2,128,1,float16,float16,1,0.03384479880332947
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,1,128,1,float16,float16,1023,0.1800544023513794
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,2,128,1,float16,float16,3,0.03363839983940124
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,2,128,1,float16,float16,7,0.033816000819206236
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,2,128,1,float16,float16,15,0.03593760132789612
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,1,128,1,float16,float16,2047,0.2915071964263916
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,2,128,1,float16,float16,31,0.034267199039459226
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,2,128,1,float16,float16,63,0.03527680039405823
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,8,128,1,float16,float16,4095,1.736720085144043
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,2,128,1,float16,float16,127,0.052236801385879515
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,2,128,1,float16,float16,255,0.07322720289230347
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,2,128,1,float16,float16,511,0.16137759685516356
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,4,128,1,float16,float16,1,0.05321440100669861
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,4,128,1,float16,float16,3,0.05168160200119019
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,4,128,1,float16,float16,7,0.051648002862930295
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,4,128,1,float16,float16,15,0.05199199914932251
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,2,128,1,float16,float16,1023,0.2712176084518433
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,4,128,1,float16,float16,63,0.061406397819519044
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,4,128,1,float16,float16,31,0.053641599416732785
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,4,128,1,float16,float16,127,0.08037760257720947
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,4,128,1,float16,float16,255,0.12415039539337158
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,2,128,1,float16,float16,2047,0.4963808059692383
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,8,128,1,float16,float16,1,0.08882079720497131
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,4,128,1,float16,float16,511,0.29741439819335935
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,8,128,1,float16,float16,3,0.08874559998512269
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,8,128,1,float16,float16,15,0.08942080140113831
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,8,128,1,float16,float16,7,0.09268479943275451
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,8,128,1,float16,float16,31,0.09267200231552124
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,8,128,1,float16,float16,63,0.09939680099487305
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,8,128,1,float16,float16,127,0.13732000589370727
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,4,128,1,float16,float16,1023,0.5194784164428711
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,8,128,1,float16,float16,255,0.2295072078704834
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,1,128,1,float16,float16,1,0.03839359879493713
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,8,128,1,float16,float16,8191,3.478646469116211
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,4,128,1,float16,float16,2047,0.9298064231872558
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,1,128,1,float16,float16,3,0.03925600051879883
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,1,128,1,float16,float16,7,0.03850559890270233
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,8,128,1,float16,float16,511,0.5516992092132569
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,1,128,1,float16,float16,127,0.055446398258209226
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,1,128,1,float16,float16,63,0.04105280041694641
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,1,128,1,float16,float16,31,0.03944160044193268
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,1,128,1,float16,float16,15,0.04140479862689972
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,1,128,1,float16,float16,255,0.07978240251541138
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,8,128,1,float16,float16,1023,0.9730208396911622
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,2,128,1,float16,float16,1,0.056550401449203494
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,1,128,1,float16,float16,511,0.17174559831619263
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,2,128,1,float16,float16,3,0.0599120020866394
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,2,128,1,float16,float16,7,0.056088000535964966
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,2,128,1,float16,float16,15,0.05633760094642639
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,2,128,1,float16,float16,31,0.05777919888496399
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,2,128,1,float16,float16,63,0.06537280082702637
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,1,128,1,float16,float16,1023,0.28531200885772706
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,2,128,1,float16,float16,255,0.12841440439224244
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,2,128,1,float16,float16,127,0.09083679914474488
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,4,128,1,float16,float16,1,0.0931007981300354
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,8,128,1,float16,float16,2047,1.7537599563598634
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,4,128,1,float16,float16,7,0.09330880045890808
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,4,128,1,float16,float16,3,0.09419999718666076
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,4,128,1,float16,float16,15,0.10031520128250122
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,4,128,1,float16,float16,31,0.0971664011478424
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,2,128,1,float16,float16,511,0.30906879901885986
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,2,128,1,float16,float16,1023,0.5342512130737305
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,4,128,1,float16,float16,127,0.14241280555725097
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,4,128,1,float16,float16,63,0.10825279951095582
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,8,128,1,float16,float16,1,0.164518404006958
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,4,128,1,float16,float16,255,0.23446400165557862
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,8,128,1,float16,float16,7,0.16510239839553834
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,8,128,1,float16,float16,15,0.17393280267715455
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,8,128,1,float16,float16,3,0.1645248055458069
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,8,128,1,float16,float16,31,0.17464799880981446
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,8,128,1,float16,float16,63,0.18428159952163697
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,4,128,1,float16,float16,511,0.5888432025909424
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,4,128,1,float16,float16,1023,0.9802623748779297
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,8,128,1,float16,float16,127,0.2571552038192749
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,1,128,1,float16,float16,1,0.014006400108337402
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,1,128,1,float16,float16,3,0.014414399862289429
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,1,128,1,float16,float16,31,0.014142400026321411
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,8,128,1,float16,float16,255,0.43666877746582033
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,1,128,1,float16,float16,63,0.014585599303245544
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,1,128,1,float16,float16,7,0.013980799913406372
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,1,128,1,float16,float16,127,0.016784000396728515
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,1,128,1,float16,float16,15,0.014100800454616546
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,1,128,1,float16,float16,255,0.02199520021677017
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,1,128,1,float16,float16,511,0.023233599960803986
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,1,128,1,float16,float16,1023,0.02768000066280365
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,1,128,1,float16,float16,2047,0.028591999411582948
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,2,128,1,float16,float16,1,0.01419679969549179
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,1,128,1,float16,float16,4095,0.038134399056434634
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,2,128,1,float16,float16,3,0.014164799451828003
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,1,128,1,float16,float16,8191,0.05437920093536377
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,2,128,1,float16,float16,31,0.014176000654697419
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,2,128,1,float16,float16,7,0.014230400323867798
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,1,128,1,float16,float16,16383,0.07990080118179321
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,2,128,1,float16,float16,15,0.014473600685596466
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,2,128,1,float16,float16,127,0.016972799599170686
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,2,128,1,float16,float16,255,0.021852800250053407
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,2,128,1,float16,float16,63,0.014324800670146942
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,2,128,1,float16,float16,511,0.023528000712394713
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,2,128,1,float16,float16,1023,0.028484800457954408
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,8,128,1,float16,float16,511,1.0842047691345216
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,2,128,1,float16,float16,2047,0.03673120141029358
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,2,128,1,float16,float16,4095,0.05370079874992371
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,2,128,1,float16,float16,8191,0.08147519826889038
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,4,128,1,float16,float16,1,0.014582400023937226
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,4,128,1,float16,float16,7,0.014241600036621093
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,4,128,1,float16,float16,127,0.01706240028142929
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,4,128,1,float16,float16,63,0.0147599995136261
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,4,128,1,float16,float16,31,0.01430879980325699
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,4,128,1,float16,float16,15,0.014350399374961853
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,4,128,1,float16,float16,3,0.014215999841690063
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,2,128,1,float16,float16,16383,0.21376481056213378
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,4,128,1,float16,float16,255,0.021697600185871125
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,4,128,1,float16,float16,1023,0.03721120059490204
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,4,128,1,float16,float16,511,0.028339201211929323
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,4,128,1,float16,float16,2047,0.05477439761161804
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,8,128,1,float16,float16,1023,1.941009521484375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,8,128,1,float16,float16,1,0.014291200041770934
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,4,128,1,float16,float16,4095,0.08224639892578126
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,8,128,1,float16,float16,3,0.014620800316333771
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,4,128,1,float16,float16,8191,0.13477920293807982
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,8,128,1,float16,float16,7,0.014580799639225006
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,8,128,1,float16,float16,31,0.014486399292945863
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,4,128,1,float16,float16,16383,0.351308798789978
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,8,128,1,float16,float16,127,0.0174687996506691
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,8,128,1,float16,float16,255,0.022163200378417968
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,8,128,1,float16,float16,511,0.04544639885425568
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,8,128,1,float16,float16,63,0.014737600088119506
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,8,128,1,float16,float16,15,0.014737600088119506
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,8,128,1,float16,float16,1023,0.06942880153656006
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,8,128,1,float16,float16,2047,0.1111024022102356
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,1,128,1,float16,float16,1,0.056591999530792234
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,1,128,1,float16,float16,3,0.05713919997215271
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,1,128,1,float16,float16,7,0.056683200597763064
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,8,128,1,float16,float16,4095,0.1910928010940552
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,1,128,1,float16,float16,15,0.057062399387359616
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,8,128,1,float16,float16,8191,0.35260639190673826
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,1,128,1,float16,float16,31,0.05815039873123169
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,1,128,1,float16,float16,127,0.08618239760398864
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,1,128,1,float16,float16,63,0.06798719763755798
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,8,128,1,float16,float16,16383,0.6757696151733399
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,2,128,1,float16,float16,3,0.09389439821243287
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,2,128,1,float16,float16,1,0.0936959981918335
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,2,128,1,float16,float16,7,0.09378079771995544
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,1,128,1,float16,float16,255,0.13046079874038696
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,1,128,1,float16,float16,511,0.317740797996521
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,2,128,1,float16,float16,15,0.0956816017627716
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,2,128,1,float16,float16,127,0.14436960220336914
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,2,128,1,float16,float16,31,0.10138880014419556
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,2,128,1,float16,float16,63,0.10656800270080566
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,4,128,1,float16,float16,1,0.1656816005706787
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,2,128,1,float16,float16,255,0.23126559257507323
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,4,128,1,float16,float16,31,0.1761023998260498
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,4,128,1,float16,float16,3,0.16717599630355834
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,4,128,1,float16,float16,15,0.1716048002243042
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,4,128,1,float16,float16,7,0.1658720016479492
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,4,128,1,float16,float16,63,0.18236639499664306
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,4,128,1,float16,float16,127,0.2592096090316772
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,2,128,1,float16,float16,511,0.5704160213470459
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,8,128,1,float16,float16,1,0.3102447986602783
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,8,128,1,float16,float16,3,0.3099776029586792
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,4,128,1,float16,float16,255,0.43381919860839846
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,8,128,1,float16,float16,7,0.31146719455718996
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,8,128,1,float16,float16,15,0.3219791889190674
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,8,128,1,float16,float16,31,0.3271647930145264
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,8,128,1,float16,float16,63,0.33713440895080565
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,16,1,128,1,float16,float16,1,0.09467039704322815
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,8,128,1,float16,float16,127,0.4885231971740723
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,16,1,128,1,float16,float16,3,0.09499520063400269
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,16,1,128,1,float16,float16,7,0.09589759707450866
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,16,1,128,1,float16,float16,15,0.09690080285072326
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,16,1,128,1,float16,float16,31,0.10319679975509644
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,4,128,1,float16,float16,511,1.0863136291503905
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,16,1,128,1,float16,float16,63,0.10934239625930786
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,16,1,128,1,float16,float16,255,0.23020639419555664
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,16,1,128,1,float16,float16,127,0.14712799787521363
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,16,2,128,1,float16,float16,1,0.16726720333099365
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,8,128,1,float16,float16,255,0.8354096412658691
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,16,2,128,1,float16,float16,3,0.16800479888916015
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,16,2,128,1,float16,float16,7,0.17032959461212158
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,16,2,128,1,float16,float16,15,0.176636803150177
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,16,2,128,1,float16,float16,31,0.1789247989654541
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,16,2,128,1,float16,float16,63,0.18670400381088256
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,16,2,128,1,float16,float16,127,0.2663887977600098
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,16,4,128,1,float16,float16,1,0.310532808303833
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,16,4,128,1,float16,float16,3,0.313427209854126
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,16,2,128,1,float16,float16,255,0.4439712047576904
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,16,4,128,1,float16,float16,7,0.31919040679931643
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,16,4,128,1,float16,float16,15,0.3258127927780151
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,8,128,1,float16,float16,511,2.137112045288086
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,16,4,128,1,float16,float16,31,0.3273024082183838
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,16,4,128,1,float16,float16,63,0.33978240489959716
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,16,4,128,1,float16,float16,127,0.49990081787109375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,16,8,128,1,float16,float16,1,0.6079071998596192
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,16,8,128,1,float16,float16,7,0.6110400199890137
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,16,8,128,1,float16,float16,15,0.6205327987670899
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,16,4,128,1,float16,float16,255,0.8352512359619141
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,16,8,128,1,float16,float16,3,0.6158095836639405
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,16,8,128,1,float16,float16,31,0.6310207843780518
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,1,128,1,float16,float16,7,0.01424960047006607
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,1,128,1,float16,float16,3,0.014316800236701965
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,1,128,1,float16,float16,1,0.014467200636863709
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,1,128,1,float16,float16,31,0.014201599359512328
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,1,128,1,float16,float16,15,0.014324800670146942
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,1,128,1,float16,float16,63,0.014435200393199921
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,1,128,1,float16,float16,127,0.0171984001994133
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,1,128,1,float16,float16,511,0.024001599848270418
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,1,128,1,float16,float16,255,0.021750399470329286
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,1,128,1,float16,float16,1023,0.028488001227378844
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,16,8,128,1,float16,float16,63,0.6477536201477051
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,1,128,1,float16,float16,2047,0.03749440014362335
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,2,128,1,float16,float16,1,0.014289599657058717
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,1,128,1,float16,float16,4095,0.05437279939651489
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,1,128,1,float16,float16,8191,0.08256800174713134
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,2,128,1,float16,float16,3,0.01446239948272705
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,2,128,1,float16,float16,7,0.014259199798107147
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,2,128,1,float16,float16,15,0.014444799721240997
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,16,8,128,1,float16,float16,127,0.9498512268066406
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,2,128,1,float16,float16,63,0.014595200121402741
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,2,128,1,float16,float16,31,0.014268800616264343
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,1,128,1,float16,float16,16383,0.2168287992477417
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,2,128,1,float16,float16,255,0.0220223993062973
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,2,128,1,float16,float16,127,0.01706559956073761
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,2,128,1,float16,float16,511,0.028963199257850646
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,2,128,1,float16,float16,2047,0.05386719703674316
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,2,128,1,float16,float16,1023,0.03795520067214966
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,2,128,1,float16,float16,4095,0.08253759741783143
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,2,128,1,float16,float16,8191,0.13425439596176147
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,4,128,1,float16,float16,1,0.014703999459743499
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,4,128,1,float16,float16,3,0.01436000019311905
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,4,128,1,float16,float16,7,0.014542399346828461
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,16,8,128,1,float16,float16,255,1.6524303436279297
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,4,128,1,float16,float16,15,0.0146479994058609
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,4,128,1,float16,float16,127,0.017283199727535246
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,4,128,1,float16,float16,31,0.014603200554847717
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,4,128,1,float16,float16,63,0.01481119990348816
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,4,128,1,float16,float16,255,0.022308799624443054
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,2,128,1,float16,float16,16383,0.351476788520813
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,4,128,1,float16,float16,1023,0.07038400173187256
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,4,128,1,float16,float16,511,0.05140479803085327
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,4,128,1,float16,float16,2047,0.10979520082473755
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,8,128,1,float16,float16,1,0.01626880019903183
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,4,128,1,float16,float16,4095,0.19055360555648804
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,8,128,1,float16,float16,3,0.016383999586105348
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,8,128,1,float16,float16,7,0.017324799299240114
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,8,128,1,float16,float16,15,0.01639360040426254
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,8,128,1,float16,float16,63,0.01703200042247772
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,8,128,1,float16,float16,31,0.016551999747753142
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,4,128,1,float16,float16,8191,0.3594320058822632
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,8,128,1,float16,float16,127,0.020948800444602966
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,8,128,1,float16,float16,255,0.03096800148487091
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,8,128,1,float16,float16,511,0.06824319958686828
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,4,128,1,float16,float16,16383,0.6831488132476806
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,8,128,1,float16,float16,1023,0.11269919872283936
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,1,128,1,float16,float16,1,0.015412800014019012
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,8,128,1,float16,float16,2047,0.19324159622192383
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,1,128,1,float16,float16,15,0.01576640009880066
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,1,128,1,float16,float16,7,0.015862399339675905
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,1,128,1,float16,float16,3,0.015588800609111785
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,8,128,1,float16,float16,4095,0.35767359733581544
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,1,128,1,float16,float16,63,0.015619200468063355
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,1,128,1,float16,float16,31,0.01568319946527481
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,1,128,1,float16,float16,127,0.0187376007437706
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,1,128,1,float16,float16,255,0.023332799971103668
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,1,128,1,float16,float16,1023,0.07219840288162231
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,1,128,1,float16,float16,511,0.04797120094299316
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,1,128,1,float16,float16,2047,0.11246399879455567
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,8,128,1,float16,float16,8191,0.6801184177398681
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,1,128,1,float16,float16,4095,0.19276000261306764
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,2,128,1,float16,float16,1,0.01735360026359558
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,2,128,1,float16,float16,7,0.017744000256061553
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,2,128,1,float16,float16,3,0.017472000420093538
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,2,128,1,float16,float16,15,0.017609600722789765
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,1,128,1,float16,float16,8191,0.35507199764251707
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,2,128,1,float16,float16,31,0.017932799458503724
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,2,128,1,float16,float16,63,0.017924800515174866
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,2,128,1,float16,float16,127,0.02232320010662079
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,2,128,1,float16,float16,255,0.030323201417922975
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,2,128,1,float16,float16,511,0.07128000259399414
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,8,128,1,float16,float16,16383,1.362945556640625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,1,128,1,float16,float16,16383,0.6781455993652343
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,2,128,1,float16,float16,2047,0.19451040029525757
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,2,128,1,float16,float16,1023,0.11620320081710815
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,4,128,1,float16,float16,1,0.02332800030708313
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,4,128,1,float16,float16,3,0.02335200011730194
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,2,128,1,float16,float16,4095,0.3546704053878784
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,1,128,1,float16,float16,32767,1.331118392944336
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,4,128,1,float16,float16,7,0.02348479926586151
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,4,128,1,float16,float16,15,0.023694400489330292
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,4,128,1,float16,float16,63,0.02396959960460663
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,2,128,1,float16,float16,8191,0.6920080184936523
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,4,128,1,float16,float16,31,0.023694400489330292
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,4,128,1,float16,float16,127,0.030371201038360596
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,4,128,1,float16,float16,255,0.04639039933681488
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,4,128,1,float16,float16,511,0.10188640356063842
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,2,128,1,float16,float16,16383,1.327455997467041
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,4,128,1,float16,float16,1023,0.1646783947944641
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,4,128,1,float16,float16,2047,0.28663361072540283
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,1,128,1,float16,float16,1,0.013801600039005279
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,1,128,1,float16,float16,7,0.013841600716114044
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,4,128,1,float16,float16,4095,0.5252480030059814
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,1,128,1,float16,float16,3,0.013705599308013915
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,1,128,1,float16,float16,15,0.01385280042886734
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,1,128,1,float16,float16,63,0.013849599659442902
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,1,128,1,float16,float16,127,0.016510400176048278
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,1,128,1,float16,float16,31,0.013990400731563568
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,1,128,1,float16,float16,255,0.021380800008773803
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,1,128,1,float16,float16,511,0.022361600399017335
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,4,128,1,float16,float16,8191,1.0170384407043458
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,1,128,1,float16,float16,4095,0.02792479991912842
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,1,128,1,float16,float16,1023,0.026577600836753847
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,1,128,1,float16,float16,2047,0.02686080038547516
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,1,128,1,float16,float16,16383,0.03356960117816925
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,2,128,1,float16,float16,1,0.01395999938249588
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,1,128,1,float16,float16,8191,0.029710400104522704
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,1,128,1,float16,float16,32767,0.043033599853515625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,2,128,1,float16,float16,3,0.01404000073671341
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,2,128,1,float16,float16,7,0.013896000385284425
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,2,128,1,float16,float16,63,0.014127999544143677
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,2,128,1,float16,float16,15,0.013966399431228637
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,2,128,1,float16,float16,31,0.01390880048274994
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,2,128,1,float16,float16,255,0.021473599970340727
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,2,128,1,float16,float16,127,0.01661919951438904
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,2,128,1,float16,float16,511,0.022268800437450408
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,2,128,1,float16,float16,2047,0.027187201380729675
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,2,128,1,float16,float16,32767,2.6199504852294924
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,2,128,1,float16,float16,4095,0.028196799755096435
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,2,128,1,float16,float16,1023,0.026652801036834716
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,2,128,1,float16,float16,8191,0.030372801423072814
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,2,128,1,float16,float16,16383,0.039705601334571836
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,4,128,1,float16,float16,1,0.013902400434017182
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,4,128,1,float16,float16,3,0.014030399918556213
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,4,128,1,float16,float16,15,0.01395999938249588
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,4,128,1,float16,float16,7,0.01398559957742691
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,2,128,1,float16,float16,32767,0.05802400112152099
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,4,128,1,float16,float16,31,0.014108799397945404
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,4,128,1,float16,float16,127,0.016697600483894348
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,4,128,1,float16,float16,63,0.014347200095653535
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,4,128,1,float16,float16,16383,2.006460762023926
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,4,128,1,float16,float16,255,0.021521599590778352
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,4,128,1,float16,float16,1023,0.026867198944091796
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,4,128,1,float16,float16,2047,0.027515199780464173
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,4,128,1,float16,float16,511,0.022310400009155275
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,4,128,1,float16,float16,4095,0.028760001063346863
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,1,128,1,float16,float16,1,0.01398559957742691
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,1,128,1,float16,float16,3,0.013812799751758576
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,4,128,1,float16,float16,8191,0.038494399189949034
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,4,128,1,float16,float16,16383,0.05538560152053833
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,4,128,1,float16,float16,32767,0.08288480043411255
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,1,128,1,float16,float16,15,0.013841600716114044
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,1,128,1,float16,float16,7,0.013913600146770478
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,1,128,1,float16,float16,31,0.013911999762058258
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,1,128,1,float16,float16,63,0.014321599900722504
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,1,128,1,float16,float16,511,0.02239840030670166
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,1,128,1,float16,float16,255,0.021457600593566894
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,1,128,1,float16,float16,127,0.01661760061979294
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,1,128,1,float16,float16,1023,0.026788800954818726
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,1,128,1,float16,float16,8191,0.030403199791908263
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,1,128,1,float16,float16,2047,0.02748799920082092
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,1,128,1,float16,float16,16383,0.037099200487136844
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,1,128,1,float16,float16,4095,0.028913599252700806
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,2,128,1,float16,float16,1,0.013988800346851349
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,1,128,1,float16,float16,32767,0.05426080226898193
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,2,128,1,float16,float16,3,0.014123199880123139
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,2,128,1,float16,float16,15,0.013940800726413728
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,2,128,1,float16,float16,63,0.014163200557231904
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,2,128,1,float16,float16,31,0.014548799395561219
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,2,128,1,float16,float16,7,0.014017599821090698
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,2,128,1,float16,float16,255,0.02165600061416626
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,2,128,1,float16,float16,127,0.016446399688720702
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,2,128,1,float16,float16,511,0.02255840003490448
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,2,128,1,float16,float16,4095,0.02892000079154968
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,2,128,1,float16,float16,2047,0.028126400709152222
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,2,128,1,float16,float16,8191,0.03775359988212586
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,2,128,1,float16,float16,1023,0.026878398656845093
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,2,128,1,float16,float16,16383,0.052190399169921874
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,2,128,1,float16,float16,32767,0.07983360290527344
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,4,128,1,float16,float16,1,0.01395999938249588
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,4,128,1,float16,float16,3,0.014392000436782838
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,4,128,1,float16,float16,7,0.013961599767208099
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,4,128,1,float16,float16,15,0.014115199446678162
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,4,128,1,float16,float16,31,0.014054399728775025
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,4,128,1,float16,float16,127,0.016681599617004394
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,4,128,1,float16,float16,63,0.013910399377346038
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,4,128,1,float16,float16,255,0.021617600321769716
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,4,128,1,float16,float16,1023,0.02704800069332123
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,4,128,1,float16,float16,511,0.02276639938354492
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,4,128,1,float16,float16,4095,0.0369951993227005
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,4,128,1,float16,float16,2047,0.02810879945755005
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,4,128,1,float16,float16,8191,0.05401440262794495
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,1,128,1,float16,float16,1,0.014817599952220917
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,4,128,1,float16,float16,16383,0.07828800082206726
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,1,128,1,float16,float16,3,0.014828799664974213
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,1,128,1,float16,float16,7,0.014715200662612915
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,1,128,1,float16,float16,15,0.014844800531864166
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,4,128,1,float16,float16,32767,0.12813440561294556
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,4,128,1,float16,float16,32767,3.991900634765625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,1,128,1,float16,float16,31,0.014945599436759948
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,1,128,1,float16,float16,127,0.017638400197029114
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,1,128,1,float16,float16,255,0.022422400116920472
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,1,128,1,float16,float16,63,0.014912000298500061
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,1,128,1,float16,float16,1023,0.03761439919471741
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,1,128,1,float16,float16,511,0.029790401458740234
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,1,128,1,float16,float16,2047,0.054953598976135255
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,1,128,1,float16,float16,4095,0.08211039900779724
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,1,128,1,float16,float16,8191,0.13519200086593627
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,1,128,1,float16,float16,16383,0.35363519191741943
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,2,128,1,float16,float16,1,0.015150399506092071
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,2,128,1,float16,float16,7,0.0149167999625206
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,2,128,1,float16,float16,3,0.018296000361442567
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,2,128,1,float16,float16,15,0.015095999836921692
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,2,128,1,float16,float16,31,0.015054400265216827
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,2,128,1,float16,float16,63,0.015278400480747223
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,2,128,1,float16,float16,127,0.017841599881649017
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,2,128,1,float16,float16,255,0.022651199996471406
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,2,128,1,float16,float16,511,0.04582239985466004
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,2,128,1,float16,float16,1023,0.07137600183486939
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,1,128,1,float16,float16,32767,0.672596788406372
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,4,128,1,float16,float16,1,0.01683039963245392
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,2,128,1,float16,float16,4095,0.19123040437698363
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,2,128,1,float16,float16,2047,0.1126255989074707
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,4,128,1,float16,float16,3,0.017000000178813934
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,4,128,1,float16,float16,7,0.016990399360656737
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,2,128,1,float16,float16,8191,0.3546112060546875
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,4,128,1,float16,float16,127,0.02176000028848648
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,4,128,1,float16,float16,63,0.01738879978656769
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,4,128,1,float16,float16,15,0.016996799409389494
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,4,128,1,float16,float16,31,0.017308799922466277
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,4,128,1,float16,float16,255,0.02993600070476532
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,2,128,1,float16,float16,16383,0.6781504154205322
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,4,128,1,float16,float16,511,0.07054880261421204
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,4,128,1,float16,float16,2047,0.19287519454956054
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,4,128,1,float16,float16,1023,0.11515519618988038
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,2,128,1,float16,float16,32767,1.3225343704223633
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,1,128,1,float16,float16,1,0.014073599874973298
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,4,128,1,float16,float16,4095,0.35778400897979734
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,1,128,1,float16,float16,3,0.01387999951839447
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,1,128,1,float16,float16,15,0.01408960074186325
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,1,128,1,float16,float16,63,0.01398719996213913
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,4,128,1,float16,float16,8191,0.6806896209716797
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,1,128,1,float16,float16,127,0.016700799763202667
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,1,128,1,float16,float16,31,0.013990400731563568
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,1,128,1,float16,float16,7,0.014004799723625182
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,1,128,1,float16,float16,255,0.021495999395847322
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,1,128,1,float16,float16,1023,0.026929599046707154
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,1,128,1,float16,float16,511,0.022515200078487396
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,1,128,1,float16,float16,8191,0.03764159977436066
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,1,128,1,float16,float16,4095,0.028727999329566954
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,1,128,1,float16,float16,2047,0.027641600370407103
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,1,128,1,float16,float16,16383,0.05283839702606201
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,2,128,1,float16,float16,1,0.013988800346851349
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,2,128,1,float16,float16,7,0.013911999762058258
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,2,128,1,float16,float16,3,0.01403840035200119
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,2,128,1,float16,float16,31,0.014043200016021728
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,2,128,1,float16,float16,15,0.014148800075054169
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,2,128,1,float16,float16,63,0.014102399349212646
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,2,128,1,float16,float16,127,0.01661919951438904
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,2,128,1,float16,float16,255,0.021612800657749176
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,1,128,1,float16,float16,32767,0.07955679893493653
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,4,128,1,float16,float16,16383,1.3284288406372071
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,2,128,1,float16,float16,511,0.022784000635147093
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,2,128,1,float16,float16,1023,0.027163198590278624
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,2,128,1,float16,float16,4095,0.037083199620246886
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,2,128,1,float16,float16,2047,0.02825759947299957
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,2,128,1,float16,float16,8191,0.054441601037979126
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,2,128,1,float16,float16,32767,0.12789920568466187
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,4,128,1,float16,float16,3,0.0141184002161026
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,4,128,1,float16,float16,1,0.0141744002699852
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,2,128,1,float16,float16,16383,0.07875040173530579
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,4,128,1,float16,float16,7,0.014155200123786927
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,4,128,1,float16,float16,15,0.01417279988527298
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,4,128,1,float16,float16,63,0.014416000247001648
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,4,128,1,float16,float16,31,0.01408800035715103
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,4,128,1,float16,float16,127,0.016935999691486358
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,4,128,1,float16,float16,1023,0.027779200673103334
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,4,128,1,float16,float16,511,0.02329760044813156
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,4,128,1,float16,float16,255,0.021855999529361726
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,4,128,1,float16,float16,2047,0.036831998825073244
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,4,128,1,float16,float16,4095,0.053648000955581664
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,1,128,1,float16,float16,1,0.018643200397491455
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,1,128,1,float16,float16,3,0.01849759966135025
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,1,128,1,float16,float16,15,0.018724800646305086
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,4,128,1,float16,float16,16383,0.21301119327545165
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,4,128,1,float16,float16,8191,0.08158879876136779
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,1,128,1,float16,float16,7,0.01857440024614334
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,1,128,1,float16,float16,31,0.0187376007437706
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,4,128,1,float16,float16,32767,2.620939254760742
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,1,128,1,float16,float16,127,0.02313600033521652
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,4,128,1,float16,float16,32767,0.39696640968322755
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,1,128,1,float16,float16,63,0.01907840073108673
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,1,128,1,float16,float16,255,0.0315200001001358
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,1,128,1,float16,float16,511,0.07278079986572265
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,2,128,1,float16,float16,3,0.02457599937915802
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,2,128,1,float16,float16,1,0.02447039932012558
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,1,128,1,float16,float16,1023,0.11714400053024292
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,2,128,1,float16,float16,7,0.024633599817752837
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,2,128,1,float16,float16,15,0.024588799476623534
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,1,128,1,float16,float16,4095,0.3562096118927002
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,1,128,1,float16,float16,2047,0.19657280445098876
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,2,128,1,float16,float16,63,0.02526240050792694
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,2,128,1,float16,float16,31,0.024931199848651886
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,2,128,1,float16,float16,127,0.03131519854068756
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,2,128,1,float16,float16,255,0.048475199937820436
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,1,128,1,float16,float16,8191,0.683513593673706
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,2,128,1,float16,float16,511,0.10153440237045289
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,2,128,1,float16,float16,1023,0.16777440309524536
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,4,128,1,float16,float16,1,0.032041600346565245
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,2,128,1,float16,float16,2047,0.2880239963531494
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,4,128,1,float16,float16,3,0.03247039914131165
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,2,128,1,float16,float16,4095,0.5261519908905029
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,4,128,1,float16,float16,15,0.032476800680160525
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,4,128,1,float16,float16,31,0.032841598987579344
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,4,128,1,float16,float16,7,0.032118400931358336
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,1,128,1,float16,float16,16383,1.3283408164978028
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,4,128,1,float16,float16,127,0.04988479912281037
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,4,128,1,float16,float16,63,0.033878400921821594
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,4,128,1,float16,float16,255,0.07139840126037597
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,2,128,1,float16,float16,8191,1.0172656059265137
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,4,128,1,float16,float16,1023,0.2704063892364502
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,4,128,1,float16,float16,511,0.15931040048599243
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,1,128,1,float16,float16,1,0.026256000995635985
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,1,128,1,float16,float16,3,0.026267200708389282
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,4,128,1,float16,float16,2047,0.4697103977203369
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,1,128,1,float16,float16,7,0.026422399282455444
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,1,128,1,float16,float16,15,0.02624480128288269
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,1,128,1,float16,float16,31,0.026572799682617186
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,1,128,1,float16,float16,127,0.03314079940319061
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,1,128,1,float16,float16,63,0.027049601078033447
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,1,128,1,float16,float16,255,0.049348801374435425
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,4,128,1,float16,float16,4095,0.8678400039672851
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,1,128,1,float16,float16,511,0.10442399978637695
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,1,128,1,float16,float16,2047,0.29061920642852784
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,2,128,1,float16,float16,16383,1.9997135162353517
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,2,128,1,float16,float16,1,0.03362239897251129
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,1,128,1,float16,float16,1023,0.17109440565109252
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,2,128,1,float16,float16,15,0.033959999680519104
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,2,128,1,float16,float16,3,0.0337007999420166
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,2,128,1,float16,float16,7,0.03421759903430939
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,2,128,1,float16,float16,31,0.03446399867534637
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,2,128,1,float16,float16,63,0.03478240072727203
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,1,128,1,float16,float16,4095,0.5308671951293945
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,2,128,1,float16,float16,127,0.05074399709701538
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,2,128,1,float16,float16,255,0.07368959784507752
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,4,128,1,float16,float16,8191,1.7195920944213867
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,2,128,1,float16,float16,511,0.16195839643478394
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,2,128,1,float16,float16,1023,0.270795202255249
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,4,128,1,float16,float16,1,0.052209597826004026
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,4,128,1,float16,float16,3,0.05562719702720642
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,4,128,1,float16,float16,15,0.052072000503540036
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,4,128,1,float16,float16,7,0.051969599723815915
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,2,128,1,float16,float16,2047,0.4707024097442627
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,4,128,1,float16,float16,31,0.05315520167350769
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,4,128,1,float16,float16,63,0.06529920101165772
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,4,128,1,float16,float16,127,0.08027039766311646
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,4,128,1,float16,float16,255,0.12386720180511475
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,4,128,1,float16,float16,511,0.29890880584716795
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,1,128,1,float16,float16,1,0.03840000033378601
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,2,128,1,float16,float16,4095,0.8735695838928222
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,1,128,1,float16,float16,7,0.03838239908218384
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,1,128,1,float16,float16,3,0.0383103996515274
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,4,128,1,float16,float16,1023,0.5261199951171875
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,1,128,1,float16,float16,31,0.03877919912338257
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,1,128,1,float16,float16,63,0.0394239991903305
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,4,128,1,float16,float16,16383,3.3733470916748045
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,1,128,1,float16,float16,15,0.038494399189949034
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,1,128,1,float16,float16,127,0.055534398555755614
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,1,128,1,float16,float16,255,0.07674400210380554
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,4,128,1,float16,float16,2047,0.9419280052185058
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,2,128,1,float16,float16,3,0.05605599880218506
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,2,128,1,float16,float16,1,0.05564000010490418
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,1,128,1,float16,float16,511,0.17043360471725463
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,2,128,1,float16,float16,15,0.05633599758148193
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,2,128,1,float16,float16,7,0.05745760202407837
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,2,128,1,float16,float16,31,0.05636799931526184
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,1,128,1,float16,float16,1023,0.2888736009597778
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,2,128,1,float16,float16,63,0.06521599888801574
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,2,128,1,float16,float16,127,0.08439199924468994
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,4,128,1,float16,float16,4095,1.7266176223754883
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,2,128,1,float16,float16,255,0.12701120376586914
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,1,128,1,float16,float16,2047,0.481385612487793
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,4,128,1,float16,float16,1,0.09344959855079651
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,4,128,1,float16,float16,3,0.09304640293121338
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,4,128,1,float16,float16,31,0.09618719816207885
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,2,128,1,float16,float16,511,0.31093440055847166
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,4,128,1,float16,float16,15,0.09393600225448609
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,4,128,1,float16,float16,7,0.09530720114707947
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,4,128,1,float16,float16,63,0.1037071943283081
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,4,128,1,float16,float16,127,0.14190399646759033
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,2,128,1,float16,float16,1023,0.5298719882965088
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,4,128,1,float16,float16,255,0.23326399326324462
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,1,128,1,float16,float16,1,0.014051200449466705
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,1,128,1,float16,float16,3,0.013982400298118591
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,1,128,1,float16,float16,15,0.013864000141620637
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,1,128,1,float16,float16,7,0.014188799262046813
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,2,128,1,float16,float16,2047,0.9398624420166015
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,1,128,1,float16,float16,31,0.013963200151920319
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,1,128,1,float16,float16,63,0.014120000600814819
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,4,128,1,float16,float16,511,0.5585616111755372
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,1,128,1,float16,float16,127,0.016791999340057373
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,1,128,1,float16,float16,1023,0.027241599559783936
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,1,128,1,float16,float16,511,0.022707200050354003
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,1,128,1,float16,float16,255,0.02181600034236908
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,1,128,1,float16,float16,2047,0.028462401032447814
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,1,128,1,float16,float16,4095,0.03750239908695221
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,4,128,1,float16,float16,1023,0.9913007736206054
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,1,128,1,float16,float16,8191,0.054099202156066895
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,2,128,1,float16,float16,1,0.014084799587726593
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,2,128,1,float16,float16,3,0.014395199716091156
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,1,128,1,float16,float16,16383,0.07911840081214905
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,2,128,1,float16,float16,15,0.014185599982738495
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,2,128,1,float16,float16,63,0.01462559998035431
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,2,128,1,float16,float16,7,0.014235199987888336
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,2,128,1,float16,float16,31,0.01427839994430542
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,2,128,1,float16,float16,127,0.016910399496555328
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,1,128,1,float16,float16,32767,0.1304751992225647
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,2,128,1,float16,float16,255,0.0217616006731987
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,2,128,1,float16,float16,511,0.023830400407314302
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,2,128,1,float16,float16,1023,0.02792159914970398
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,2,128,1,float16,float16,2047,0.036524799466133115
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,4,128,1,float16,float16,2047,1.7410383224487305
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,4,128,1,float16,float16,1,0.014396800100803376
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,2,128,1,float16,float16,4095,0.0538752019405365
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,2,128,1,float16,float16,8191,0.08247519731521606
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,4,128,1,float16,float16,3,0.01430719941854477
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,2,128,1,float16,float16,16383,0.2123568058013916
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,4,128,1,float16,float16,15,0.014481599628925323
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,4,128,1,float16,float16,31,0.014292800426483154
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,4,128,1,float16,float16,7,0.014839999377727509
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,4,128,1,float16,float16,127,0.017080000042915343
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,4,128,1,float16,float16,63,0.01449279934167862
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,4,128,1,float16,float16,255,0.021942399442195892
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,4,128,1,float16,float16,1023,0.03697920143604279
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,4,128,1,float16,float16,511,0.028332799673080444
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,2,128,1,float16,float16,32767,0.3980992078781128
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,4,128,1,float16,float16,2047,0.05553600192070007
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,4,128,1,float16,float16,4095,0.08080319762229919
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,8,1,128,1,float16,float16,1,0.05661119818687439
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,4,128,1,float16,float16,8191,0.134932804107666
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,8,1,128,1,float16,float16,3,0.05655999779701233
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,8,1,128,1,float16,float16,7,0.05763840079307556
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,4,128,1,float16,float16,16383,0.3508080005645752
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,8,1,128,1,float16,float16,15,0.05681440234184265
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,8,1,128,1,float16,float16,63,0.06626399755477905
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,8,1,128,1,float16,float16,31,0.05730239748954773
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,8,1,128,1,float16,float16,127,0.08526560068130493
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,4,128,1,float16,float16,32767,0.6727136135101318
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,8,2,128,1,float16,float16,3,0.09349759817123413
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,8,2,128,1,float16,float16,1,0.09321759939193726
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,8,1,128,1,float16,float16,511,0.31621599197387695
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,8,2,128,1,float16,float16,7,0.09395679831504822
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,8,1,128,1,float16,float16,255,0.12956000566482545
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,8,2,128,1,float16,float16,15,0.09390400052070617
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,8,2,128,1,float16,float16,31,0.09829919934272766
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,8,1,128,1,float16,float16,1023,0.5349999904632569
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,8,2,128,1,float16,float16,63,0.10661599636077881
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,8,2,128,1,float16,float16,255,0.22951838970184327
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,8,2,128,1,float16,float16,127,0.14326720237731932
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,8,4,128,1,float16,float16,7,0.16575520038604735
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,8,4,128,1,float16,float16,1,0.16590240001678466
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,8,4,128,1,float16,float16,31,0.17538559436798096
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,8,2,128,1,float16,float16,511,0.5735007762908936
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,8,4,128,1,float16,float16,15,0.1688256025314331
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,8,4,128,1,float16,float16,3,0.16585439443588257
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,8,1,128,1,float16,float16,1,0.09396799802780151
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,8,4,128,1,float16,float16,63,0.1833888053894043
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,8,2,128,1,float16,float16,1023,0.9871647834777832
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,8,4,128,1,float16,float16,127,0.2613248109817505
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,8,4,128,1,float16,float16,255,0.43224320411682127
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,8,1,128,1,float16,float16,3,0.09397280216217041
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,8,1,128,1,float16,float16,7,0.09403039813041687
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,8,1,128,1,float16,float16,15,0.0949343979358673
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,8,1,128,1,float16,float16,31,0.10721440315246582
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,8,1,128,1,float16,float16,63,0.11282720565795898
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,8,1,128,1,float16,float16,127,0.14344799518585205
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,8,1,128,1,float16,float16,255,0.22915999889373778
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,8,2,128,1,float16,float16,1,0.16527520418167113
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,8,2,128,1,float16,float16,3,0.16609760522842407
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,8,2,128,1,float16,float16,7,0.1663424015045166
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,8,2,128,1,float16,float16,31,0.17627040147781373
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,8,4,128,1,float16,float16,511,1.0848575592041017
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,8,2,128,1,float16,float16,15,0.1723871946334839
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,8,2,128,1,float16,float16,63,0.18467999696731568
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,8,1,128,1,float16,float16,511,0.5909215927124023
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,8,2,128,1,float16,float16,127,0.25947999954223633
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,8,4,128,1,float16,float16,1023,1.9376176834106444
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,8,4,128,1,float16,float16,1,0.31184000968933107
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,8,4,128,1,float16,float16,3,0.3117743968963623
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,8,2,128,1,float16,float16,255,0.4389296054840088
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,8,4,128,1,float16,float16,7,0.3180032014846802
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,8,4,128,1,float16,float16,63,0.3375200033187866
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,8,4,128,1,float16,float16,31,0.3271888017654419
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,8,4,128,1,float16,float16,15,0.33371360301971437
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,8,2,128,1,float16,float16,511,1.1025872230529785
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,1,128,1,float16,float16,1,0.014153599739074707
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,1,128,1,float16,float16,7,0.014430400729179383
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,8,4,128,1,float16,float16,127,0.4957136154174805
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,1,128,1,float16,float16,3,0.01419840008020401
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,1,128,1,float16,float16,31,0.01419840008020401
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,1,128,1,float16,float16,15,0.014235199987888336
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,8,4,128,1,float16,float16,255,0.8360063552856445
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,1,128,1,float16,float16,63,0.014574399590492249
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,1,128,1,float16,float16,255,0.0217071995139122
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,1,128,1,float16,float16,511,0.02369920015335083
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,1,128,1,float16,float16,127,0.01703519970178604
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,1,128,1,float16,float16,1023,0.027764800190925597
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,1,128,1,float16,float16,2047,0.038176000118255615
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,1,128,1,float16,float16,8191,0.08212000131607056
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,2,128,1,float16,float16,1,0.014232000708580017
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,1,128,1,float16,float16,4095,0.05517920255661011
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,2,128,1,float16,float16,15,0.01430879980325699
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,1,128,1,float16,float16,16383,0.21284799575805663
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,2,128,1,float16,float16,7,0.014343999326229095
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,2,128,1,float16,float16,3,0.014865599572658539
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,2,128,1,float16,float16,31,0.01462240070104599
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,2,128,1,float16,float16,255,0.021996800601482392
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,2,128,1,float16,float16,63,0.014417600631713868
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,2,128,1,float16,float16,127,0.017071999609470367
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,2,128,1,float16,float16,511,0.028808000683784484
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,2,128,1,float16,float16,1023,0.03805600106716156
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,2,128,1,float16,float16,2047,0.053495997190475465
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,1,128,1,float16,float16,32767,0.39781439304351807
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,2,128,1,float16,float16,8191,0.13445600271224975
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,4,128,1,float16,float16,1,0.014542399346828461
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,8,4,128,1,float16,float16,511,2.1446352005004883
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,2,128,1,float16,float16,4095,0.08158879876136779
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,4,128,1,float16,float16,3,0.014414399862289429
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,4,128,1,float16,float16,7,0.014420799911022186
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,4,128,1,float16,float16,15,0.014646400511264802
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,4,128,1,float16,float16,127,0.01722240000963211
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,4,128,1,float16,float16,255,0.022097599506378175
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,4,128,1,float16,float16,511,0.04525119960308075
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,4,128,1,float16,float16,63,0.014603200554847717
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,4,128,1,float16,float16,31,0.014854399859905243
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,2,128,1,float16,float16,16383,0.356059193611145
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,4,128,1,float16,float16,1023,0.07031679749488831
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,4,128,1,float16,float16,2047,0.1104848027229309
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,2,128,1,float16,float16,32767,0.6797135829925537
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,1,0.01555040031671524
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,4,128,1,float16,float16,4095,0.19058560132980346
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,3,0.015697599947452547
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,4,128,1,float16,float16,8191,0.35399041175842283
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,15,0.015625600516796113
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,7,0.015932799875736238
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,63,0.015744000673294067
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,31,0.015915200114250183
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,255,0.023228800296783446
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,127,0.018459199368953703
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,1023,0.07330080270767211
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,511,0.0488400012254715
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,2047,0.11390880346298218
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,4,128,1,float16,float16,16383,0.6877359867095947
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,4095,0.19350240230560303
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,8191,0.35855998992919924
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,3,0.017475199699401856
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,1,0.017686399817466735
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,7,0.017500799894332886
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,15,0.01762080043554306
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,31,0.017956799268722533
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,127,0.022169600427150726
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,63,0.017931200563907623
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,255,0.030692800879478455
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,4,128,1,float16,float16,32767,1.3349231719970702
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,511,0.07135519981384278
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,16383,0.6856112003326416
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,1023,0.11446239948272705
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,32767,1.3366127967834474
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,2047,0.19737919569015502
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,4095,0.3553407907485962
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,8191,0.6816207885742187
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,1,0.014035199582576752
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,3,0.01383039951324463
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,15,0.013948799669742584
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,7,0.013790400326251983
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,31,0.013982400298118591
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,63,0.014209599792957306
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,127,0.016550399363040924
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,255,0.021568000316619873
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,511,0.023651200532913207
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,1023,0.02763360142707825
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,2047,0.027955201268196107
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,4095,0.028262400627136232
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,16383,1.330239963531494
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,8191,0.03059839904308319
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,16383,0.035596799850463864
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,1,0.01387840062379837
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,65535,2.6269535064697265
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,65535,0.06257920265197754
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,32767,0.04504640102386474
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,3,0.014017599821090698
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,7,0.013903999328613281
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,127,0.016465599834918975
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,63,0.013998399674892425
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,31,0.014134399592876434
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,255,0.021508799493312837
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,511,0.023228800296783446
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,1023,0.027632001042366027
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,15,0.014209599792957306
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,2047,0.028003200888633728
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,8191,0.031035199761390686
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,4095,0.028670400381088257
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,16383,0.04036000072956085
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,1,0.013814400136470794
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,3,0.013819199800491334
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,65535,0.08500319719314575
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,32767,0.05783039927482605
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,15,0.01387999951839447
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,32767,2.6173967361450194
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,31,0.014107200503349304
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,7,0.013891200721263885
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,63,0.014363199472427368
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,255,0.021425600349903106
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,127,0.016739200055599212
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,1023,0.02805599868297577
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,4095,0.028433600068092348
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,8191,0.03144800066947937
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,2047,0.028118398785591126
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,511,0.02330079972743988
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,1,0.013857600092887879
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,16383,0.04047839939594269
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,32767,0.05803840160369873
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,65535,0.08408640027046203
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,3,0.013952000439167023
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,15,0.013822400569915771
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,7,0.013911999762058258
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,31,0.014215999841690063
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,63,0.014097599685192109
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,255,0.021515199542045595
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,127,0.01674239933490753
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,511,0.023863999545574187
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,2047,0.02852480113506317
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,4095,0.029135999083518983
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,8191,0.03987999856472015
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,1023,0.027910399436950683
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,16383,0.053793597221374514
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,32767,0.08119999766349792
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,1,0.014795200526714325
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,3,0.01478559970855713
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,7,0.014740799367427827
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,15,0.014859199523925781
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,63,0.014947199821472168
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,31,0.014852799475193024
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,127,0.01754080057144165
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,255,0.022375999391078948
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,65535,0.13559039831161498
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,511,0.029956799745559693
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,1023,0.03933599889278412
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,2047,0.055358397960662845
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,4095,0.08384479880332947
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,1,0.01496960073709488
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,7,0.015049600601196289
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,3,0.015108799934387207
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,8191,0.1385472059249878
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,15,0.015006400644779205
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,31,0.015105600655078887
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,16383,0.35973920822143557
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,127,0.017859199643135072
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,255,0.022654399275779724
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,63,0.015193599462509155
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,32767,0.676367998123169
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,511,0.047844800353050235
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,1023,0.07174239754676819
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,2047,0.11281759738922119
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,65535,5.22630729675293
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,4095,0.19335199594497682
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,8191,0.3575439929962158
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,65535,1.325648021697998
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,1,0.013891200721263885
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,3,0.013935999572277069
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,7,0.014123199880123139
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,31,0.014144000411033631
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,63,0.014126400649547576
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,127,0.01685599982738495
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,16383,0.6827023983001709
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,255,0.02156960070133209
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,15,0.013924799859523773
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,4095,0.029262399673461913
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,511,0.023824000358581544
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,1023,0.028004801273345946
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,8191,0.03984479904174805
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,2047,0.02872479856014252
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,1,0.01432960033416748
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,16383,0.058152002096176145
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,65535,0.1379680037498474
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,32767,1.324135971069336
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,32767,0.08437600135803222
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,7,0.014134399592876434
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,3,0.014343999326229095
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,15,0.01406719982624054
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,63,0.014079999923706055
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,127,0.01696320027112961
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,31,0.014076800644397735
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,255,0.021686400473117828
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,511,0.024004800617694853
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,1023,0.02849920094013214
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,2047,0.029032000899314882
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,4095,0.0376336008310318
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,16383,0.0812928020954132
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,8191,0.057043200731277464
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,32767,0.13114240169525146
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,1,128,1,float16,float16,1,0.018486399948596955
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,1,128,1,float16,float16,3,0.01847359985113144
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,1,128,1,float16,float16,63,0.019036799669265747
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,1,128,1,float16,float16,15,0.018622399866580965
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,1,128,1,float16,float16,7,0.018681600689888
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,1,128,1,float16,float16,31,0.01867839992046356
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,65535,0.2359935998916626
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,1,128,1,float16,float16,127,0.023240000009536743
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,1,128,1,float16,float16,255,0.03115360140800476
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,1,128,1,float16,float16,511,0.07345439791679383
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,65535,2.6222911834716798
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,1,128,1,float16,float16,1023,0.11733759641647339
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,1,128,1,float16,float16,2047,0.19714399576187133
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,1,128,1,float16,float16,4095,0.35535359382629395
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,2,128,1,float16,float16,1,0.02430880069732666
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,2,128,1,float16,float16,3,0.024596799910068513
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,2,128,1,float16,float16,7,0.024505600333213806
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,2,128,1,float16,float16,15,0.02459840029478073
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,2,128,1,float16,float16,31,0.02483679950237274
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,1,128,1,float16,float16,8191,0.6826848030090332
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,2,128,1,float16,float16,63,0.024979199469089507
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,2,128,1,float16,float16,127,0.031040000915527343
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,2,128,1,float16,float16,255,0.04839999973773956
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,2,128,1,float16,float16,511,0.10234880447387695
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,2,128,1,float16,float16,1023,0.16785759925842286
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,2,128,1,float16,float16,2047,0.2863440036773682
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,2,128,1,float16,float16,4095,0.5273312091827392
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,4,1,128,1,float16,float16,1,0.02614240050315857
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,1,128,1,float16,float16,16383,1.3292655944824219
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,4,1,128,1,float16,float16,3,0.02619520127773285
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,4,1,128,1,float16,float16,7,0.02617120146751404
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,4,1,128,1,float16,float16,31,0.026523199677467347
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,4,1,128,1,float16,float16,63,0.02693760097026825
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,4,1,128,1,float16,float16,15,0.026495999097824095
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,4,1,128,1,float16,float16,127,0.0332399994134903
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,4,1,128,1,float16,float16,255,0.0488400012254715
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,2,128,1,float16,float16,8191,1.0159104347229004
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,4,1,128,1,float16,float16,511,0.10582560300827026
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,4,1,128,1,float16,float16,2047,0.2901024103164673
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,4,1,128,1,float16,float16,1023,0.1710368037223816
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,1,128,1,float16,float16,32767,2.621471977233887
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,4,2,128,1,float16,float16,1,0.03368160128593445
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,4,2,128,1,float16,float16,3,0.03374719917774201
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,4,1,128,1,float16,float16,4095,0.5303664207458496
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,4,2,128,1,float16,float16,15,0.033976000547409055
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,4,2,128,1,float16,float16,7,0.03385440111160278
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,4,2,128,1,float16,float16,31,0.03427520096302032
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,4,2,128,1,float16,float16,63,0.03503200113773346
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,4,2,128,1,float16,float16,127,0.049270400404930116
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,2,128,1,float16,float16,16383,2.0011199951171874
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,4,2,128,1,float16,float16,255,0.07310400009155274
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,4,2,128,1,float16,float16,511,0.16171679496765137
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,4,1,128,1,float16,float16,1,0.03823359906673431
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,4,2,128,1,float16,float16,1023,0.27211039066314696
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,4,1,128,1,float16,float16,3,0.03813279867172241
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,4,1,128,1,float16,float16,7,0.038145598769187924
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,4,2,128,1,float16,float16,2047,0.4695119857788086
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,4,1,128,1,float16,float16,15,0.03840799927711487
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,4,1,128,1,float16,float16,31,0.038700801134109494
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,4,1,128,1,float16,float16,8191,1.0238911628723144
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,4,1,128,1,float16,float16,63,0.039878401160240176
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,4,1,128,1,float16,float16,255,0.07630079984664917
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,4,1,128,1,float16,float16,127,0.05375360250473023
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,4,2,128,1,float16,float16,4095,0.8859567642211914
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,4,1,128,1,float16,float16,511,0.17108319997787474
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,4,1,128,1,float16,float16,1023,0.2802687883377075
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,4,1,128,1,float16,float16,2047,0.47768640518188477
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,4,2,128,1,float16,float16,1,0.05652639865875244
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,4,2,128,1,float16,float16,3,0.056225597858428955
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,4,2,128,1,float16,float16,7,0.05596640110015869
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,4,2,128,1,float16,float16,15,0.05624480247497558
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,4,2,128,1,float16,float16,31,0.05748479962348938
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,4,2,128,1,float16,float16,63,0.06532160043716431
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,4,2,128,1,float16,float16,127,0.08444640040397644
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,2,128,1,float16,float16,32767,3.9351966857910154
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,4,1,128,1,float16,float16,4095,0.8807744026184082
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,4,2,128,1,float16,float16,255,0.1287824034690857
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,4,2,128,1,float16,float16,8191,1.7134992599487304
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,3,0.013996799290180207
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,1,0.014035199582576752
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,15,0.013910399377346038
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,7,0.014727999269962311
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,4,2,128,1,float16,float16,511,0.310697603225708
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,4,2,128,1,float16,float16,1023,0.5289584159851074
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,31,0.014046399295330048
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,63,0.015033599734306336
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,511,0.023929600417613984
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,2047,0.02913439869880676
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,127,0.01687840074300766
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,255,0.021779200434684752
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,1023,0.03009760081768036
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,4095,0.03766719996929169
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,4,2,128,1,float16,float16,2047,0.9268976211547851
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,8191,0.05635039806365967
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,1,0.014193600416183472
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,15,0.014060799777507783
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,16383,0.08532320261001587
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,7,0.014204800128936768
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,3,0.014159999787807465
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,65535,0.24569759368896485
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,31,0.014105600118637086
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,32767,0.13540480136871338
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,127,0.016896000504493712
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,63,0.014452800154685974
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,1023,0.02879199981689453
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,255,0.021588799357414246
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,511,0.02502560019493103
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,4095,0.054737597703933716
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,2047,0.037755200266838075
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,8191,0.08475679755210877
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,4,1,128,1,float16,float16,1,0.05618879795074463
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,16383,0.21414880752563475
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,4,1,128,1,float16,float16,3,0.05631999969482422
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,4,2,128,1,float16,float16,4095,1.749742317199707
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,4,1,128,1,float16,float16,15,0.056595200300216676
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,4,1,128,1,float16,float16,7,0.05651519894599914
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,4,1,128,1,float16,float16,63,0.06447839736938477
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,4,1,128,1,float16,float16,31,0.0568992018699646
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,32767,0.3989504098892212
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,4,1,128,1,float16,float16,127,0.08604800105094909
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,4,1,128,1,float16,float16,255,0.12667360305786132
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,4,2,128,1,float16,float16,1,0.0934719979763031
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,65535,0.779040002822876
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,4,2,128,1,float16,float16,3,0.09457759857177735
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,4,2,128,1,float16,float16,7,0.09365280270576477
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,4,2,128,1,float16,float16,15,0.093859201669693
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,4,1,128,1,float16,float16,511,0.31775999069213867
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,4,1,128,1,float16,float16,1023,0.5351215839385987
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,4,2,128,1,float16,float16,31,0.09859200119972229
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,4,2,128,1,float16,float16,63,0.10709760189056397
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,4,2,128,1,float16,float16,127,0.1417232036590576
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,4,1,128,1,float16,float16,1,0.09533920288085937
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,4,2,128,1,float16,float16,255,0.22954881191253662
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,4,1,128,1,float16,float16,3,0.09358239769935608
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,4,1,128,1,float16,float16,2047,0.9308544158935547
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,4,1,128,1,float16,float16,7,0.09560800194740296
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,4,1,128,1,float16,float16,31,0.09717119932174682
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,4,1,128,1,float16,float16,15,0.09402400255203247
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,4,1,128,1,float16,float16,63,0.1060479998588562
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,4,2,128,1,float16,float16,511,0.5678207874298096
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,4,1,128,1,float16,float16,127,0.14470399618148805
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,4,1,128,1,float16,float16,255,0.22559199333190919
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,4,2,128,1,float16,float16,1,0.1662608027458191
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,4,2,128,1,float16,float16,1023,0.9924816131591797
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,4,2,128,1,float16,float16,3,0.16587519645690918
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,4,2,128,1,float16,float16,7,0.1660640001296997
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,4,1,128,1,float16,float16,511,0.5854144096374512
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,4,2,128,1,float16,float16,15,0.16995840072631835
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,4,2,128,1,float16,float16,31,0.1795632004737854
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,4,2,128,1,float16,float16,63,0.18600319623947142
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,4,2,128,1,float16,float16,127,0.2578191995620728
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,1,0.014158399403095245
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,4,2,128,1,float16,float16,2047,1.7748672485351562
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,3,0.01404000073671341
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,4,2,128,1,float16,float16,255,0.4294095993041992
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,4,1,128,1,float16,float16,1023,1.0076944351196289
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,7,0.013996799290180207
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,31,0.014291200041770934
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,255,0.02176959961652756
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,63,0.014403200149536133
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,511,0.024771200120449068
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,15,0.014468799531459808
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,1023,0.02911520004272461
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,127,0.016756799817085267
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,4095,0.054953598976135255
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,2047,0.03747360110282898
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,4,2,128,1,float16,float16,511,1.1064463615417481
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,1,0.014267200231552124
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,8191,0.08659840226173401
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,7,0.014273600280284881
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,16383,0.2197551965713501
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,15,0.014452800154685974
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,31,0.01430879980325699
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,3,0.014235199987888336
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,32767,0.4026383876800537
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,255,0.021804800629615782
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,63,0.01438560038805008
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,127,0.017155200242996216
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,511,0.02938719987869263
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,2047,0.05511999726295471
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,1023,0.03844000101089477
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,65535,0.7652751922607421
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,4095,0.08279039859771728
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,8191,0.1394047975540161
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,4,2,128,1,float16,float16,1023,1.9755424499511718
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,1,0.01555359959602356
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,3,0.015761600434780122
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,15,0.01563519984483719
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,16383,0.3548111915588379
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,31,0.01595039963722229
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,7,0.01624480038881302
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,63,0.015675200521945952
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,127,0.018436799943447112
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,255,0.023499199748039247
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,32767,0.6750207901000976
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,511,0.04968000054359436
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,1023,0.07517439723014832
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,4095,0.19381760358810424
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,2047,0.11469440460205078
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,1,0.013672000169754029
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,8191,0.3600559949874878
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,3,0.013734400272369385
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,7,0.014153599739074707
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,16383,0.6861567974090577
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,65535,1.3261280059814453
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,15,0.01372320055961609
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,31,0.014217600226402283
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,255,0.021352000534534454
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,511,0.023771199584007262
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,63,0.013785600662231445
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,127,0.016448000073432924
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,1023,0.027647998929023743
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,2047,0.028041601181030273
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,4095,0.02808319926261902
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,8191,0.030859199166297913
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,16383,0.03601920008659363
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,1,0.01377439945936203
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,32767,0.04555999934673309
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,3,0.013910399377346038
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,32767,1.3410335540771485
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,65535,0.06277279853820801
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,7,0.014140799641609192
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,131071,0.09057760238647461
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,15,0.01388159990310669
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,127,0.016673600673675536
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,63,0.014079999923706055
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,511,0.023737600445747374
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,1023,0.02818720042705536
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,255,0.02154400050640106
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,31,0.014023999869823455
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,2047,0.02831999957561493
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,4095,0.028543999791145323
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,8191,0.031737598776817325
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,16383,0.04108799993991852
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,65535,0.08526880145072938
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,1,0.014947199821472168
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,131071,0.13921120166778564
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,32767,0.05950719714164734
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,15,0.014979200065135955
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,3,0.014777599275112152
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,7,0.014672000706195832
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,127,0.017543999850749968
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,31,0.014870400726795196
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,63,0.014921599626541137
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,511,0.03041279911994934
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,255,0.022468799352645875
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,1023,0.039705601334571836
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,2047,0.05572320222854614
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,8191,0.13856799602508546
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,65535,2.6721696853637695
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,4095,0.08413280248641967
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,16383,0.3597248077392578
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,3,0.013870400190353394
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,1,0.01422400027513504
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,15,0.014056000113487243
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,7,0.01390880048274994
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,31,0.014305600523948669
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,63,0.01414719969034195
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,127,0.016705599427223206
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,32767,0.682476806640625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,255,0.0217071995139122
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,511,0.024163199961185454
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,1023,0.028483200073242187
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,2047,0.028780800104141236
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,4095,0.02928000092506409
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,16383,0.05878080129623413
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,8191,0.040321600437164304
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,1,0.018488000333309173
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,32767,0.08626559972763062
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,3,0.018615999817848207
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,65535,1.3305439949035645
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,65535,0.14023679494857788
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,7,0.01873600035905838
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,131071,0.24401440620422363
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,15,0.01865759938955307
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,63,0.019247999787330626
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,31,0.01865919977426529
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,255,0.03133440017700195
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,127,0.023401600122451783
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,511,0.0731872022151947
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,1023,0.11704959869384765
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,2047,0.19710079431533814
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,4095,0.3559423923492432
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,131071,5.218566513061523
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,8191,0.6852159976959229
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,131071,2.6221263885498045
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,2,1,128,1,float16,float16,1,0.026134398579597474
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,2,1,128,1,float16,float16,3,0.026348799467086792
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,2,1,128,1,float16,float16,7,0.026153600215911864
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,2,1,128,1,float16,float16,15,0.026358398795127868
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,2,1,128,1,float16,float16,31,0.02651520073413849
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,2,1,128,1,float16,float16,63,0.026836800575256347
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,2,1,128,1,float16,float16,127,0.03287360072135925
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,2,1,128,1,float16,float16,255,0.050256001949310306
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,16383,1.3318415641784669
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,2,1,128,1,float16,float16,511,0.1044160008430481
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,2,1,128,1,float16,float16,1023,0.1708400011062622
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,2,1,128,1,float16,float16,2047,0.291265606880188
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,2,1,128,1,float16,float16,1,0.038008001446723935
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,2,1,128,1,float16,float16,3,0.03812159895896912
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,2,1,128,1,float16,float16,4095,0.5318431854248047
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,2,1,128,1,float16,float16,7,0.03824479877948761
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,2,1,128,1,float16,float16,15,0.03850080072879791
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,2,1,128,1,float16,float16,31,0.03884960114955902
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,2,1,128,1,float16,float16,63,0.03941279947757721
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,2,1,128,1,float16,float16,8191,1.0227855682373046
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,2,1,128,1,float16,float16,127,0.054897600412368776
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,2,1,128,1,float16,float16,255,0.07747359871864319
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,2,1,128,1,float16,float16,511,0.17100319862365723
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,2,1,128,1,float16,float16,1023,0.28114559650421145
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,32767,2.621371269226074
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,2,1,128,1,float16,float16,2047,0.48059520721435545
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,1,0.014108799397945404
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,3,0.013798399269580841
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,15,0.014139199256896972
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,7,0.014035199582576752
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,31,0.014139199256896972
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,2,1,128,1,float16,float16,16383,1.9896944046020508
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,63,0.014286400377750396
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,127,0.016697600483894348
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,255,0.02154400050640106
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,1023,0.028638398647308348
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,2047,0.029108801484107973
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,511,0.024225600063800812
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,4095,0.03789600133895874
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,2,1,128,1,float16,float16,4095,0.8836624145507812
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,8191,0.058081597089767456
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,16383,0.0868128001689911
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,32767,0.13674399852752686
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,2,1,128,1,float16,float16,1,0.056148797273635864
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,65535,0.2390415906906128
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,2,1,128,1,float16,float16,3,0.05621439814567566
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,2,1,128,1,float16,float16,7,0.05630559921264648
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,2,1,128,1,float16,float16,15,0.05673120021820068
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,131071,0.45424637794494627
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,2,1,128,1,float16,float16,31,0.05700640082359314
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,2,1,128,1,float16,float16,63,0.06529600024223328
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,2,1,128,1,float16,float16,127,0.08480160236358643
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,2,1,128,1,float16,float16,255,0.12666239738464355
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,2,1,128,1,float16,float16,8191,1.7146799087524414
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,2,1,128,1,float16,float16,511,0.31748480796813966
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,2,1,128,1,float16,float16,1,0.09375200271606446
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,2,1,128,1,float16,float16,3,0.09667840003967285
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,65535,5.213800048828125
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,2,1,128,1,float16,float16,15,0.09416000247001648
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,2,1,128,1,float16,float16,7,0.09408479928970337
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,2,1,128,1,float16,float16,1023,0.5368383884429931
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,2,1,128,1,float16,float16,31,0.09936320185661315
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,2,1,128,1,float16,float16,127,0.14209120273590087
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,2,1,128,1,float16,float16,2047,0.936580753326416
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,2,1,128,1,float16,float16,63,0.10588480234146118
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,2,1,128,1,float16,float16,255,0.22611520290374756
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,3,0.014236800372600555
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,1,0.014206400513648987
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,7,0.014209599792957306
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,15,0.014337599277496338
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,2,1,128,1,float16,float16,511,0.5908527851104737
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,31,0.014443199336528777
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,63,0.014368000626564025
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,127,0.017028799653053282
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,511,0.025224000215530396
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,255,0.021750399470329286
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,1023,0.029388800263404846
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,2,1,128,1,float16,float16,4095,1.7421520233154297
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,2,1,128,1,float16,float16,1023,1.0242320060729981
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,2047,0.03850879967212677
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,4095,0.056492799520492555
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,16383,0.22018721103668212
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,8191,0.085835200548172
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,32767,0.40845279693603515
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,2,1,128,1,float16,float16,2047,1.7936800003051758
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,65535,0.7857759952545166
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,131071,1.5094863891601562
