framework,version,device,op_name,kernel_source,batch_size,isl,num_heads,num_key_value_heads,head_dim,beam_width,attn_dtype,kv_cache_dtype,step,latency
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,64,128,1,float16,float16,1,0.15296159982681273
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,64,128,1,float16,float16,3,0.15311039686203004
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,64,128,1,float16,float16,7,0.15379199981689454
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,64,128,1,float16,float16,15,0.15703519582748413
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,64,128,1,float16,float16,31,0.1633631944656372
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,64,128,1,float16,float16,63,0.17321280241012574
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,64,128,1,float16,float16,127,0.24606080055236818
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,64,128,1,float16,float16,255,0.4173439979553223
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,64,128,1,float16,float16,511,1.0508319854736328
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,64,128,1,float16,float16,2047,3.573923110961914
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,64,128,1,float16,float16,1023,1.8889888763427733
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,64,128,1,float16,float16,1,0.01706400066614151
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,64,128,1,float16,float16,7,0.017115199565887453
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,64,128,1,float16,float16,15,0.01717119961977005
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,64,128,1,float16,float16,31,0.01722400039434433
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,64,128,1,float16,float16,63,0.017351999878883362
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,64,128,1,float16,float16,127,0.020366400480270386
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,64,128,1,float16,float16,255,0.025654399394989015
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,64,128,1,float16,float16,511,0.052990400791168214
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,64,128,1,float16,float16,1023,0.07612000107765197
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,64,128,1,float16,float16,2047,0.11686079502105713
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,64,128,1,float16,float16,1,0.01815200001001358
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,64,128,1,float16,float16,3,0.018182399868965148
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,64,128,1,float16,float16,7,0.018169599771499633
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,64,128,1,float16,float16,3,0.016603200137615202
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,64,128,1,float16,float16,15,0.018136000633239745
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,64,128,1,float16,float16,31,0.018361599743366243
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,64,128,1,float16,float16,63,0.018457600474357606
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,64,128,1,float16,float16,127,0.02282560020685196
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,64,128,1,float16,float16,255,0.031553599238395694
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,64,128,1,float16,float16,511,0.07257120013236999
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,64,128,1,float16,float16,1023,0.11901600360870361
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,64,128,1,float16,float16,2047,0.19694240093231202
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,64,128,1,float16,float16,3,0.08451840281486511
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,64,128,1,float16,float16,1,0.08521119952201843
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,64,128,1,float16,float16,7,0.08451520204544068
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,64,128,1,float16,float16,15,0.08606560230255127
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,64,128,1,float16,float16,31,0.08903200030326844
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,64,128,1,float16,float16,63,0.09852319955825806
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,64,128,1,float16,float16,127,0.13476959466934205
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,64,128,1,float16,float16,255,0.22342240810394287
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,64,128,1,float16,float16,511,0.5357600212097168
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,64,128,1,float16,float16,1023,0.957420825958252
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,64,128,1,float16,float16,1,0.023296000063419343
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,64,128,1,float16,float16,3,0.02332800030708313
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,64,128,1,float16,float16,7,0.023236800730228425
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,64,128,1,float16,float16,15,0.02356960028409958
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,64,128,1,float16,float16,31,0.023676800727844238
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,64,128,1,float16,float16,63,0.024060800671577454
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,64,128,1,float16,float16,127,0.030420801043510436
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,64,128,1,float16,float16,255,0.04936000108718872
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,64,128,1,float16,float16,511,0.11066080331802368
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,64,128,1,float16,float16,1023,0.16882239580154418
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,64,128,1,float16,float16,2047,0.28646399974823
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,64,128,1,float16,float16,1,0.2940335988998413
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,64,128,1,float16,float16,2047,1.7253072738647461
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,64,128,1,float16,float16,7,0.30453920364379883
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,64,128,1,float16,float16,15,0.30757439136505127
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,64,128,1,float16,float16,31,0.31242079734802247
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,64,128,1,float16,float16,63,0.32766079902648926
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,64,128,1,float16,float16,3,0.2941648006439209
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,64,128,1,float16,float16,127,0.4698768138885498
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,64,128,1,float16,float16,255,0.7960656166076661
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,64,128,1,float16,float16,1,0.5814688205718994
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,64,128,1,float16,float16,3,0.5794144153594971
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,64,128,1,float16,float16,1023,3.8796192169189454
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,64,128,1,float16,float16,7,0.5845536231994629
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,64,128,1,float16,float16,511,2.0633312225341798
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,64,128,1,float16,float16,15,0.6001152038574219
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,64,128,1,float16,float16,31,0.6023200035095215
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,64,128,1,float16,float16,63,0.6352287769317627
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,64,128,1,float16,float16,127,0.9239312171936035
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,64,64,128,1,float16,float16,1,1.1515376091003418
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,64,64,128,1,float16,float16,7,1.1570159912109375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,64,128,1,float16,float16,255,1.5798720359802245
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,64,64,128,1,float16,float16,3,1.1556991577148437
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,64,64,128,1,float16,float16,15,1.1674896240234376
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,64,128,1,float16,float16,7,0.030859199166297913
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,64,64,128,1,float16,float16,31,1.2235247611999511
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,64,64,128,1,float16,float16,63,1.2356608390808106
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,64,128,1,float16,float16,31,0.031455999612808226
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,64,128,1,float16,float16,63,0.03505919873714447
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,64,128,1,float16,float16,127,0.05172640085220337
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,64,128,1,float16,float16,511,0.1572991967201233
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,64,128,1,float16,float16,2047,0.46978559494018557
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,64,128,1,float16,float16,1,0.03087199926376343
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,64,64,128,1,float16,float16,1,2.3121519088745117
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,64,64,128,1,float16,float16,3,2.306716728210449
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,64,128,1,float16,float16,1023,0.2655087947845459
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,64,64,128,1,float16,float16,127,1.8369695663452148
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,64,128,1,float16,float16,255,0.07316799759864807
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,64,64,128,1,float16,float16,7,2.291652870178223
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,64,64,128,1,float16,float16,15,2.319273567199707
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,64,64,128,1,float16,float16,31,2.360425567626953
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,64,128,1,float16,float16,15,0.0313647985458374
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,64,128,1,float16,float16,1,0.04819360077381134
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,64,128,1,float16,float16,3,0.04825280010700226
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,64,64,128,1,float16,float16,63,2.4596511840820314
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,64,128,1,float16,float16,7,0.048342400789260866
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,64,128,1,float16,float16,15,0.048825600743293764
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,64,128,1,float16,float16,31,0.049902400374412535
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,64,128,1,float16,float16,63,0.057920002937316896
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,64,128,1,float16,float16,127,0.07869600057601929
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,64,128,1,float16,float16,255,0.12267839908599854
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,64,64,128,1,float16,float16,3,4.541364669799805
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,64,128,1,float16,float16,511,0.28988959789276125
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,64,64,128,1,float16,float16,1,4.712908935546875
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,64,128,1,float16,float16,1023,0.5105008125305176
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,48,48,128,1,float16,float16,1,0.12084000110626221
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,48,48,128,1,float16,float16,7,0.12129600048065185
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,48,48,128,1,float16,float16,3,0.1191648006439209
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,64,128,1,float16,float16,3,0.030755200982093812
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,48,48,128,1,float16,float16,15,0.12215039730072022
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,64,128,1,float16,float16,2047,0.908505630493164
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,48,48,128,1,float16,float16,31,0.1275904059410095
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,48,48,128,1,float16,float16,127,0.19388320446014404
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,48,48,128,1,float16,float16,63,0.1355664014816284
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,64,64,128,1,float16,float16,31,4.728182220458985
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,48,48,128,1,float16,float16,511,0.7974495887756348
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,48,48,128,1,float16,float16,3,0.015459200739860535
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,48,48,128,1,float16,float16,1023,1.444219207763672
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,48,48,128,1,float16,float16,7,0.015404799580574035
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,48,48,128,1,float16,float16,1,0.01542080044746399
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,48,48,128,1,float16,float16,31,0.015454399585723876
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,48,48,128,1,float16,float16,15,0.015902400016784668
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,48,48,128,1,float16,float16,63,0.015588800609111785
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,48,48,128,1,float16,float16,127,0.018174399435520173
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,48,48,128,1,float16,float16,511,0.034764799475669864
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,48,48,128,1,float16,float16,1023,0.045259198546409606
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,48,48,128,1,float16,float16,255,0.02350880056619644
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,48,48,128,1,float16,float16,1,0.015833599865436553
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,48,48,128,1,float16,float16,2047,0.07032960057258605
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,48,48,128,1,float16,float16,7,0.015945599973201753
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,64,64,128,1,float16,float16,7,4.569680023193359
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,48,48,128,1,float16,float16,31,0.0160288006067276
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,48,48,128,1,float16,float16,15,0.01592320054769516
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,48,48,128,1,float16,float16,127,0.018908800184726716
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,48,48,128,1,float16,float16,255,0.023768000304698944
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,64,64,128,1,float16,float16,15,4.612689590454101
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,48,48,128,1,float16,float16,511,0.053495997190475465
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,48,48,128,1,float16,float16,1023,0.06875680088996887
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,48,48,128,1,float16,float16,1,0.0670144021511078
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,48,48,128,1,float16,float16,63,0.01650079935789108
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,48,48,128,1,float16,float16,7,0.06723200082778931
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,48,48,128,1,float16,float16,3,0.06736000180244446
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,48,48,128,1,float16,float16,2047,2.6872560501098635
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,48,48,128,1,float16,float16,31,0.07018240094184876
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,48,48,128,1,float16,float16,2047,0.10862560272216797
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,48,48,128,1,float16,float16,63,0.07876960039138795
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,48,48,128,1,float16,float16,255,0.17101279497146607
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,48,48,128,1,float16,float16,127,0.11473759412765502
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,48,48,128,1,float16,float16,1,0.018881599605083465
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,48,48,128,1,float16,float16,255,0.32151360511779786
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,48,48,128,1,float16,float16,3,0.018719999492168425
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,48,48,128,1,float16,float16,511,0.42336478233337405
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,48,48,128,1,float16,float16,7,0.017766399681568144
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,48,48,128,1,float16,float16,63,0.018401600420475006
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,48,48,128,1,float16,float16,31,0.017720000445842744
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,48,48,128,1,float16,float16,127,0.0226623997092247
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,48,48,128,1,float16,float16,1023,0.7593200206756592
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,48,48,128,1,float16,float16,255,0.03368319869041443
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,48,48,128,1,float16,float16,511,0.07468000054359436
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,48,48,128,1,float16,float16,1023,0.11866400241851807
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,48,48,128,1,float16,float16,2047,0.19797439575195314
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,48,48,128,1,float16,float16,1,0.2257391929626465
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,48,48,128,1,float16,float16,3,0.22640159130096435
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,48,48,128,1,float16,float16,2047,1.3435407638549806
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,48,48,128,1,float16,float16,7,0.22757439613342284
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,48,48,128,1,float16,float16,15,0.2319711923599243
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,48,48,128,1,float16,float16,31,0.23966081142425538
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,48,48,128,1,float16,float16,3,0.01613759994506836
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,48,48,128,1,float16,float16,63,0.26203839778900145
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,48,48,128,1,float16,float16,127,0.36285440921783446
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,48,48,128,1,float16,float16,255,0.6093264102935791
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,48,48,128,1,float16,float16,1,0.4478464126586914
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,48,48,128,1,float16,float16,15,0.06778879761695862
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,48,48,128,1,float16,float16,3,0.44179520606994627
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,48,48,128,1,float16,float16,7,0.4479824066162109
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,48,48,128,1,float16,float16,15,0.45725598335266116
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,48,48,128,1,float16,float16,31,0.4610735893249512
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,48,48,128,1,float16,float16,511,1.564080047607422
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,48,48,128,1,float16,float16,63,0.4798592090606689
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,48,48,128,1,float16,float16,15,0.018223999440670012
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,48,48,128,1,float16,float16,127,0.7020976066589355
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,48,48,128,1,float16,float16,1,0.8766207695007324
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,48,48,128,1,float16,float16,15,0.8926048278808594
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,48,48,128,1,float16,float16,7,0.8772496223449707
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,48,48,128,1,float16,float16,3,0.8945759773254395
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,48,48,128,1,float16,float16,1,0.02508319914340973
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,48,48,128,1,float16,float16,3,0.02528960108757019
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,48,48,128,1,float16,float16,15,0.02534720003604889
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,48,48,128,1,float16,float16,31,0.9069359779357911
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,48,48,128,1,float16,float16,31,0.025751999020576476
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,48,48,128,1,float16,float16,63,0.026072001457214354
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,48,48,128,1,float16,float16,1023,2.905504035949707
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,48,48,128,1,float16,float16,255,0.05769439935684204
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,48,48,128,1,float16,float16,511,0.1271391987800598
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,48,48,128,1,float16,float16,63,0.9448575973510742
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,48,48,128,1,float16,float16,1023,0.22497758865356446
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,48,48,128,1,float16,float16,2047,0.3713520050048828
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,48,48,128,1,float16,float16,127,1.374067211151123
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,48,48,128,1,float16,float16,1,1.7269872665405273
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,48,48,128,1,float16,float16,3,1.7371088027954102
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,48,48,128,1,float16,float16,7,1.7708927154541017
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,48,48,128,1,float16,float16,255,1.1998784065246582
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,48,48,128,1,float16,float16,7,0.025387200713157653
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,48,48,128,1,float16,float16,15,1.805816078186035
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,48,48,128,1,float16,float16,127,0.03782399892807007
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,48,48,128,1,float16,float16,31,1.783510398864746
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,48,48,128,1,float16,float16,1,0.03963040113449097
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,48,48,128,1,float16,float16,63,1.8646736145019531
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,48,48,128,1,float16,float16,3,0.03979839980602264
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,48,48,128,1,float16,float16,7,0.04007680118083954
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,48,48,128,1,float16,float16,15,0.04012640118598938
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,48,48,128,1,float16,float16,31,0.04066720008850098
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,48,48,128,1,float16,float16,63,0.044689598679542544
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,48,48,128,1,float16,float16,127,0.06470239758491517
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,48,48,128,1,float16,float16,255,0.09882240295410157
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,48,48,128,1,float16,float16,511,0.2350383996963501
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,48,48,128,1,float16,float16,1023,0.40897440910339355
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,48,48,128,1,float16,float16,1,3.446084976196289
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,40,40,128,1,float16,float16,1,0.10091520547866821
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,40,40,128,1,float16,float16,7,0.10138720273971558
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,40,40,128,1,float16,float16,15,0.10254559516906739
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,40,40,128,1,float16,float16,31,0.10715839862823487
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,48,48,128,1,float16,float16,7,3.445966339111328
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,48,48,128,1,float16,float16,15,3.509409713745117
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,48,48,128,1,float16,float16,3,3.465188980102539
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,40,40,128,1,float16,float16,63,0.11607840061187744
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,40,40,128,1,float16,float16,127,0.164792001247406
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,48,48,128,1,float16,float16,31,3.557651138305664
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,40,40,128,1,float16,float16,511,0.665723180770874
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,40,40,128,1,float16,float16,1,0.015316799283027649
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,40,40,128,1,float16,float16,3,0.015273599326610566
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,40,40,128,1,float16,float16,15,0.015390400588512421
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,40,40,128,1,float16,float16,31,0.015185600519180298
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,40,40,128,1,float16,float16,63,0.015390400588512421
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,40,40,128,1,float16,float16,1023,1.1983951568603515
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,40,40,128,1,float16,float16,7,0.015140800178050995
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,40,40,128,1,float16,float16,127,0.01789119988679886
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,40,40,128,1,float16,float16,255,0.022867199778556824
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,40,40,128,1,float16,float16,1,0.01552480012178421
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,40,40,128,1,float16,float16,1023,0.04286080002784729
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,40,40,128,1,float16,float16,511,0.03245759904384613
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,40,40,128,1,float16,float16,3,0.015497599542140961
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,40,40,128,1,float16,float16,7,0.015281599760055543
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,40,40,128,1,float16,float16,15,0.015340800583362579
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,40,40,128,1,float16,float16,63,0.015617600083351136
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,40,40,128,1,float16,float16,127,0.018265600502490997
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,40,40,128,1,float16,float16,255,0.02308479994535446
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,40,40,128,1,float16,float16,511,0.04163680076599121
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,40,40,128,1,float16,float16,1023,0.07291839718818664
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,40,40,128,1,float16,float16,3,0.1007423996925354
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,40,40,128,1,float16,float16,2047,0.11585919857025147
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,40,40,128,1,float16,float16,1,0.05578240156173706
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,40,40,128,1,float16,float16,3,0.05591679811477661
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,40,40,128,1,float16,float16,15,0.05630559921264648
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,40,40,128,1,float16,float16,31,0.05719519853591919
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,40,40,128,1,float16,float16,63,0.0658896028995514
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,48,48,128,1,float16,float16,2047,0.7315375804901123
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,40,40,128,1,float16,float16,2047,2.2050479888916015
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,40,40,128,1,float16,float16,255,0.1450543999671936
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,40,40,128,1,float16,float16,511,0.3464047908782959
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,40,40,128,1,float16,float16,1,0.01744319945573807
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,40,40,128,1,float16,float16,255,0.27273120880126955
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,40,40,128,1,float16,float16,3,0.01743199974298477
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,40,40,128,1,float16,float16,7,0.017316800355911256
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,40,40,128,1,float16,float16,15,0.017603200674057008
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,40,40,128,1,float16,float16,31,0.017553600668907165
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,40,40,128,1,float16,float16,63,0.018036800622940063
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,40,40,128,1,float16,float16,127,0.022172799706459044
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,40,40,128,1,float16,float16,255,0.03148959875106812
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,40,40,128,1,float16,float16,2047,1.0824543952941894
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,40,40,128,1,float16,float16,1023,0.11678880453109741
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,40,40,128,1,float16,float16,511,0.07252640128135682
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,40,40,128,1,float16,float16,2047,0.19587359428405762
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,40,40,128,1,float16,float16,1,0.19088319540023804
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,40,40,128,1,float16,float16,3,0.1900879979133606
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,40,40,128,1,float16,float16,2047,0.08255680203437805
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,40,40,128,1,float16,float16,15,0.1949072003364563
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,40,40,128,1,float16,float16,7,0.19054559469223023
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,40,40,128,1,float16,float16,31,0.20159358978271485
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,40,40,128,1,float16,float16,31,0.015659199655056
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,40,40,128,1,float16,float16,63,0.21258399486541749
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,40,40,128,1,float16,float16,127,0.30508320331573485
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,40,40,128,1,float16,float16,1,0.368067193031311
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,40,40,128,1,float16,float16,7,0.05600000023841858
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,40,40,128,1,float16,float16,255,0.5184879779815674
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,40,40,128,1,float16,float16,3,0.37126080989837645
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,40,40,128,1,float16,float16,127,0.09047840237617492
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,40,40,128,1,float16,float16,7,0.37087039947509765
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,40,40,128,1,float16,float16,15,0.3809216022491455
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,40,40,128,1,float16,float16,511,1.3040464401245118
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,40,40,128,1,float16,float16,31,0.3883359909057617
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,40,40,128,1,float16,float16,63,0.40610241889953613
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,40,40,128,1,float16,float16,127,0.5868303775787354
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,40,40,128,1,float16,float16,1023,0.6494624137878418
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,40,40,128,1,float16,float16,1,0.7320640087127686
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,40,40,128,1,float16,float16,1023,2.4104896545410157
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,40,40,128,1,float16,float16,255,0.9958864212036133
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,40,40,128,1,float16,float16,7,0.7357279777526855
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,40,40,128,1,float16,float16,15,0.7483200073242188
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,40,40,128,1,float16,float16,3,0.7346896171569824
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,40,40,128,1,float16,float16,1,0.024716800451278685
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,40,40,128,1,float16,float16,3,0.02311519980430603
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,40,40,128,1,float16,float16,15,0.02375520020723343
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,40,40,128,1,float16,float16,127,0.031886398792266846
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,40,40,128,1,float16,float16,255,0.055934399366378784
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,40,40,128,1,float16,float16,511,0.10238560438156127
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,40,40,128,1,float16,float16,127,1.1612239837646485
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,40,40,128,1,float16,float16,1023,0.16907999515533448
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,40,40,128,1,float16,float16,2047,0.29907519817352296
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,40,40,128,1,float16,float16,1,1.4416000366210937
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,40,40,128,1,float16,float16,3,1.4671199798583985
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,40,40,128,1,float16,float16,31,0.7587935924530029
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,40,40,128,1,float16,float16,63,0.8018735885620117
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,40,40,128,1,float16,float16,31,0.02481440007686615
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,40,40,128,1,float16,float16,7,1.450656032562256
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,40,40,128,1,float16,float16,15,1.4837455749511719
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,40,40,128,1,float16,float16,31,1.4987711906433105
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,40,40,128,1,float16,float16,63,1.5680352210998536
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,40,40,128,1,float16,float16,1,2.862063980102539
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,40,40,128,1,float16,float16,1,0.032892799377441405
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,40,40,128,1,float16,float16,3,0.032888001203536986
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,40,40,128,1,float16,float16,7,0.033051198720932005
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,40,40,128,1,float16,float16,15,0.03330720067024231
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,40,40,128,1,float16,float16,31,0.033641600608825685
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,40,40,128,1,float16,float16,7,0.023423999547958374
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,40,40,128,1,float16,float16,63,0.0350959986448288
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,40,40,128,1,float16,float16,7,2.8789039611816407
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,40,40,128,1,float16,float16,127,0.053320002555847165
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,40,40,128,1,float16,float16,255,0.08064960241317749
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,40,40,128,1,float16,float16,511,0.1825152039527893
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,40,40,128,1,float16,float16,15,2.9205743789672853
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,40,40,128,1,float16,float16,1023,0.31447041034698486
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,40,40,128,1,float16,float16,31,3.003116798400879
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,32,128,1,float16,float16,1,0.08445760011672973
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,40,40,128,1,float16,float16,2047,0.5507184028625488
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,32,128,1,float16,float16,15,0.08582080006599427
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,32,128,1,float16,float16,63,0.09787359833717346
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,32,128,1,float16,float16,31,0.08977919816970825
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,32,128,1,float16,float16,255,0.21742079257965088
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,32,128,1,float16,float16,127,0.13575199842453003
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,32,128,1,float16,float16,511,0.536294412612915
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,32,128,1,float16,float16,1,0.014820800721645355
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,32,128,1,float16,float16,3,0.014843200147151948
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,40,40,128,1,float16,float16,3,2.8662864685058596
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,32,128,1,float16,float16,7,0.01485760062932968
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,32,128,1,float16,float16,31,0.014947199821472168
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,32,128,1,float16,float16,63,0.015089599788188935
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,32,128,1,float16,float16,1023,0.9638735771179199
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,32,128,1,float16,float16,127,0.017548799514770508
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,32,128,1,float16,float16,255,0.02242400050163269
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,32,128,1,float16,float16,511,0.02861439883708954
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,32,128,1,float16,float16,2047,0.05472319722175598
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,32,128,1,float16,float16,4095,0.08243520259857177
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,32,128,1,float16,float16,3,0.08478080034255982
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,32,128,1,float16,float16,1,0.015139199793338776
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,32,128,1,float16,float16,7,0.08519200086593628
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,32,128,1,float16,float16,2047,1.7197328567504884
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,32,128,1,float16,float16,7,0.015068799257278442
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,32,128,1,float16,float16,3,0.01499200016260147
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,32,128,1,float16,float16,15,0.015108799934387207
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,32,128,1,float16,float16,63,0.01523360013961792
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,32,128,1,float16,float16,127,0.017880000174045563
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,32,128,1,float16,float16,511,0.04700160026550293
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,32,128,1,float16,float16,2047,0.11222720146179199
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,32,128,1,float16,float16,4095,0.1927791953086853
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,32,128,1,float16,float16,1,0.048158401250839235
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,32,128,1,float16,float16,3,0.048291200399398805
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,32,128,1,float16,float16,7,0.048500800132751466
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,32,128,1,float16,float16,15,0.04875200092792511
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,32,128,1,float16,float16,31,0.04903680086135864
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,32,128,1,float16,float16,4095,3.415827178955078
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,32,128,1,float16,float16,63,0.056436800956726076
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,32,128,1,float16,float16,127,0.08468639850616455
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,40,40,128,1,float16,float16,63,0.024959999322891235
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,32,128,1,float16,float16,255,0.1213487982749939
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,32,128,1,float16,float16,15,0.014895999431610107
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,32,128,1,float16,float16,511,0.3040175914764404
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,32,128,1,float16,float16,1023,0.038524800539016725
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,32,128,1,float16,float16,1,0.017473599314689635
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,32,128,1,float16,float16,1023,0.5085087776184082
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,32,128,1,float16,float16,7,0.01754560023546219
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,32,128,1,float16,float16,15,0.017047999799251555
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,32,128,1,float16,float16,31,0.017849600315093993
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,32,128,1,float16,float16,63,0.0174687996506691
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,32,128,1,float16,float16,127,0.022473600506782532
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,32,128,1,float16,float16,255,0.0299919992685318
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,32,128,1,float16,float16,31,0.015960000455379486
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,32,128,1,float16,float16,511,0.07220320105552673
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,32,128,1,float16,float16,1023,0.07179359793663025
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,32,128,1,float16,float16,1023,0.11524000167846679
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,32,128,1,float16,float16,2047,0.19551199674606323
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,32,128,1,float16,float16,1,0.15556639432907104
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,32,128,1,float16,float16,3,0.15523359775543213
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,32,128,1,float16,float16,7,0.15665600299835206
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,32,128,1,float16,float16,4095,0.35926079750061035
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,32,128,1,float16,float16,4095,1.7099472045898438
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,32,128,1,float16,float16,15,0.15791200399398803
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,32,128,1,float16,float16,31,0.16477760076522827
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,32,128,1,float16,float16,63,0.17416160106658934
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,32,128,1,float16,float16,127,0.2540560007095337
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,32,128,1,float16,float16,255,0.43317279815673826
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,32,128,1,float16,float16,1,0.30870239734649657
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,32,128,1,float16,float16,511,1.051979160308838
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,32,128,1,float16,float16,3,0.30675840377807617
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,32,128,1,float16,float16,3,0.016872000694274903
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,32,128,1,float16,float16,7,0.3017823934555054
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,32,128,1,float16,float16,15,0.3151679992675781
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,32,128,1,float16,float16,2047,0.907102394104004
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,32,128,1,float16,float16,31,0.31371679306030276
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,32,128,1,float16,float16,1023,1.9021440505981446
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,32,128,1,float16,float16,63,0.3300335884094238
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,32,128,1,float16,float16,255,0.022835199534893037
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,32,128,1,float16,float16,127,0.4860400199890137
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,32,128,1,float16,float16,3,0.5928271770477295
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,32,128,1,float16,float16,1,0.6005055904388428
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,32,128,1,float16,float16,15,0.6090384006500245
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,32,128,1,float16,float16,31,0.610423994064331
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,32,128,1,float16,float16,63,0.6500815868377685
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,32,128,1,float16,float16,1,0.022881600260734557
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,32,128,1,float16,float16,2047,3.5779678344726564
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,32,128,1,float16,float16,3,0.022968000173568724
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,32,128,1,float16,float16,127,0.9364224433898926
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,32,128,1,float16,float16,511,2.0928735733032227
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,32,128,1,float16,float16,15,0.023158399760723113
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,32,128,1,float16,float16,31,0.023347200453281404
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,32,128,1,float16,float16,7,0.024622400104999543
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,32,128,1,float16,float16,127,0.030088001489639284
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,32,128,1,float16,float16,255,0.04745280146598816
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,32,128,1,float16,float16,511,0.10315680503845215
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,32,128,1,float16,float16,1023,0.1669103980064392
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,32,128,1,float16,float16,2047,0.28751680850982664
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,32,128,1,float16,float16,255,1.5706656455993653
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,32,128,1,float16,float16,4095,0.5380127906799317
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,32,128,1,float16,float16,7,0.5956528186798096
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,32,32,128,1,float16,float16,1,1.1856143951416016
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,32,32,128,1,float16,float16,3,1.1675215721130372
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,32,32,128,1,float16,float16,7,1.2418671607971192
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,32,32,128,1,float16,float16,31,1.20066556930542
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,32,32,128,1,float16,float16,63,1.2500191688537599
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,32,128,1,float16,float16,63,0.02378239929676056
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,32,32,128,1,float16,float16,127,1.8648048400878907
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,32,32,128,1,float16,float16,1,2.3525327682495116
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,32,128,1,float16,float16,255,0.8005743980407715
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,32,32,128,1,float16,float16,3,2.3056800842285154
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,32,32,128,1,float16,float16,7,2.332297515869141
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,32,32,128,1,float16,float16,15,2.3536752700805663
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,32,128,1,float16,float16,1,0.030551999807357788
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,32,32,128,1,float16,float16,15,1.2092960357666016
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,32,128,1,float16,float16,7,0.03059520125389099
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,32,128,1,float16,float16,3,0.0304639995098114
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,32,128,1,float16,float16,31,0.03123840093612671
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,32,128,1,float16,float16,63,0.03268480002880096
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,32,128,1,float16,float16,127,0.04829919934272766
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,32,128,1,float16,float16,255,0.06930400133132934
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,32,128,1,float16,float16,511,0.15614399909973145
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,32,32,128,1,float16,float16,31,2.4542768478393553
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,32,128,1,float16,float16,1023,0.2661679983139038
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,24,24,128,1,float16,float16,1,0.0672432005405426
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,32,128,1,float16,float16,2047,0.46859040260314944
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,24,24,128,1,float16,float16,31,0.06969919800758362
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,24,24,128,1,float16,float16,3,0.06736159920692444
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,24,24,128,1,float16,float16,127,0.10610239505767823
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,24,24,128,1,float16,float16,255,0.1732319951057434
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,24,24,128,1,float16,float16,63,0.07874720096588135
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,32,128,1,float16,float16,4095,0.8660896301269532
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,24,24,128,1,float16,float16,511,0.42292962074279783
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,32,32,128,1,float16,float16,63,2.48022403717041
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,24,24,128,1,float16,float16,1,0.014710399508476257
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,24,24,128,1,float16,float16,7,0.014654399454593658
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,24,24,128,1,float16,float16,15,0.01467359960079193
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,24,24,128,1,float16,float16,1023,0.7613887786865234
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,24,24,128,1,float16,float16,3,0.014771200716495514
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,24,24,128,1,float16,float16,31,0.014710399508476257
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,24,24,128,1,float16,float16,63,0.014727999269962311
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,24,24,128,1,float16,float16,255,0.022155199944972993
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,24,24,128,1,float16,float16,511,0.02776640057563782
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,24,24,128,1,float16,float16,1023,0.03286559879779816
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,24,24,128,1,float16,float16,2047,1.3533503532409668
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,24,24,128,1,float16,float16,4095,0.06870080232620239
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,24,24,128,1,float16,float16,2047,0.044409599900245664
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,24,24,128,1,float16,float16,3,0.014856000244617463
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,32,128,1,float16,float16,15,0.03113119900226593
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,24,24,128,1,float16,float16,7,0.014899200201034546
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,24,24,128,1,float16,float16,15,0.015036800503730774
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,24,24,128,1,float16,float16,63,0.01501920074224472
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,24,24,128,1,float16,float16,255,0.022575999796390533
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,24,24,128,1,float16,float16,31,0.015171200037002563
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,24,24,128,1,float16,float16,511,0.03318400084972382
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,24,24,128,1,float16,float16,1023,0.04369440078735352
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,24,24,128,1,float16,float16,4095,0.10850880146026612
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,24,24,128,1,float16,float16,2047,0.0690991997718811
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,24,24,128,1,float16,float16,1,0.040214401483535764
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,24,24,128,1,float16,float16,3,0.04011679887771606
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,24,24,128,1,float16,float16,15,0.06801760196685791
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,24,24,128,1,float16,float16,7,0.040479999780654904
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,24,24,128,1,float16,float16,15,0.04056960046291351
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,24,24,128,1,float16,float16,4095,2.6270095825195314
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,24,24,128,1,float16,float16,127,0.06354079842567444
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,24,24,128,1,float16,float16,255,0.0974735975265503
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,24,24,128,1,float16,float16,511,0.2344127893447876
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,24,24,128,1,float16,float16,1023,0.41139998435974123
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,24,24,128,1,float16,float16,127,0.01751520037651062
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,24,24,128,1,float16,float16,1,0.01488800048828125
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,24,24,128,1,float16,float16,127,0.017726400494575502
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,24,24,128,1,float16,float16,1,0.015169599652290344
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,24,24,128,1,float16,float16,2047,0.7305808067321777
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,24,24,128,1,float16,float16,7,0.015281599760055543
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,24,24,128,1,float16,float16,3,0.015305599570274353
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,24,24,128,1,float16,float16,31,0.015329599380493164
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,24,24,128,1,float16,float16,63,0.015699200332164764
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,24,24,128,1,float16,float16,15,0.015177600085735321
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,24,24,128,1,float16,float16,127,0.018212799727916718
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,24,24,128,1,float16,float16,255,0.023287999629974365
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,24,24,128,1,float16,float16,1023,0.06682239770889283
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,24,24,128,1,float16,float16,511,0.04277600049972534
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,24,24,128,1,float16,float16,2047,0.10634399652481079
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,24,24,128,1,float16,float16,4095,1.3622112274169922
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,24,24,128,1,float16,float16,4095,0.18585439920425414
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,24,24,128,1,float16,float16,3,0.12122880220413208
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,24,24,128,1,float16,float16,31,0.040838399529457094
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,24,24,128,1,float16,float16,7,0.06758880019187927
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,24,24,128,1,float16,float16,7,0.12526559829711914
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,24,24,128,1,float16,float16,31,0.1288416028022766
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,24,24,128,1,float16,float16,15,0.12304480075836181
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,24,24,128,1,float16,float16,63,0.13639039993286134
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,24,24,128,1,float16,float16,127,0.1958624005317688
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,24,24,128,1,float16,float16,255,0.3268735885620117
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,24,24,128,1,float16,float16,1,0.22897601127624512
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,24,24,128,1,float16,float16,3,0.23161280155181885
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,24,24,128,1,float16,float16,511,0.8016575813293457
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,24,24,128,1,float16,float16,7,0.23043200969696045
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,24,24,128,1,float16,float16,15,0.24025120735168456
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,24,24,128,1,float16,float16,31,0.24348800182342528
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,24,24,128,1,float16,float16,1023,1.448350429534912
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,24,24,128,1,float16,float16,127,0.3714015960693359
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,24,24,128,1,float16,float16,255,0.6132512092590332
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,24,24,128,1,float16,float16,1,0.45211358070373536
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,24,24,128,1,float16,float16,3,0.45015840530395507
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,24,24,128,1,float16,float16,2047,2.6852848052978517
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,24,24,128,1,float16,float16,7,0.45534720420837405
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,24,24,128,1,float16,float16,63,0.0434224009513855
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,24,24,128,1,float16,float16,1,0.12119200229644775
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,24,24,128,1,float16,float16,511,1.6060575485229491
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,24,24,128,1,float16,float16,15,0.4610432147979736
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,24,24,128,1,float16,float16,1,0.016995200514793397
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,24,24,128,1,float16,float16,3,0.016803200542926788
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,24,24,128,1,float16,float16,7,0.017151999473571777
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,24,24,128,1,float16,float16,63,0.4905695915222168
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,24,24,128,1,float16,float16,15,0.0171424001455307
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,24,24,128,1,float16,float16,31,0.4959424018859863
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,24,24,128,1,float16,float16,31,0.017292800545692443
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,24,24,128,1,float16,float16,127,0.7111135959625244
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,24,24,128,1,float16,float16,127,0.0218639999628067
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,24,24,128,1,float16,float16,255,0.03289600014686585
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,24,24,128,1,float16,float16,511,0.07265120148658752
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,24,24,128,1,float16,float16,1023,0.11723359823226928
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,24,24,128,1,float16,float16,2047,0.19462239742279053
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,24,24,128,1,float16,float16,255,1.2102543830871582
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,24,24,128,1,float16,float16,4095,0.36428639888763426
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,24,24,128,1,float16,float16,63,0.2578975915908813
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,24,24,128,1,float16,float16,1,0.8951359748840332
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,24,24,128,1,float16,float16,3,0.892750358581543
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,24,24,128,1,float16,float16,7,0.8886719703674316
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,24,24,128,1,float16,float16,15,0.9422207832336426
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,24,24,128,1,float16,float16,31,0.9369695663452149
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,24,24,128,1,float16,float16,63,0.9566399574279785
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,24,24,128,1,float16,float16,127,1.4116512298583985
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,24,24,128,1,float16,float16,1,1.752742385864258
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,24,24,128,1,float16,float16,3,1.7485776901245118
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,24,24,128,1,float16,float16,3,0.025044798851013184
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,24,24,128,1,float16,float16,63,0.017561599612236023
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,24,24,128,1,float16,float16,1,0.025889599323272706
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,24,24,128,1,float16,float16,7,0.02503040134906769
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,24,24,128,1,float16,float16,7,1.779897689819336
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,24,24,128,1,float16,float16,31,0.025391998887062072
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,24,24,128,1,float16,float16,15,0.025360000133514405
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,24,24,128,1,float16,float16,127,0.03541919887065888
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,24,24,128,1,float16,float16,63,0.02653760015964508
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,24,24,128,1,float16,float16,255,0.05694559812545776
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,24,24,128,1,float16,float16,511,0.12573920488357543
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,24,24,128,1,float16,float16,31,1.8143087387084962
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,24,24,128,1,float16,float16,1023,0.21459519863128662
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,24,24,128,1,float16,float16,2047,0.37084479331970216
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,16,128,1,float16,float16,1,0.04905920028686524
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,16,128,1,float16,float16,15,0.04939039945602417
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,16,128,1,float16,float16,63,0.05766559839248657
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,24,24,128,1,float16,float16,4095,0.7029263973236084
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,16,128,1,float16,float16,31,0.049614399671554565
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,16,128,1,float16,float16,127,0.07749760150909424
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,16,128,1,float16,float16,255,0.1207200050354004
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,24,24,128,1,float16,float16,63,1.915003204345703
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,16,128,1,float16,float16,511,0.294707202911377
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,16,128,1,float16,float16,1,0.014467200636863709
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,16,128,1,float16,float16,3,0.014539200067520141
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,16,128,1,float16,float16,7,0.014427199959754944
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,16,128,1,float16,float16,15,0.014523200690746307
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,16,128,1,float16,float16,2047,0.9139583587646485
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,16,128,1,float16,float16,31,0.014561599493026734
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,16,128,1,float16,float16,63,0.014660799503326416
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,16,128,1,float16,float16,127,0.017334400117397307
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,16,128,1,float16,float16,255,0.022043199837207796
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,16,128,1,float16,float16,511,0.023571200668811798
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,24,24,128,1,float16,float16,15,1.8092575073242188
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,16,128,1,float16,float16,1023,0.02757279872894287
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,16,128,1,float16,float16,4095,1.7313407897949218
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,16,128,1,float16,float16,2047,0.03696799874305725
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,16,128,1,float16,float16,8191,0.08644160032272338
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,16,128,1,float16,float16,1,0.014455999433994293
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,16,128,1,float16,float16,3,0.048526400327682497
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,16,128,1,float16,float16,7,0.014451199769973755
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,16,128,1,float16,float16,15,0.014457599818706512
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,16,128,1,float16,float16,63,0.014611199498176575
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,16,128,1,float16,float16,127,0.017313599586486816
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,16,128,1,float16,float16,255,0.022203199565410614
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,16,128,1,float16,float16,511,0.028417599201202393
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,16,128,1,float16,float16,1023,0.03776960074901581
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,16,128,1,float16,float16,2047,0.053598397970199586
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,16,128,1,float16,float16,1023,0.510641622543335
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,16,128,1,float16,float16,4095,0.08142880201339722
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,16,128,1,float16,float16,1,0.03059520125389099
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,16,128,1,float16,float16,8191,3.512795257568359
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,16,128,1,float16,float16,3,0.03062559962272644
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,16,128,1,float16,float16,15,0.030900800228118898
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,16,128,1,float16,float16,7,0.03340159952640533
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,16,128,1,float16,float16,31,0.031123200058937074
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,16,128,1,float16,float16,63,0.03492160141468048
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,16,128,1,float16,float16,127,0.04568960070610047
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,16,128,1,float16,float16,255,0.0721455991268158
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,16,128,1,float16,float16,511,0.15447200536727906
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,16,128,1,float16,float16,4095,0.05411520004272461
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,16,128,1,float16,float16,1023,0.27556159496307375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,16,128,1,float16,float16,3,0.015004800260066986
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,16,128,1,float16,float16,7,0.048716801404953006
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,16,128,1,float16,float16,1,0.014752000570297241
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,16,128,1,float16,float16,3,0.014870400726795196
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,16,128,1,float16,float16,7,0.014707200229167938
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,16,128,1,float16,float16,2047,0.4633615970611572
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,16,128,1,float16,float16,15,0.01480800062417984
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,16,128,1,float16,float16,31,0.014883199334144592
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,16,128,1,float16,float16,127,0.017713600397109987
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,16,128,1,float16,float16,255,0.02255360037088394
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,16,128,1,float16,float16,63,0.014849600195884705
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,16,128,1,float16,float16,1023,0.07048959732055664
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,16,128,1,float16,float16,4095,0.862063980102539
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,16,128,1,float16,float16,511,0.045665600895881654
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,16,128,1,float16,float16,2047,0.11191999912261963
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,16,128,1,float16,float16,1,0.086627197265625
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,16,128,1,float16,float16,4095,0.1916815996170044
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,16,128,1,float16,float16,3,0.08642879724502564
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,16,128,1,float16,float16,8191,0.35539200305938723
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,16,128,1,float16,float16,7,0.08681120276451111
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,16,128,1,float16,float16,15,0.08787680268287659
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,16,128,1,float16,float16,8191,0.13517600297927856
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,16,128,1,float16,float16,31,0.09014400243759155
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,16,128,1,float16,float16,63,0.09849119782447815
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,16,128,1,float16,float16,127,0.13514560461044312
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,16,128,1,float16,float16,255,0.21950559616088866
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,16,128,1,float16,float16,8191,1.7218576431274415
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,16,128,1,float16,float16,511,0.544536018371582
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,16,128,1,float16,float16,1,0.1591439962387085
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,16,128,1,float16,float16,3,0.1629088044166565
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,16,128,1,float16,float16,7,0.159332799911499
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,16,128,1,float16,float16,15,0.16136319637298585
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,16,128,1,float16,float16,31,0.17361439466476442
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,16,128,1,float16,float16,63,0.178164803981781
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,16,128,1,float16,float16,1023,0.9665056228637695
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,16,128,1,float16,float16,127,0.25233280658721924
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,16,128,1,float16,float16,255,0.42526397705078123
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,16,128,1,float16,float16,31,0.014425599575042724
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,16,128,1,float16,float16,1,0.30706560611724854
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,16,128,1,float16,float16,3,0.31104159355163574
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,16,128,1,float16,float16,2047,1.7392768859863281
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,16,128,1,float16,float16,15,0.3163935899734497
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,16,128,1,float16,float16,511,1.0692319869995117
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,16,128,1,float16,float16,31,0.32827680110931395
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,16,128,1,float16,float16,63,0.35870239734649656
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,16,128,1,float16,float16,127,0.48311362266540525
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,16,128,1,float16,float16,1023,1.9456447601318358
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,16,128,1,float16,float16,4095,3.4110767364501955
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,16,128,1,float16,float16,1,0.016686399281024934
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,16,128,1,float16,float16,31,0.01703519970178604
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,16,128,1,float16,float16,3,0.016361600160598753
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,16,128,1,float16,float16,15,0.017505599558353423
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,16,128,1,float16,float16,7,0.016942399740219116
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,16,128,1,float16,float16,127,0.021454399824142455
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,16,128,1,float16,float16,255,0.02932479977607727
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,16,128,1,float16,float16,255,0.8100416183471679
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,16,128,1,float16,float16,7,0.30778400897979735
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,16,128,1,float16,float16,511,0.07130399942398072
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,16,128,1,float16,float16,2047,0.19289920330047608
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,16,128,1,float16,float16,4095,0.35289599895477297
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,16,128,1,float16,float16,1,0.5936944007873535
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,16,128,1,float16,float16,8191,0.6863039970397949
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,16,128,1,float16,float16,7,0.6041232109069824
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,16,128,1,float16,float16,3,0.6139264106750488
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,16,128,1,float16,float16,15,0.6133759975433349
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,16,128,1,float16,float16,511,2.1276607513427734
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,16,128,1,float16,float16,63,0.6495151996612549
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,16,128,1,float16,float16,31,0.6216879844665527
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,16,128,1,float16,float16,127,0.9410112380981446
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,16,16,128,1,float16,float16,1,1.182680034637451
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,16,16,128,1,float16,float16,7,1.191659164428711
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,16,128,1,float16,float16,63,0.01736319959163666
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,16,128,1,float16,float16,255,1.6225791931152345
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,16,128,1,float16,float16,1023,0.11627039909362794
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,16,128,1,float16,float16,1,0.024352000653743745
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,16,16,128,1,float16,float16,15,1.2285663604736328
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,16,16,128,1,float16,float16,31,1.2241040229797364
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,16,128,1,float16,float16,3,0.02259040027856827
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,16,128,1,float16,float16,7,0.024502399563789367
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,16,128,1,float16,float16,15,0.022870400547981264
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,16,128,1,float16,float16,63,0.02294880002737045
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,16,128,1,float16,float16,31,0.02267040014266968
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,16,128,1,float16,float16,127,0.030806401371955873
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,16,128,1,float16,float16,255,0.04670720100402832
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,16,128,1,float16,float16,511,0.09700959920883179
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,16,128,1,float16,float16,1023,0.16397600173950194
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,16,16,128,1,float16,float16,63,1.2648719787597655
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,16,128,1,float16,float16,2047,0.29529759883880613
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,12,12,128,1,float16,float16,1,0.04106079936027527
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,16,128,1,float16,float16,4095,0.5299232006072998
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,12,12,128,1,float16,float16,15,0.04138559997081757
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,12,12,128,1,float16,float16,63,0.04447680115699768
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,16,16,128,1,float16,float16,127,1.8895584106445313
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,12,12,128,1,float16,float16,31,0.04185760021209717
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,12,12,128,1,float16,float16,127,0.06557919979095458
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,16,128,1,float16,float16,8191,1.022697639465332
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,12,12,128,1,float16,float16,255,0.10348960161209106
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,12,12,128,1,float16,float16,511,0.23756799697875977
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,12,12,128,1,float16,float16,1023,0.4144911766052246
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,12,12,128,1,float16,float16,1,0.0144896000623703
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,16,16,128,1,float16,float16,3,1.2003999710083009
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,12,12,128,1,float16,float16,3,0.014425599575042724
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,12,12,128,1,float16,float16,15,0.014585599303245544
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,12,12,128,1,float16,float16,2047,0.7289519786834717
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,12,12,128,1,float16,float16,7,0.014731200039386749
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,12,12,128,1,float16,float16,127,0.017179200053215028
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,12,12,128,1,float16,float16,63,0.014441600441932679
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,12,12,128,1,float16,float16,511,0.024459199607372285
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,12,12,128,1,float16,float16,255,0.021929599344730377
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,12,12,128,1,float16,float16,1023,0.028788799047470094
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,12,12,128,1,float16,float16,4095,1.3909791946411132
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,12,12,128,1,float16,float16,4095,0.045649600028991696
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,12,12,128,1,float16,float16,8191,0.07210239768028259
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,12,12,128,1,float16,float16,3,0.015063999593257904
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,12,12,128,1,float16,float16,7,0.014876799285411834
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,12,12,128,1,float16,float16,7,0.04128159880638123
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,12,12,128,1,float16,float16,15,0.014977599680423736
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,12,12,128,1,float16,float16,31,0.014908799529075622
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,12,12,128,1,float16,float16,63,0.015080000460147857
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,12,12,128,1,float16,float16,127,0.017582400143146514
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,12,12,128,1,float16,float16,255,0.022595199942588805
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,12,12,128,1,float16,float16,8191,2.6865007400512697
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,12,12,128,1,float16,float16,511,0.02922559976577759
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,12,12,128,1,float16,float16,1023,0.03544799983501434
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,12,12,128,1,float16,float16,2047,0.047358399629592894
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,12,12,128,1,float16,float16,4095,0.07102239727973939
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,12,12,128,1,float16,float16,1,0.026561599969863892
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,12,12,128,1,float16,float16,8191,0.11207360029220581
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,12,12,128,1,float16,float16,7,0.026547199487686156
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,12,12,128,1,float16,float16,3,0.025622400641441345
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,12,12,128,1,float16,float16,15,0.02617599964141846
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,12,12,128,1,float16,float16,31,0.026929599046707154
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,12,12,128,1,float16,float16,63,0.026668798923492432
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,12,12,128,1,float16,float16,31,0.014574399590492249
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,12,12,128,1,float16,float16,255,0.059803199768066403
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,12,12,128,1,float16,float16,511,0.1286944031715393
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,12,12,128,1,float16,float16,1023,0.21515519618988038
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,12,12,128,1,float16,float16,2047,0.03377119898796081
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,12,12,128,1,float16,float16,1,0.014931200444698334
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,12,12,128,1,float16,float16,3,0.014819200336933135
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,12,12,128,1,float16,float16,7,0.014771200716495514
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,12,12,128,1,float16,float16,31,0.01496800035238266
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,12,12,128,1,float16,float16,63,0.014849600195884705
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,12,12,128,1,float16,float16,1,0.014588800072669984
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,12,12,128,1,float16,float16,127,0.01763039976358414
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,12,12,128,1,float16,float16,2047,0.37542080879211426
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,12,12,128,1,float16,float16,255,0.02258560061454773
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,12,12,128,1,float16,float16,511,0.03436479866504669
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,12,12,128,1,float16,float16,4095,0.7111567974090576
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,12,12,128,1,float16,float16,1023,0.04588159918785095
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,12,12,128,1,float16,float16,2047,0.06972799897193908
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,12,12,128,1,float16,float16,4095,0.10974400043487549
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,12,12,128,1,float16,float16,3,0.041212800145149234
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,12,12,128,1,float16,float16,3,0.06934720277786255
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,12,12,128,1,float16,float16,1,0.06935200095176697
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,12,12,128,1,float16,float16,8191,0.19407360553741454
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,12,12,128,1,float16,float16,7,0.06963040232658387
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,12,12,128,1,float16,float16,31,0.07130879759788514
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,12,12,128,1,float16,float16,15,0.07019360065460205
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,12,12,128,1,float16,float16,127,0.11203680038452149
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,12,12,128,1,float16,float16,8191,1.342046356201172
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,12,12,128,1,float16,float16,63,0.08009920120239258
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,12,12,128,1,float16,float16,255,0.1742591977119446
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,12,12,128,1,float16,float16,3,0.12507840394973754
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,12,12,128,1,float16,float16,511,0.43251681327819824
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,12,12,128,1,float16,float16,1,0.12575199604034423
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,12,12,128,1,float16,float16,7,0.12580480575561523
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,12,12,128,1,float16,float16,15,0.1260800004005432
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,12,12,128,1,float16,float16,31,0.13267680406570434
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,12,12,128,1,float16,float16,1023,0.771452808380127
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,12,12,128,1,float16,float16,127,0.03709760010242462
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,12,12,128,1,float16,float16,63,0.1384335994720459
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,12,12,128,1,float16,float16,127,0.19701119661331176
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,12,12,128,1,float16,float16,2047,1.3768239974975587
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,12,12,128,1,float16,float16,255,0.33143200874328616
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,12,12,128,1,float16,float16,15,0.014849600195884705
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,12,12,128,1,float16,float16,1,0.23662080764770507
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,12,12,128,1,float16,float16,7,0.237608003616333
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,12,12,128,1,float16,float16,15,0.24349920749664306
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,12,12,128,1,float16,float16,511,0.8186207771301269
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,12,12,128,1,float16,float16,3,0.2379535913467407
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,12,12,128,1,float16,float16,31,0.2502239942550659
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,12,12,128,1,float16,float16,63,0.2625312089920044
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,12,12,128,1,float16,float16,1,0.015110400319099427
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,12,12,128,1,float16,float16,3,0.014988799393177033
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,12,12,128,1,float16,float16,7,0.015134400129318238
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,12,12,128,1,float16,float16,127,0.3746623992919922
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,12,12,128,1,float16,float16,15,0.015086400508880615
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,12,12,128,1,float16,float16,31,0.01526239961385727
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,12,12,128,1,float16,float16,1023,1.4681039810180665
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,12,12,128,1,float16,float16,63,0.01563359946012497
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,12,12,128,1,float16,float16,255,0.6326879978179931
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,12,12,128,1,float16,float16,255,0.022988800704479218
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,12,12,128,1,float16,float16,4095,2.5982688903808593
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,12,12,128,1,float16,float16,511,0.04376640021800995
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,12,12,128,1,float16,float16,2047,0.10681120157241822
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,12,12,128,1,float16,float16,1023,0.0675872027873993
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,12,12,128,1,float16,float16,4095,0.18618559837341309
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,12,12,128,1,float16,float16,1,0.46569437980651857
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,12,12,128,1,float16,float16,8191,0.3844399929046631
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,12,12,128,1,float16,float16,3,0.4552703857421875
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,12,12,128,1,float16,float16,7,0.4560351848602295
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,12,12,128,1,float16,float16,15,0.46871199607849123
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,12,12,128,1,float16,float16,511,1.6075456619262696
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,12,12,128,1,float16,float16,31,0.4852992057800293
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,12,12,128,1,float16,float16,63,0.5306384086608886
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,12,12,128,1,float16,float16,127,0.7181439876556397
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,12,12,128,1,float16,float16,1,0.8941904067993164
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,12,12,128,1,float16,float16,3,0.9026608467102051
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,12,12,128,1,float16,float16,7,0.9092415809631348
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,12,12,128,1,float16,float16,15,0.9868720054626465
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,12,12,128,1,float16,float16,1,0.020899200439453126
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,12,12,128,1,float16,float16,31,0.9289664268493653
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,12,12,128,1,float16,float16,3,0.017481599748134614
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,12,12,128,1,float16,float16,63,0.9567647933959961
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,12,12,128,1,float16,float16,31,0.017505599558353423
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,12,12,128,1,float16,float16,63,0.01757120043039322
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,12,12,128,1,float16,float16,127,0.018027199804782866
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,12,12,128,1,float16,float16,255,0.031788799166679385
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,12,12,128,1,float16,float16,511,0.07171679735183716
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,12,12,128,1,float16,float16,1023,0.11763999462127686
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,12,12,128,1,float16,float16,2047,0.19654560089111328
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,12,12,128,1,float16,float16,127,1.442307186126709
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,12,12,128,1,float16,float16,4095,0.35655200481414795
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,12,12,128,1,float16,float16,8191,0.6815663814544678
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,8,128,1,float16,float16,1,0.031630399823188785
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,8,128,1,float16,float16,15,0.031995201110839845
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,8,128,1,float16,float16,7,0.03128800094127655
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,8,128,1,float16,float16,63,0.03239200115203857
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,8,128,1,float16,float16,127,0.047830399870872495
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,8,128,1,float16,float16,255,0.07087839841842651
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,12,12,128,1,float16,float16,255,1.2363807678222656
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,12,12,128,1,float16,float16,7,0.01723040044307709
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,8,128,1,float16,float16,511,0.15737279653549194
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,8,128,1,float16,float16,1023,0.2667248010635376
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,12,12,128,1,float16,float16,15,0.017027199268341064
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,12,12,128,1,float16,float16,127,0.02223680019378662
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,8,128,1,float16,float16,3,0.014201599359512328
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,8,128,1,float16,float16,7,0.014127999544143677
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,8,128,1,float16,float16,15,0.014182400703430176
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,8,128,1,float16,float16,31,0.014256000518798828
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,8,128,1,float16,float16,4095,0.8671631813049316
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,8,128,1,float16,float16,63,0.014268800616264343
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,8,128,1,float16,float16,255,0.02168319970369339
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,8,128,1,float16,float16,127,0.016707199811935424
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,8,128,1,float16,float16,511,0.02260479927062988
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,8,128,1,float16,float16,2047,0.028088000416755677
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,8,128,1,float16,float16,1023,0.026972800493240356
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,8,128,1,float16,float16,4095,0.0370608001947403
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,8,128,1,float16,float16,3,0.03107680082321167
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,8,128,1,float16,float16,8191,1.6910160064697266
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,8,128,1,float16,float16,8191,0.05454720258712768
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,8,128,1,float16,float16,16383,0.08173120021820068
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,8,128,1,float16,float16,7,0.014166399836540222
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,8,128,1,float16,float16,15,0.014086399972438813
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,8,128,1,float16,float16,31,0.03218559920787811
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,8,128,1,float16,float16,63,0.01427839994430542
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,8,128,1,float16,float16,31,0.014176000654697419
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,8,128,1,float16,float16,127,0.017073599994182585
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,8,128,1,float16,float16,255,0.02178879976272583
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,8,128,1,float16,float16,1023,0.027900800108909607
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,8,128,1,float16,float16,511,0.023387199640274046
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,8,128,1,float16,float16,2047,0.03673279881477356
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,8,128,1,float16,float16,4095,0.05351200103759766
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,8,128,1,float16,float16,1,0.02292799949645996
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,8,128,1,float16,float16,8191,0.08172640204429626
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,8,128,1,float16,float16,16383,0.21347520351409913
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,8,128,1,float16,float16,7,0.022993600368499754
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,8,128,1,float16,float16,15,0.023174400627613067
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,8,128,1,float16,float16,31,0.023391999304294586
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,8,128,1,float16,float16,63,0.02364480048418045
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,8,128,1,float16,float16,127,0.029795199632644653
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,8,128,1,float16,float16,2047,0.46650080680847167
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,8,128,1,float16,float16,255,0.0455487996339798
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,8,128,1,float16,float16,511,0.09832800030708314
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,8,128,1,float16,float16,1023,0.1641551971435547
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,8,128,1,float16,float16,2047,0.2839871883392334
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,8,128,1,float16,float16,4095,0.5230976104736328
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,8,128,1,float16,float16,1,0.014256000518798828
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,8,128,1,float16,float16,1,0.01430879980325699
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,8,128,1,float16,float16,3,0.014209599792957306
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,8,128,1,float16,float16,3,0.015080000460147857
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,8,128,1,float16,float16,7,0.014407999813556671
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,8,128,1,float16,float16,15,0.014470399916172027
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,8,128,1,float16,float16,31,0.014452800154685974
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,8,128,1,float16,float16,8191,1.0126272201538087
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,8,128,1,float16,float16,63,0.014582400023937226
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,8,128,1,float16,float16,127,0.017041599750518797
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,8,128,1,float16,float16,511,0.02847360074520111
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,8,128,1,float16,float16,1023,0.0372191995382309
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,8,128,1,float16,float16,255,0.02184640020132065
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,8,128,1,float16,float16,2047,0.0539359986782074
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,8,128,1,float16,float16,4095,0.08123520016670227
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,8,128,1,float16,float16,3,0.022784000635147093
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,8,128,1,float16,float16,8191,0.13426560163497925
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,8,128,1,float16,float16,3,0.050108802318572995
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,8,128,1,float16,float16,1,0.04995039999485016
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,8,128,1,float16,float16,7,0.04991840124130249
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,8,128,1,float16,float16,15,0.05026400089263916
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,8,128,1,float16,float16,31,0.05084480047225952
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,8,128,1,float16,float16,16383,0.35034880638122556
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,8,128,1,float16,float16,63,0.058432000875473025
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,8,128,1,float16,float16,127,0.07847840189933777
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,8,128,1,float16,float16,255,0.12159680128097534
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,8,128,1,float16,float16,16383,3.4551055908203123
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,8,128,1,float16,float16,16383,1.9870271682739258
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,8,128,1,float16,float16,1,0.013929599523544311
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,8,128,1,float16,float16,3,0.08864960074424744
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,8,128,1,float16,float16,1,0.08984320163726807
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,8,128,1,float16,float16,1023,0.5169439792633057
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,8,128,1,float16,float16,7,0.08893920183181762
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,8,128,1,float16,float16,15,0.09061279892921448
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,8,128,1,float16,float16,31,0.09236320257186889
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,8,128,1,float16,float16,63,0.09880480170249939
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,8,128,1,float16,float16,127,0.13996000289916993
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,8,128,1,float16,float16,2047,0.9173551559448242
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,8,128,1,float16,float16,255,0.22402238845825195
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,8,128,1,float16,float16,1,0.1642303943634033
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,8,128,1,float16,float16,511,0.5531439781188965
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,8,128,1,float16,float16,3,0.16438239812850952
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,8,128,1,float16,float16,4095,1.7316543579101562
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,8,128,1,float16,float16,7,0.16482720375061036
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,8,128,1,float16,float16,15,0.16543840169906615
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,8,128,1,float16,float16,1023,0.9872367858886719
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,8,128,1,float16,float16,63,0.18167999982833863
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,8,128,1,float16,float16,31,0.18566880226135254
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,8,128,1,float16,float16,127,0.2563119888305664
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,8,128,1,float16,float16,1,0.014440000057220459
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,8,128,1,float16,float16,3,0.014427199959754944
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,8,128,1,float16,float16,255,0.43128161430358886
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,8,128,1,float16,float16,7,0.014300799369812012
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,8,128,1,float16,float16,2047,1.7520687103271484
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,8,128,1,float16,float16,15,0.014619199931621552
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,8,128,1,float16,float16,31,0.014497600495815277
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,8,128,1,float16,float16,255,0.02216159999370575
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,8,128,1,float16,float16,127,0.0173567995429039
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,8,128,1,float16,float16,63,0.015255999565124512
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,8,128,1,float16,float16,511,0.045286399126052854
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,8,128,1,float16,float16,2047,0.11352800130844116
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,8,128,1,float16,float16,1023,0.07045279741287232
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,8,128,1,float16,float16,8191,3.529220962524414
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,8,128,1,float16,float16,511,1.0782527923583984
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,8,128,1,float16,float16,4095,0.1901568055152893
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,8,128,1,float16,float16,8191,0.36275520324707033
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,8,8,128,1,float16,float16,7,0.30994400978088377
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,8,8,128,1,float16,float16,3,0.3093775987625122
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,8,8,128,1,float16,float16,1,0.32446720600128176
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,8,8,128,1,float16,float16,15,0.326092791557312
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,8,128,1,float16,float16,1023,1.985856056213379
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,8,128,1,float16,float16,16383,0.6798960208892822
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,8,8,128,1,float16,float16,31,0.32389121055603026
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,8,8,128,1,float16,float16,63,0.3373183965682983
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,8,8,128,1,float16,float16,127,0.5025040149688721
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,8,8,128,1,float16,float16,1,0.6054272174835205
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,8,8,128,1,float16,float16,3,0.6012928009033203
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,8,8,128,1,float16,float16,7,0.6074031829833985
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,8,8,128,1,float16,float16,255,0.8390735626220703
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,8,128,1,float16,float16,1,0.01669120043516159
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,8,8,128,1,float16,float16,15,0.6290463924407959
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,8,128,1,float16,float16,3,0.016553600132465363
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,8,128,1,float16,float16,7,0.01641920059919357
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,8,8,128,1,float16,float16,31,0.6302112102508545
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,8,8,128,1,float16,float16,63,0.6419583797454834
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,8,128,1,float16,float16,15,0.016883200407028197
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,8,128,1,float16,float16,31,0.016551999747753142
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,8,128,1,float16,float16,63,0.016897599399089813
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,8,128,1,float16,float16,127,0.021116800606250763
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,8,128,1,float16,float16,255,0.029791998863220214
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,8,8,128,1,float16,float16,127,0.9460495948791504
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,8,128,1,float16,float16,511,0.06933919787406921
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,8,128,1,float16,float16,511,0.29549760818481446
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,8,128,1,float16,float16,1023,0.11337120532989502
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,8,8,128,1,float16,float16,511,2.1873775482177735
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,8,128,1,float16,float16,2047,0.1958672046661377
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,8,128,1,float16,float16,4095,0.359935998916626
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,4,128,1,float16,float16,1,0.023454399406909944
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,4,128,1,float16,float16,31,0.023680000007152556
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,8,8,128,1,float16,float16,255,1.6433055877685547
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,8,128,1,float16,float16,8191,0.6874927997589111
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,4,128,1,float16,float16,63,0.023844799399375914
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,4,128,1,float16,float16,15,0.023678399622440338
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,4,128,1,float16,float16,511,0.0996288001537323
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,4,128,1,float16,float16,1023,0.16807839870452881
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,4,128,1,float16,float16,127,0.0326447993516922
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,8,128,1,float16,float16,16383,1.3240303993225098
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,4,128,1,float16,float16,2047,0.2849519968032837
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,4,128,1,float16,float16,4095,0.5296288013458252
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,4,128,1,float16,float16,1,0.014108799397945404
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,4,128,1,float16,float16,3,0.01419519931077957
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,4,128,1,float16,float16,7,0.01408960074186325
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,4,128,1,float16,float16,8191,1.045518398284912
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,4,128,1,float16,float16,15,0.014051200449466705
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,4,128,1,float16,float16,63,0.014286400377750396
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,4,128,1,float16,float16,31,0.014182400703430176
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,4,128,1,float16,float16,127,0.016808000206947327
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,4,128,1,float16,float16,255,0.02175839990377426
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,4,128,1,float16,float16,511,0.023647999763488768
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,4,128,1,float16,float16,1023,0.027913600206375122
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,4,128,1,float16,float16,2047,0.028200000524520874
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,4,128,1,float16,float16,4095,0.02935200035572052
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,4,128,1,float16,float16,8191,0.037540799379348753
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,4,128,1,float16,float16,16383,0.05660319924354553
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,4,128,1,float16,float16,16383,1.9934831619262696
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,4,128,1,float16,float16,3,0.02354400008916855
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,4,128,1,float16,float16,7,0.014103999733924866
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,4,128,1,float16,float16,3,0.014472000300884247
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,4,128,1,float16,float16,15,0.014057600498199463
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,4,128,1,float16,float16,31,0.014608000218868256
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,4,128,1,float16,float16,63,0.014211200177669525
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,4,128,1,float16,float16,127,0.017156800627708434
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,4,128,1,float16,float16,255,0.04759680032730103
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,4,128,1,float16,float16,255,0.021631999313831328
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,4,128,1,float16,float16,511,0.024355199933052064
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,4,128,1,float16,float16,1023,0.02844800055027008
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,4,128,1,float16,float16,2047,0.029131200909614564
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,4,128,1,float16,float16,8191,0.05505759716033935
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,4,128,1,float16,float16,16383,0.07966079711914062
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,4,128,1,float16,float16,32767,0.13091360330581664
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,4,128,1,float16,float16,1,0.01682240068912506
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,4,128,1,float16,float16,3,0.016947199404239655
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,4,128,1,float16,float16,7,0.016969600319862367
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,4,128,1,float16,float16,15,0.01711679995059967
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,4,128,1,float16,float16,31,0.017049600183963776
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,4,128,1,float16,float16,32767,3.9793521881103517
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,4,128,1,float16,float16,63,0.017334400117397307
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,4,128,1,float16,float16,127,0.02166240066289902
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,4,128,1,float16,float16,255,0.030827200412750243
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,4,128,1,float16,float16,511,0.07056159973144531
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,4,128,1,float16,float16,1023,0.1141327977180481
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,4,128,1,float16,float16,2047,0.1986016035079956
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,4,128,1,float16,float16,4095,0.35359199047088624
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,4,128,1,float16,float16,32767,0.08272640109062195
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,4,128,1,float16,float16,1,0.013977600634098053
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,4,128,1,float16,float16,1,0.014081600308418273
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,4,128,1,float16,float16,7,0.02343199998140335
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,4,128,1,float16,float16,3,0.014030399918556213
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,4,128,1,float16,float16,7,0.014000000059604644
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,4,128,1,float16,float16,15,0.014131200313568116
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,4,128,1,float16,float16,8191,0.6834767818450928
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,4,128,1,float16,float16,31,0.014126400649547576
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,4,128,1,float16,float16,63,0.014148800075054169
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,4,128,1,float16,float16,127,0.01687840074300766
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,4,128,1,float16,float16,511,0.0244719997048378
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,4,128,1,float16,float16,1023,0.0289247989654541
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,4,128,1,float16,float16,2047,0.03768320083618164
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,4,128,1,float16,float16,4095,0.05484319925308227
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,4,128,1,float16,float16,8191,0.08240479826927186
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,4,128,1,float16,float16,16383,0.213702392578125
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,4,128,1,float16,float16,1,0.03203200101852417
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,4,128,1,float16,float16,4095,0.03857919871807099
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,4,128,1,float16,float16,16383,1.3419103622436523
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,4,128,1,float16,float16,3,0.03215200006961823
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,4,128,1,float16,float16,32767,0.3997567892074585
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,4,128,1,float16,float16,15,0.03258079886436462
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,4,128,1,float16,float16,63,0.03354400098323822
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,4,128,1,float16,float16,31,0.033103999495506284
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,4,128,1,float16,float16,127,0.04970720112323761
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,4,128,1,float16,float16,255,0.0714847981929779
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,4,128,1,float16,float16,511,0.15883519649505615
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,4,128,1,float16,float16,1023,0.2718591928482056
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,4,128,1,float16,float16,2047,0.4688528060913086
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,4,128,1,float16,float16,32767,2.6137487411499025
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,4,4,128,1,float16,float16,1,0.05156959891319275
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,4,128,1,float16,float16,4095,0.8655232429504395
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,4,4,128,1,float16,float16,7,0.05171040296554565
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,4,4,128,1,float16,float16,3,0.051692801713943484
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,4,4,128,1,float16,float16,15,0.05264959931373596
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,4,4,128,1,float16,float16,31,0.052407997846603396
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,4,4,128,1,float16,float16,63,0.06159359812736511
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,4,4,128,1,float16,float16,127,0.08014559745788574
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,4,4,128,1,float16,float16,255,0.12294080257415771
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,4,128,1,float16,float16,8191,1.6987712860107422
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,4,128,1,float16,float16,255,0.021828800439834595
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,4,4,128,1,float16,float16,511,0.30254719257354734
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,4,4,128,1,float16,float16,1,0.09335200190544128
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,4,4,128,1,float16,float16,3,0.09425439834594726
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,4,4,128,1,float16,float16,1023,0.5203392028808593
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,4,128,1,float16,float16,7,0.03214080035686493
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,4,4,128,1,float16,float16,15,0.09478399753570557
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,4,4,128,1,float16,float16,31,0.09692639708518982
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,4,4,128,1,float16,float16,63,0.10311520099639893
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,4,4,128,1,float16,float16,2047,0.915987205505371
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,4,4,128,1,float16,float16,127,0.14445279836654662
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,4,4,128,1,float16,float16,255,0.23022079467773438
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,4,128,1,float16,float16,1,0.013974399864673614
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,4,128,1,float16,float16,3,0.014105600118637086
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,4,128,1,float16,float16,7,0.014244799315929414
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,4,4,128,1,float16,float16,511,0.5568272113800049
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,4,128,1,float16,float16,15,0.014273600280284881
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,4,4,128,1,float16,float16,4095,1.7464624404907227
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,4,128,1,float16,float16,63,0.01438560038805008
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,4,128,1,float16,float16,16383,3.4432449340820312
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,4,4,128,1,float16,float16,1023,0.9776896476745606
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,4,128,1,float16,float16,255,0.021891200542449953
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,4,128,1,float16,float16,31,0.014236800372600555
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,4,128,1,float16,float16,127,0.017688000202178956
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,4,128,1,float16,float16,2047,0.05539199709892273
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,4,128,1,float16,float16,511,0.03151040077209473
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,4,128,1,float16,float16,1023,0.03799999952316284
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,4,128,1,float16,float16,8191,0.1393936038017273
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,4,4,128,1,float16,float16,2047,1.7497535705566407
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,4,128,1,float16,float16,16383,0.35174880027770994
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,4,4,128,1,float16,float16,3,0.16809760332107543
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,4,4,128,1,float16,float16,15,0.16747839450836183
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,4,4,128,1,float16,float16,31,0.17847360372543336
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,4,4,128,1,float16,float16,7,0.09349439740180969
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,4,4,128,1,float16,float16,63,0.1804159998893738
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,4,4,128,1,float16,float16,127,0.26335840225219725
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,4,4,128,1,float16,float16,255,0.4302783966064453
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,4,4,128,1,float16,float16,1,0.31016480922698975
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,4,4,128,1,float16,float16,3,0.31052000522613527
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,4,4,128,1,float16,float16,511,1.0795151710510253
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,4,4,128,1,float16,float16,7,0.3107248067855835
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,4,4,128,1,float16,float16,15,0.32236480712890625
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,4,4,128,1,float16,float16,31,0.3259615898132324
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,4,128,1,float16,float16,4095,0.08294879794120788
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,4,4,128,1,float16,float16,63,0.3386528015136719
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,4,4,128,1,float16,float16,1,0.16537760496139525
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,4,4,128,1,float16,float16,1023,1.9566160202026368
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,4,128,1,float16,float16,1,0.014212800562381745
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,4,128,1,float16,float16,3,0.014478400349617004
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,4,128,1,float16,float16,15,0.014392000436782838
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,4,128,1,float16,float16,31,0.014537599682807923
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,4,128,1,float16,float16,63,0.014539200067520141
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,4,128,1,float16,float16,7,0.015003199875354766
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,4,128,1,float16,float16,32767,0.6928095817565918
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,4,128,1,float16,float16,127,0.0173552006483078
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,4,128,1,float16,float16,255,0.02194560021162033
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,4,128,1,float16,float16,511,0.04781599938869476
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,4,128,1,float16,float16,1023,0.07148000001907348
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,4,128,1,float16,float16,2047,0.11166080236434936
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,4,4,128,1,float16,float16,255,0.8288496017456055
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,4,128,1,float16,float16,4095,0.1927791953086853
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,4,128,1,float16,float16,8191,0.36179840564727783
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,1,0.017591999471187593
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,15,0.017817600071430205
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,7,0.01804479956626892
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,4,128,1,float16,float16,16383,0.6799183845520019
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,31,0.01797119975090027
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,63,0.018329599499702455
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,255,0.030840000510215758
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,127,0.02221599966287613
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,511,0.07252960205078125
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,4,4,128,1,float16,float16,511,2.1304832458496095
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,4,128,1,float16,float16,32767,1.3243632316589355
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,2047,0.19519200325012206
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,4095,0.36054880619049073
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,8191,0.6896912097930908
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,4,4,128,1,float16,float16,127,0.48651838302612305
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,4,4,128,1,float16,float16,7,0.17512799501419068
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,1,0.013886399567127228
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,15,0.013924799859523773
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,3,0.01379680037498474
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,31,0.014079999923706055
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,63,0.013964800536632538
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,127,0.01674560010433197
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,255,0.02138399928808212
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,511,0.024094399809837342
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,1023,0.028075200319290162
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,2047,0.02863200008869171
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,3,0.017382399737834932
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,8191,0.03178719878196716
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,16383,0.04041439890861511
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,32767,0.05845119953155518
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,65535,0.08536319732666016
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,1,0.013955199718475341
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,3,0.013886399567127228
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,7,0.013937599956989288
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,32767,2.6191055297851564
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,15,0.013953599333763122
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,1023,0.11753280162811279
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,63,0.014116799831390381
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,31,0.014060799777507783
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,255,0.021556800603866576
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,127,0.016728000342845918
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,2047,0.028591999411582948
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,511,0.02409760057926178
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,4095,0.028958401083946227
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,8191,0.04014720022678375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,16383,0.05384640097618103
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,32767,0.08232640027999878
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,65535,0.13548799753189086
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,1,0.015086400508880615
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,3,0.014937600493431092
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,15,0.014920000731945039
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,7,0.015113599598407745
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,31,0.014987200498580933
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,63,0.015248000621795654
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,127,0.017657600343227386
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,16383,1.3271679878234863
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,7,0.014127999544143677
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,511,0.047856000065803525
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,255,0.022697600722312927
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,1023,0.07360000014305115
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,2047,0.1128432035446167
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,4095,0.19372639656066895
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,4095,0.028601598739624024
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,1,0.01390880048274994
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,8191,0.3608527898788452
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,3,0.013948799669742584
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,15,0.014116799831390381
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,7,0.014227199554443359
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,31,0.014091199636459351
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,16383,0.6854959964752197
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,65535,5.204248046875
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,127,0.016689600050449373
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,1023,0.028803199529647827
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,255,0.021784000098705292
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,2047,0.029241600632667543
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,4095,0.03834879994392395
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,8191,0.057014399766922
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,32767,1.3250847816467286
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,16383,0.08218560218811036
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,1023,0.02847520112991333
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,32767,0.13156640529632568
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,2,2,128,1,float16,float16,3,0.02447039932012558
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,2,2,128,1,float16,float16,1,0.024561600387096406
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,2,2,128,1,float16,float16,7,0.024539199471473695
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,2,2,128,1,float16,float16,15,0.024604800343513488
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,2,2,128,1,float16,float16,31,0.02489120066165924
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,2,2,128,1,float16,float16,63,0.02510559856891632
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,2,2,128,1,float16,float16,255,0.048265600204467775
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,2,2,128,1,float16,float16,1023,0.1673535943031311
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,65535,2.618182373046875
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,2,2,128,1,float16,float16,2047,0.2867471933364868
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,2,2,128,1,float16,float16,4095,0.5285103797912598
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,63,0.014180800318717957
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,2,2,128,1,float16,float16,8191,1.0162591934204102
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,511,0.02431039959192276
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,2,2,128,1,float16,float16,1,0.03364799916744232
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,2,2,128,1,float16,float16,3,0.03395679891109467
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,2,2,128,1,float16,float16,7,0.03376800119876862
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,2,2,128,1,float16,float16,15,0.03426080048084259
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,65535,0.23726398944854737
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,2,2,128,1,float16,float16,63,0.03552800118923187
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,2,2,128,1,float16,float16,127,0.03142080008983612
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,2,2,128,1,float16,float16,255,0.07323520183563233
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,2,2,128,1,float16,float16,511,0.1016319990158081
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,2,2,128,1,float16,float16,16383,2.0003311157226564
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,2,2,128,1,float16,float16,511,0.16328320503234864
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,2,2,128,1,float16,float16,1023,0.2713599920272827
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,2,2,128,1,float16,float16,3,0.05584480166435242
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,2,2,128,1,float16,float16,1,0.05631679892539978
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,2,2,128,1,float16,float16,2047,0.4708655834197998
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,2,2,128,1,float16,float16,7,0.05597760081291199
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,2,2,128,1,float16,float16,15,0.05677919983863831
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,2,2,128,1,float16,float16,31,0.05681440234184265
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,2,2,128,1,float16,float16,255,0.12868640422821045
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,2,2,128,1,float16,float16,4095,0.8742095947265625
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,2,2,128,1,float16,float16,127,0.0856607973575592
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,2,2,128,1,float16,float16,511,0.3081568002700806
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,2,2,128,1,float16,float16,1023,0.5308656215667724
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,2,2,128,1,float16,float16,8191,1.7375024795532226
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,1,0.01404159963130951
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,7,0.014105600118637086
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,2,2,128,1,float16,float16,2047,0.9322640419006347
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,15,0.014105600118637086
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,63,0.014351999759674073
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,2,2,128,1,float16,float16,32767,3.927153778076172
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,127,0.017019200325012206
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,2,2,128,1,float16,float16,31,0.034332799911499026
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,1023,0.0291375994682312
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,2,2,128,1,float16,float16,127,0.04907520115375519
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,255,0.022008000314235686
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,2047,0.03799520134925842
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,4095,0.05482239723205566
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,8191,0.08621600270271301
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,16383,0.216593599319458
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,2,2,128,1,float16,float16,4095,1.7366592407226562
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,32767,0.4032400131225586
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,2,2,128,1,float16,float16,1,0.09876800179481507
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,2,2,128,1,float16,float16,3,0.0945248007774353
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,65535,0.7664544105529785
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,2,2,128,1,float16,float16,7,0.09709280133247375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,2,2,128,1,float16,float16,15,0.0949999988079071
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,2,2,128,1,float16,float16,31,0.0980288028717041
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,2,2,128,1,float16,float16,63,0.10837279558181763
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,2,2,128,1,float16,float16,127,0.1437376022338867
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,2,2,128,1,float16,float16,255,0.23181281089782715
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,2,2,128,1,float16,float16,511,0.5766816139221191
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,3,0.015279999375343323
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,31,0.014431999623775482
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,2,2,128,1,float16,float16,3,0.16792479753494263
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,2,2,128,1,float16,float16,1,0.16705119609832764
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,2,2,128,1,float16,float16,7,0.16669440269470215
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,2,2,128,1,float16,float16,15,0.16889280080795288
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,2,2,128,1,float16,float16,31,0.17844959497451782
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,2,2,128,1,float16,float16,1023,0.9889936447143555
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,2,2,128,1,float16,float16,63,0.1835055947303772
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,2,2,128,1,float16,float16,127,0.26025118827819826
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,1,0.014212800562381745
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,2,2,128,1,float16,float16,255,0.437662410736084
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,3,0.014185599982738495
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,2,2,128,1,float16,float16,63,0.06342719793319702
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,2,2,128,1,float16,float16,2047,1.7488288879394531
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,15,0.014291200041770934
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,7,0.014393599331378936
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,31,0.014215999841690063
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,2,2,128,1,float16,float16,511,1.1054287910461427
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,127,0.016939200460910797
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,511,0.029651200771331786
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,63,0.01539199948310852
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,255,0.022091199457645417
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,2047,0.055395197868347165
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,1023,0.03930239975452423
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,4095,0.08297600150108338
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,8191,0.1423248052597046
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,1,0.015544000267982482
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,7,0.015614399313926696
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,16383,0.35627360343933107
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,15,0.015740799903869628
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,2,2,128,1,float16,float16,1023,1.9555135726928712
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,3,0.016014400124549865
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,32767,0.6738016128540039
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,31,0.015992000699043274
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,255,0.02391680032014847
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,511,0.024849599599838255
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,127,0.01889919936656952
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,1023,0.07493280172348023
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,2047,0.11602239608764649
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,65535,1.3271136283874512
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,4095,0.19466559886932372
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,8191,0.36544480323791506
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,1,0.013972799479961395
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,3,0.014256000518798828
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,7,0.013979199528694152
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,15,0.014286400377750396
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,31,0.013948799669742584
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,16383,0.6867455959320068
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,127,0.016505600512027742
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,63,0.014417600631713868
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,255,0.021686400473117828
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,511,0.023683199286460878
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,1023,0.02880159914493561
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,4095,0.028148800134658813
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,2047,0.028321599960327147
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,8191,0.03170399963855743
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,16383,0.035945600271224974
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,32767,0.04541600048542023
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,32767,1.3523664474487305
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,131071,0.08987039923667908
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,1,0.013955199718475341
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,7,0.013876800239086152
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,15,0.013963200151920319
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,3,0.014025600254535675
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,31,0.014120000600814819
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,63,0.014078399538993836
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,127,0.016740800440311433
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,255,0.02154400050640106
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,511,0.02372319996356964
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,2047,0.028350400924682616
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,1023,0.028284800052642823
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,4095,0.028465598821640015
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,8191,0.03153600096702576
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,32767,0.058499199151992795
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,16383,0.04152320027351379
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,65535,0.08597440123558045
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,63,0.015769599378108977
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,1,0.014761599898338317
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,7,0.014985600113868713
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,3,0.014881600439548493
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,131071,0.14022560119628907
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,15,0.015028800070285796
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,63,0.014907200634479523
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,127,0.01762239933013916
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,255,0.022486400604248048
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,511,0.03046559989452362
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,1023,0.039547199010849
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,65535,2.631711959838867
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,2047,0.056094402074813844
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,4095,0.08405920267105102
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,8191,0.1397760033607483
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,16383,0.3604896068572998
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,32767,0.6809328079223633
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,1,0.014097599685192109
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,3,0.014107200503349304
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,7,0.014134399592876434
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,15,0.014105600118637086
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,65535,0.06420159935951233
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,31,0.014102399349212646
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,65535,1.3322943687438964
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,63,0.0143312007188797
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,127,0.016729600727558136
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,511,0.024380800127983094
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,255,0.02150239944458008
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,2047,0.02887679934501648
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,131071,5.203371047973633
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,4095,0.028958401083946227
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,8191,0.04032639861106872
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,16383,0.05937759876251221
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,65535,0.14009759426116944
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,32767,0.08657280206680298
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,31,0.014982399344444276
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,1,0.01838400065898895
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,3,0.018534399569034576
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,15,0.018751999735832213
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,511,0.049342399835586546
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,131071,0.24367198944091797
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,131071,2.6272207260131837
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,63,0.019038400053977965
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,127,0.023156799376010895
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,255,0.03123359978199005
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,1023,0.11721279621124267
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,2047,0.19641599655151368
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,4095,0.35800321102142335
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,8191,0.6850351810455322
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,16383,1.3263952255249023
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,1023,0.028265601396560668
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,1,1,128,1,float16,float16,3,0.026204800605773924
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,1,1,128,1,float16,float16,1,0.026153600215911864
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,1,1,128,1,float16,float16,7,0.026283198595047
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,1,1,128,1,float16,float16,31,0.026502400636672974
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,1,1,128,1,float16,float16,15,0.02619200050830841
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,1,1,128,1,float16,float16,63,0.026716798543930054
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,1,1,128,1,float16,float16,127,0.03339200019836426
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,1,1,128,1,float16,float16,255,0.04939039945602417
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,7,0.018529599905014037
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,1,1,128,1,float16,float16,511,0.10503360033035278
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,1,1,128,1,float16,float16,1023,0.1706431984901428
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,1,1,128,1,float16,float16,2047,0.2911952018737793
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,511,0.07437919974327087
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,1,1,128,1,float16,float16,1,0.038252800703048706
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,1,1,128,1,float16,float16,3,0.03830719888210297
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,1,1,128,1,float16,float16,4095,0.5311520099639893
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,1,1,128,1,float16,float16,7,0.038283199071884155
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,1,1,128,1,float16,float16,31,0.038764798641204835
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,1,1,128,1,float16,float16,15,0.03838239908218384
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,1,1,128,1,float16,float16,63,0.039417600631713866
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,1,1,128,1,float16,float16,127,0.05462719798088074
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,1,1,128,1,float16,float16,8191,1.0217040061950684
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,1,1,128,1,float16,float16,255,0.07617440223693847
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,1,1,128,1,float16,float16,511,0.1702623963356018
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,1,1,128,1,float16,float16,1023,0.28112640380859377
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,1,1,128,1,float16,float16,2047,0.48142237663269044
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,1,0.01403679996728897
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,7,0.014073599874973298
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,3,0.01417119950056076
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,15,0.01414719969034195
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,63,0.01419519931077957
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,1,1,128,1,float16,float16,16383,1.9969087600708009
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,31,0.014252799749374389
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,127,0.01674720048904419
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,1023,0.028703999519348145
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,511,0.024476799368858337
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,2047,0.029129600524902342
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,4095,0.03804959952831268
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,8191,0.05714079737663269
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,65535,5.218983840942383
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,31,0.018646399676799773
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,32767,0.13554079532623292
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,16383,0.08665760159492493
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,1,1,128,1,float16,float16,8191,1.7044448852539062
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,32767,2.6278831481933596
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,1,1,128,1,float16,float16,3,0.056390398740768434
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,1,1,128,1,float16,float16,1,0.05599200129508972
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,65535,0.24096159934997557
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,1,1,128,1,float16,float16,31,0.05719360113143921
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,131071,0.4397024154663086
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,1,1,128,1,float16,float16,7,0.05713919997215271
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,1,1,128,1,float16,float16,63,0.06403040289878845
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,1,1,128,1,float16,float16,15,0.05662879943847656
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,1,1,128,1,float16,float16,127,0.08532959818840027
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,1,1,128,1,float16,float16,255,0.12674880027770996
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,1,1,128,1,float16,float16,1,0.0940559983253479
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,1,1,128,1,float16,float16,511,0.316430401802063
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,1,1,128,1,float16,float16,3,0.09384480118751526
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,1,1,128,1,float16,float16,7,0.09430879950523377
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,1,1,128,1,float16,float16,15,0.09412159919738769
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,1,1,128,1,float16,float16,1023,0.5422959804534913
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,1,1,128,1,float16,float16,31,0.09798240065574645
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,1,1,128,1,float16,float16,63,0.10436320304870605
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,1,1,128,1,float16,float16,127,0.14345279932022095
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,1,1,128,1,float16,float16,2047,0.9331472396850586
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,1,1,128,1,float16,float16,255,0.22790079116821288
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,1,0.014326399564743042
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,1,1,128,1,float16,float16,511,0.5853151798248291
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,1,1,128,1,float16,float16,4095,0.8828847885131836
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,3,0.014299200475215912
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,31,0.0143327996134758
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,255,0.02168159931898117
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,7,0.014219200611114502
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,15,0.014158399403095245
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,1,1,128,1,float16,float16,4095,1.7508256912231446
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,127,0.01668799966573715
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,511,0.02489600032567978
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,2047,0.03782239854335785
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,1,1,128,1,float16,float16,1023,1.0089632034301759
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,1023,0.030136001110076905
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,4095,0.05627999901771545
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,8191,0.08519039750099182
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,16383,0.221726393699646
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,32767,0.41539840698242186
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,1,128,1,float16,float16,1,0.016553600132465363
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,1,1,128,1,float16,float16,2047,1.7855295181274413
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,1,128,1,float16,float16,3,0.017123199999332428
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,1,128,1,float16,float16,7,0.01650400012731552
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,65535,0.7754879951477051
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,1,128,1,float16,float16,31,0.017129600048065186
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,1,128,1,float16,float16,15,0.0169855996966362
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,1,128,1,float16,float16,127,0.019577600061893463
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,1,128,1,float16,float16,255,0.024588799476623534
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,1,128,1,float16,float16,511,0.06716639995574951
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,2,128,1,float16,float16,1,0.01849440038204193
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,131071,1.4969696044921874
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,1,128,1,float16,float16,2047,0.130459201335907
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,2,128,1,float16,float16,3,0.018329599499702455
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,2,128,1,float16,float16,7,0.01801439970731735
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,2,128,1,float16,float16,31,0.01863359957933426
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,2,128,1,float16,float16,15,0.0185248002409935
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,2,128,1,float16,float16,63,0.018542400002479552
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,2,128,1,float16,float16,127,0.02298080027103424
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,2,128,1,float16,float16,255,0.03190079927444458
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,2,128,1,float16,float16,511,0.0752240002155304
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,63,0.014433600008487701
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,2,128,1,float16,float16,1023,0.11969599723815919
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,4,128,1,float16,float16,1,0.02393440008163452
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,2,128,1,float16,float16,2047,0.20069119930267335
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,255,0.021782399713993074
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,4,128,1,float16,float16,3,0.023871999979019166
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,4,128,1,float16,float16,7,0.024167999625205994
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,4,128,1,float16,float16,15,0.02407519966363907
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,4,128,1,float16,float16,63,0.024540799856185912
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,4,128,1,float16,float16,31,0.024563199281692503
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,4,128,1,float16,float16,127,0.03138239979743958
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,4,128,1,float16,float16,511,0.10269919633865357
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,8,128,1,float16,float16,1,0.031497600674629214
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,4,128,1,float16,float16,1023,0.1676095962524414
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,4,128,1,float16,float16,255,0.05029600262641907
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,8,128,1,float16,float16,7,0.03153280019760132
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,4,128,1,float16,float16,2047,0.29072160720825196
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,8,128,1,float16,float16,15,0.031574401259422305
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,8,128,1,float16,float16,31,0.032313600182533264
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,8,128,1,float16,float16,63,0.033102399110794066
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,8,128,1,float16,float16,255,0.07131360173225403
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,8,128,1,float16,float16,511,0.16191359758377075
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,1,128,1,float16,float16,63,0.016726399958133697
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,1,128,1,float16,float16,3,0.014435200393199921
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,1,128,1,float16,float16,1,0.014110399782657624
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,8,128,1,float16,float16,1023,0.270086407661438
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,1,128,1,float16,float16,7,0.01449120044708252
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,1,128,1,float16,float16,15,0.014236800372600555
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,1,128,1,float16,float16,63,0.014636799693107605
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,1,128,1,float16,float16,31,0.01472959965467453
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,1,128,1,float16,float16,127,0.017028799653053282
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,1,128,1,float16,float16,255,0.02215680032968521
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,1,128,1,float16,float16,1023,0.08918880224227906
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,1,128,1,float16,float16,511,0.025016000866889952
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,1,128,1,float16,float16,2047,0.029655998945236205
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,2,128,1,float16,float16,1,0.014137600362300873
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,2,128,1,float16,float16,3,0.014473600685596466
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,2,128,1,float16,float16,7,0.014179199934005737
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,2,128,1,float16,float16,15,0.014233599603176116
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,2,128,1,float16,float16,63,0.014343999326229095
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,2,128,1,float16,float16,31,0.014620800316333771
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,2,128,1,float16,float16,127,0.01685599982738495
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,2,128,1,float16,float16,255,0.021793599426746368
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,2,128,1,float16,float16,511,0.023801599442958832
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,2,128,1,float16,float16,1023,0.027768000960350037
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,2,128,1,float16,float16,2047,0.028300800919532777
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,4,128,1,float16,float16,1,0.014273600280284881
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,4,128,1,float16,float16,3,0.01401599943637848
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,4,128,1,float16,float16,7,0.01406719982624054
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,4,128,1,float16,float16,15,0.0141184002161026
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,4,128,1,float16,float16,31,0.014060799777507783
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,4,128,1,float16,float16,63,0.014142400026321411
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,4,128,1,float16,float16,127,0.01695840060710907
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,4,128,1,float16,float16,255,0.021512000262737273
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,4,128,1,float16,float16,511,0.023019200563430785
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,4,128,1,float16,float16,1023,0.027398398518562316
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,4,128,1,float16,float16,2047,0.02805759906768799
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,8,128,1,float16,float16,1,0.014070400595664978
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,8,128,1,float16,float16,3,0.01401119977235794
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,8,128,1,float16,float16,7,0.01406400054693222
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,8,128,1,float16,float16,15,0.014086399972438813
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,8,128,1,float16,float16,31,0.014137600362300873
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,8,128,1,float16,float16,63,0.01427839994430542
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,8,128,1,float16,float16,127,0.01684480011463165
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,8,128,1,float16,float16,255,0.021687999367713928
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,8,128,1,float16,float16,511,0.023155200481414794
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,8,128,1,float16,float16,1023,0.027561599016189577
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,8,128,1,float16,float16,2047,0.02872479856014252
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,1,128,1,float16,float16,1,0.014604799449443817
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,1,128,1,float16,float16,3,0.014590400457382201
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,1,128,1,float16,float16,7,0.014577600359916686
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,1,128,1,float16,float16,15,0.014595200121402741
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,1,128,1,float16,float16,31,0.014633600413799287
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,1,128,1,float16,float16,63,0.014750400185585022
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,1,128,1,float16,float16,127,0.017342400550842286
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,1,128,1,float16,float16,255,0.02213120013475418
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,8,128,1,float16,float16,3,0.03147040009498596
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,1,128,1,float16,float16,511,0.025116801261901855
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,1,128,1,float16,float16,1023,0.029241600632667543
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,1,128,1,float16,float16,2047,0.030024001002311708
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,2,128,1,float16,float16,1,0.014135999977588654
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,2,128,1,float16,float16,3,0.014044800400733947
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,2,128,1,float16,float16,15,0.014227199554443359
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,2,128,1,float16,float16,7,0.014297600090503692
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,8,128,1,float16,float16,127,0.05052319765090942
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,2,128,1,float16,float16,63,0.014176000654697419
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,2,128,1,float16,float16,31,0.014271999895572662
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,2,128,1,float16,float16,255,0.021700799465179443
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,2,128,1,float16,float16,511,0.023787200450897217
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,4,128,1,float16,float16,1,0.01401280015707016
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,2,128,1,float16,float16,2047,0.028790399432182312
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,4,128,1,float16,float16,3,0.013940800726413728
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,4,128,1,float16,float16,7,0.013977600634098053
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,4,128,1,float16,float16,15,0.014310400187969207
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,4,128,1,float16,float16,31,0.01406719982624054
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,4,128,1,float16,float16,63,0.014137600362300873
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,4,128,1,float16,float16,127,0.016862399876117706
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,4,128,1,float16,float16,255,0.021558399498462676
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,4,128,1,float16,float16,511,0.023262399435043334
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,4,128,1,float16,float16,1023,0.028040000796318056
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,128,8,128,1,float16,float16,2047,0.46593599319458007
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,4,128,1,float16,float16,2047,0.028916800022125246
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,8,128,1,float16,float16,1,0.014103999733924866
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,8,128,1,float16,float16,3,0.014299200475215912
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,8,128,1,float16,float16,31,0.014159999787807465
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,8,128,1,float16,float16,15,0.014116799831390381
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,8,128,1,float16,float16,63,0.01446239948272705
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,8,128,1,float16,float16,7,0.014209599792957306
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,8,128,1,float16,float16,127,0.016816000640392303
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,8,128,1,float16,float16,255,0.021715199947357176
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,8,128,1,float16,float16,2047,0.03728640079498291
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,1,128,1,float16,float16,1,0.015430399775505066
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,8,128,1,float16,float16,1023,0.028227201104164122
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,1,128,1,float16,float16,3,0.015611200034618378
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,1,128,1,float16,float16,15,0.01563200056552887
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,1,128,1,float16,float16,31,0.01568479984998703
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,1,128,1,float16,float16,127,0.018323199450969697
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,1,128,1,float16,float16,63,0.015596799552440643
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,1,128,1,float16,float16,255,0.023192000389099122
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,1,128,1,float16,float16,1023,0.04631359875202179
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,1,128,1,float16,float16,511,0.035604798793792726
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,2,128,1,float16,float16,1,0.015406399965286255
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,1,128,1,float16,float16,2047,0.06280159950256348
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,2,128,1,float16,float16,3,0.015758399665355683
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,2,128,1,float16,float16,15,0.015385599434375763
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,2,128,1,float16,float16,31,0.015532800555229187
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,2,128,1,float16,float16,63,0.01555519998073578
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,2,128,1,float16,float16,127,0.01825280040502548
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,2,128,1,float16,float16,255,0.02292319983243942
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,2,128,1,float16,float16,511,0.052236801385879515
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,2,128,1,float16,float16,1023,0.07727680206298829
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,2,128,1,float16,float16,127,0.016910399496555328
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,2,128,1,float16,float16,2047,0.11705759763717652
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,4,128,1,float16,float16,1,0.016969600319862367
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,4,128,1,float16,float16,3,0.016995200514793397
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,4,128,1,float16,float16,7,0.01711200028657913
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,4,128,1,float16,float16,15,0.017150400578975676
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,4,128,1,float16,float16,31,0.017265599966049195
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,4,128,1,float16,float16,63,0.01748320013284683
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,4,128,1,float16,float16,127,0.02174399942159653
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,4,128,1,float16,float16,255,0.03020159900188446
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,4,128,1,float16,float16,511,0.07119359970092773
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,128,1,128,1,float16,float16,1023,0.029337599873542786
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,4,128,1,float16,float16,1023,0.11482720375061035
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,8,128,1,float16,float16,1,0.022840000689029694
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,8,128,1,float16,float16,511,0.024161599576473236
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,8,128,1,float16,float16,3,0.023030400276184082
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,8,128,1,float16,float16,15,0.023211200535297394
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,1,128,1,float16,float16,7,0.015572799742221833
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,8,128,1,float16,float16,31,0.023336000740528107
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,8,128,1,float16,float16,63,0.02359199970960617
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,8,128,1,float16,float16,127,0.02970399856567383
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,8,128,1,float16,float16,255,0.04657280147075653
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,8,128,1,float16,float16,511,0.09970719814300537
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,8,128,1,float16,float16,1023,0.16541600227355957
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,1,128,1,float16,float16,1,0.014532800018787383
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,1,128,1,float16,float16,3,0.014550399780273438
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,8,128,1,float16,float16,2047,0.28474879264831543
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,1,128,1,float16,float16,7,0.01441120058298111
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,2,128,1,float16,float16,7,0.015196800231933594
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,1,128,1,float16,float16,31,0.014972800016403198
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,1,128,1,float16,float16,63,0.01462559998035431
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,1,128,1,float16,float16,127,0.01720159947872162
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,1,128,1,float16,float16,255,0.02205599993467331
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,1,128,1,float16,float16,511,0.025198400020599365
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,1,128,1,float16,float16,1023,0.029532799124717714
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,1,128,1,float16,float16,2047,0.03051519989967346
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,2,128,1,float16,float16,1,0.014214399456977844
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,2,128,1,float16,float16,3,0.01414559930562973
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,2,128,1,float16,float16,7,0.014211200177669525
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,2,128,1,float16,float16,15,0.01417279988527298
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,2,128,1,float16,float16,31,0.014246399700641631
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,2,128,1,float16,float16,63,0.014241600036621093
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,2,128,1,float16,float16,127,0.016952000558376312
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,2,128,1,float16,float16,255,0.0217631995677948
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,2,128,1,float16,float16,511,0.024271999299526215
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,2,128,1,float16,float16,1023,0.028615999221801757
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,2,128,1,float16,float16,2047,0.029849600791931153
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,4,128,1,float16,float16,1,0.014201599359512328
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,4,128,1,float16,float16,3,0.014396800100803376
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,4,128,1,float16,float16,7,0.014256000518798828
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,128,2,128,1,float16,float16,1023,0.028119999170303344
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,4,128,1,float16,float16,15,0.014419199526309967
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,4,128,1,float16,float16,31,0.014243200421333313
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,4,128,1,float16,float16,63,0.01424960047006607
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,4,128,1,float16,float16,127,0.017003199458122252
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,4,128,1,float16,float16,255,0.02176000028848648
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,4,128,1,float16,float16,511,0.024195200204849242
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,4,128,1,float16,float16,1023,0.028700798749923706
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,4,128,1,float16,float16,2047,0.037503999471664426
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,8,128,1,float16,float16,3,0.014444799721240997
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,8,128,1,float16,float16,7,0.01430879980325699
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,8,128,1,float16,float16,15,0.014183999598026275
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,8,128,1,float16,float16,31,0.014283199608325959
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,8,128,1,float16,float16,63,0.014393599331378936
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,8,128,1,float16,float16,127,0.016939200460910797
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,8,128,1,float16,float16,255,0.02178719937801361
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,8,128,1,float16,float16,7,0.022891199588775633
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,128,4,128,1,float16,float16,2047,0.19527679681777954
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,8,128,1,float16,float16,511,0.028696000576019287
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,8,128,1,float16,float16,1023,0.037982401251792905
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,1,128,1,float16,float16,3,0.020252799987792967
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,8,128,1,float16,float16,2047,0.0545199990272522
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,1,128,1,float16,float16,1,0.020262399315834047
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,1,128,1,float16,float16,7,0.020265600085258482
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,1,128,1,float16,float16,15,0.02037599980831146
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,1,128,1,float16,float16,31,0.020451200008392335
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,1,128,1,float16,float16,63,0.02091040015220642
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,1,128,1,float16,float16,127,0.02481440007686615
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,1,128,1,float16,float16,255,0.03375999927520752
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,2,128,1,float16,float16,1,0.02590239942073822
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,1,128,1,float16,float16,511,0.07887359857559204
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,2,128,1,float16,float16,3,0.025950399041175843
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,1,128,1,float16,float16,1023,0.12222080230712891
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,2,128,1,float16,float16,7,0.02617120146751404
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,2,128,1,float16,float16,31,0.026212799549102783
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,2,128,1,float16,float16,15,0.026225599646568298
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,2,128,1,float16,float16,63,0.026473599672317504
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,2,128,1,float16,float16,127,0.03348000049591064
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,2,128,1,float16,float16,255,0.05764639973640442
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,4,128,1,float16,float16,1,0.03286879956722259
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,2,128,1,float16,float16,511,0.10677119493484497
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,4,128,1,float16,float16,3,0.032918399572372435
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,4,128,1,float16,float16,15,0.03309119939804077
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,4,128,1,float16,float16,7,0.033032000064849854
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,2,128,1,float16,float16,1023,0.17172640562057495
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,4,128,1,float16,float16,31,0.03335520029067993
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,4,128,1,float16,float16,63,0.03513759970664978
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,4,128,1,float16,float16,127,0.052455997467041014
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,4,128,1,float16,float16,255,0.07593280076980591
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,4,128,1,float16,float16,511,0.16299359798431395
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,8,128,1,float16,float16,1,0.050361597537994386
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,8,128,1,float16,float16,3,0.050423997640609744
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,8,128,1,float16,float16,7,0.050625598430633544
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,8,128,1,float16,float16,31,0.05198079943656921
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,4,128,1,float16,float16,1023,0.27211360931396483
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,8,128,1,float16,float16,15,0.05086399912834168
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,8,128,1,float16,float16,63,0.06111680269241333
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,8,128,1,float16,float16,127,0.08124799728393554
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,8,128,1,float16,float16,255,0.12401440143585205
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,128,1,128,1,float16,float16,1,0.03044160008430481
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,128,1,128,1,float16,float16,3,0.030921599268913268
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,8,128,1,float16,float16,1,0.014239999651908874
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,8,128,1,float16,float16,511,0.299068808555603
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,128,1,128,1,float16,float16,15,0.03094879984855652
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,128,1,128,1,float16,float16,7,0.030640000104904176
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,128,1,128,1,float16,float16,31,0.030963200330734252
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,128,1,128,1,float16,float16,63,0.032292801141738894
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,128,8,128,1,float16,float16,1023,0.5210847854614258
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,128,1,128,1,float16,float16,127,0.04683519899845123
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,128,2,128,1,float16,float16,1,0.035897600650787356
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,128,1,128,1,float16,float16,255,0.06527040004730225
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,128,2,128,1,float16,float16,3,0.037003201246261594
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,128,2,128,1,float16,float16,7,0.03652800023555756
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,128,2,128,1,float16,float16,31,0.036955198645591734
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,128,2,128,1,float16,float16,127,0.05876799821853638
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,128,2,128,1,float16,float16,15,0.036668801307678224
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,128,2,128,1,float16,float16,255,0.08306879997253418
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,128,4,128,1,float16,float16,1,0.05264319777488709
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,128,4,128,1,float16,float16,3,0.05298240184783935
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,128,4,128,1,float16,float16,7,0.05366560220718384
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,128,4,128,1,float16,float16,15,0.053939199447631835
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,128,4,128,1,float16,float16,31,0.05735359787940979
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,128,4,128,1,float16,float16,63,0.06582239866256714
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,128,4,128,1,float16,float16,127,0.08723840117454529
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,128,8,128,1,float16,float16,1,0.08944640159606934
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,128,8,128,1,float16,float16,3,0.0895247995853424
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,128,4,128,1,float16,float16,255,0.12964160442352296
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,128,1,128,1,float16,float16,15,0.01467519998550415
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,128,8,128,1,float16,float16,7,0.0906112015247345
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,128,8,128,1,float16,float16,31,0.09746080040931701
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,128,8,128,1,float16,float16,63,0.10359519720077515
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,128,8,128,1,float16,float16,127,0.1438639998435974
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,128,1,128,1,float16,float16,1,0.05462880134582519
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,128,8,128,1,float16,float16,255,0.22820799350738524
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,128,1,128,1,float16,float16,15,0.05631999969482422
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,128,1,128,1,float16,float16,3,0.05528479814529419
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,128,1,128,1,float16,float16,7,0.05731520056724548
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,128,1,128,1,float16,float16,31,0.05878239870071411
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,128,1,128,1,float16,float16,63,0.06142079830169678
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,128,1,128,1,float16,float16,127,0.07349280118942261
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,128,2,128,1,float16,float16,3,0.06948320269584655
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,128,2,128,1,float16,float16,7,0.06873599886894226
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,128,2,128,1,float16,float16,15,0.07084159851074219
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,128,2,128,1,float16,float16,31,0.07196159958839417
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,128,2,128,1,float16,float16,63,0.07538080215454102
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,128,4,128,1,float16,float16,1,0.10183839797973633
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,128,4,128,1,float16,float16,3,0.10226399898529052
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,128,2,128,1,float16,float16,127,0.09767040014266967
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,128,4,128,1,float16,float16,7,0.10382239818572998
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,128,4,128,1,float16,float16,15,0.10477919578552246
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,128,4,128,1,float16,float16,31,0.10619839429855346
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,128,4,128,1,float16,float16,63,0.112609601020813
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,128,4,128,1,float16,float16,127,0.15374239683151245
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,128,2,128,1,float16,float16,63,0.04687359929084778
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,128,8,128,1,float16,float16,1,0.17221920490264891
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,128,8,128,1,float16,float16,3,0.1728783965110779
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,128,8,128,1,float16,float16,7,0.17498559951782228
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,128,8,128,1,float16,float16,15,0.1764224052429199
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,1,128,1,float16,float16,1,0.01470080018043518
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,128,8,128,1,float16,float16,31,0.17900320291519164
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,1,128,1,float16,float16,7,0.014689600467681885
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,1,128,1,float16,float16,3,0.01449279934167862
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,128,8,128,1,float16,float16,63,0.1890720009803772
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,1,128,1,float16,float16,31,0.01470080018043518
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,1,128,1,float16,float16,63,0.01462399959564209
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,1,128,1,float16,float16,255,0.02236640006303787
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,1,128,1,float16,float16,511,0.02645600140094757
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,1,128,1,float16,float16,127,0.017343999445438386
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,1,128,1,float16,float16,1023,0.03095200061798096
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,1,128,1,float16,float16,2047,0.03227039873600006
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,128,8,128,1,float16,float16,15,0.09255840182304383
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,2,128,1,float16,float16,3,0.014632000029087067
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,2,128,1,float16,float16,7,0.01422239989042282
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,2,128,1,float16,float16,31,0.014694400131702423
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,2,128,1,float16,float16,63,0.014435200393199921
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,2,128,1,float16,float16,15,0.01419679969549179
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,2,128,1,float16,float16,127,0.01736160069704056
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,2,128,1,float16,float16,255,0.02178560048341751
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,2,128,1,float16,float16,511,0.025846400856971742
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,2,128,1,float16,float16,1023,0.030059200525283814
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,2,128,1,float16,float16,2047,0.038903999328613284
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,4,128,1,float16,float16,1,0.014235199987888336
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,4,128,1,float16,float16,3,0.01446239948272705
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,4,128,1,float16,float16,15,0.014371199905872345
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,4,128,1,float16,float16,7,0.014374400675296783
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,4,128,1,float16,float16,31,0.014620800316333771
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,4,128,1,float16,float16,63,0.01438560038805008
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,4,128,1,float16,float16,127,0.017097599804401398
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,4,128,1,float16,float16,255,0.02221280038356781
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,4,128,1,float16,float16,511,0.02924480140209198
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,4,128,1,float16,float16,1023,0.03871519863605499
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,8,128,1,float16,float16,1,0.014315199851989747
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,4,128,1,float16,float16,2047,0.05628960132598877
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,8,128,1,float16,float16,3,0.014510400593280792
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,8,128,1,float16,float16,7,0.01443679928779602
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,8,128,1,float16,float16,15,0.014720000326633453
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,8,128,1,float16,float16,31,0.014601600170135499
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,8,128,1,float16,float16,63,0.014481599628925323
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,8,128,1,float16,float16,127,0.017395199835300447
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,8,128,1,float16,float16,255,0.022176000475883483
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,8,128,1,float16,float16,511,0.04635359942913055
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,8,128,1,float16,float16,1023,0.07147679924964905
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,8,128,1,float16,float16,2047,0.11200480461120606
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,128,1,128,1,float16,float16,1,0.08102560043334961
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,128,1,128,1,float16,float16,3,0.08237919807434083
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,128,1,128,1,float16,float16,7,0.08365600109100342
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,128,1,128,1,float16,float16,15,0.08497920036315917
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,128,1,128,1,float16,float16,31,0.0874127984046936
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,128,1,128,1,float16,float16,63,0.09528319835662842
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,128,2,128,1,float16,float16,1,0.11042560338973999
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,128,2,128,1,float16,float16,3,0.11352000236511231
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,128,2,128,1,float16,float16,7,0.11391999721527099
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,128,2,128,1,float16,float16,15,0.11393439769744873
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,128,2,128,1,float16,float16,31,0.11738719940185546
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,128,2,128,1,float16,float16,63,0.12209119796752929
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,1,128,1,float16,float16,15,0.014449599385261535
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,128,8,128,1,float16,float16,127,0.2707792043685913
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,128,4,128,1,float16,float16,1,0.18002079725265502
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,128,4,128,1,float16,float16,3,0.179803204536438
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,128,4,128,1,float16,float16,7,0.1825760006904602
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,128,4,128,1,float16,float16,15,0.18523679971694945
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,128,2,128,1,float16,float16,1,0.0143327996134758
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,128,4,128,1,float16,float16,31,0.18731199502944945
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,128,4,128,1,float16,float16,63,0.19739359617233276
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,128,8,128,1,float16,float16,1,0.3213279962539673
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,128,8,128,1,float16,float16,3,0.3222912073135376
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,128,8,128,1,float16,float16,7,0.3247663974761963
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,128,8,128,1,float16,float16,15,0.32991039752960205
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,128,8,128,1,float16,float16,31,0.33639039993286135
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,128,1,128,1,float16,float16,1,0.13126879930496216
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,128,8,128,1,float16,float16,63,0.35129599571228026
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,128,2,128,1,float16,float16,1,0.06729120016098022
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,128,1,128,1,float16,float16,15,0.13650879859924317
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,128,1,128,1,float16,float16,31,0.141212797164917
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,128,2,128,1,float16,float16,1,0.19297280311584472
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,128,2,128,1,float16,float16,7,0.19822560548782348
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,128,2,128,1,float16,float16,3,0.19692480564117432
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,128,2,128,1,float16,float16,15,0.1998047947883606
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,128,2,128,1,float16,float16,31,0.20459680557250975
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,128,4,128,1,float16,float16,1,0.33334400653839114
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,128,4,128,1,float16,float16,3,0.33631999492645265
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,128,4,128,1,float16,float16,7,0.33854079246520996
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,128,4,128,1,float16,float16,15,0.3441231966018677
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,128,4,128,1,float16,float16,31,0.3501375913619995
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,128,8,128,1,float16,float16,1,0.6193232059478759
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,128,8,128,1,float16,float16,3,0.623196792602539
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,128,8,128,1,float16,float16,7,0.6286736011505127
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,1,128,1,float16,float16,1,0.014753599464893342
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,1,128,1,float16,float16,3,0.014859199523925781
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,128,8,128,1,float16,float16,15,0.6362127780914306
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,1,128,1,float16,float16,15,0.014847999811172486
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,1,128,1,float16,float16,31,0.014934399724006652
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,128,8,128,1,float16,float16,31,0.6497007846832276
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,1,128,1,float16,float16,127,0.017473599314689635
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,1,128,1,float16,float16,255,0.02253919988870621
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,1,128,1,float16,float16,63,0.015031999349594117
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,1,128,1,float16,float16,1023,0.03247840106487274
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,1,128,1,float16,float16,511,0.028537601232528687
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,1,128,1,float16,float16,2047,0.04301919937133789
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,2,128,1,float16,float16,1,0.014484800398349762
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,2,128,1,float16,float16,3,0.014571200311183929
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,2,128,1,float16,float16,15,0.014641599357128143
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,2,128,1,float16,float16,31,0.014632000029087067
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,2,128,1,float16,float16,63,0.01465280055999756
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,2,128,1,float16,float16,127,0.017318400740623473
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,2,128,1,float16,float16,255,0.02210559993982315
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,2,128,1,float16,float16,511,0.031017601490020752
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,2,128,1,float16,float16,1023,0.04070239961147308
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,4,128,1,float16,float16,1,0.014511999487876893
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,2,128,1,float16,float16,2047,0.05752320289611816
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,4,128,1,float16,float16,3,0.014511999487876893
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,4,128,1,float16,float16,15,0.014561599493026734
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,4,128,1,float16,float16,31,0.014636799693107605
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,4,128,1,float16,float16,63,0.014711999893188476
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,4,128,1,float16,float16,127,0.017390400171279907
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,128,1,128,1,float16,float16,7,0.1343775987625122
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,4,128,1,float16,float16,255,0.02221119999885559
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,128,1,128,1,float16,float16,3,0.13182239532470702
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,4,128,1,float16,float16,511,0.048363199830055235
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,4,128,1,float16,float16,1023,0.072188800573349
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,8,128,1,float16,float16,1,0.016415999829769136
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,8,128,1,float16,float16,3,0.016385599970817566
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,8,128,1,float16,float16,15,0.016448000073432924
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,8,128,1,float16,float16,31,0.016568000614643096
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,8,128,1,float16,float16,63,0.016953599452972413
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,8,128,1,float16,float16,127,0.02122880071401596
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,8,128,1,float16,float16,255,0.02942720055580139
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,8,128,1,float16,float16,511,0.07017920017242432
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,8,128,1,float16,float16,1023,0.11363199949264527
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,1,128,1,float16,float16,1,0.015993599593639374
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,8,128,1,float16,float16,2047,0.19371039867401124
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,1,128,1,float16,float16,3,0.01605760008096695
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,1,128,1,float16,float16,7,0.016388800740242005
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,1,128,1,float16,float16,15,0.016087999939918517
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,1,128,1,float16,float16,31,0.016448000073432924
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,1,128,1,float16,float16,63,0.016281600296497344
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,1,128,1,float16,float16,127,0.019116799533367156
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,1,128,1,float16,float16,255,0.02374400049448013
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,1,128,1,float16,float16,7,0.014683200418949128
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,1,128,1,float16,float16,511,0.06570559740066528
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,2,128,1,float16,float16,1,0.01775519996881485
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,2,128,1,float16,float16,3,0.01793919950723648
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,1,128,1,float16,float16,1023,0.08404800295829773
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,2,128,1,float16,float16,7,0.017903999984264375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,2,128,1,float16,float16,15,0.017985600233078002
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,1,128,1,float16,float16,2047,0.12392959594726563
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,2,128,1,float16,float16,31,0.017927999794483184
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,2,128,1,float16,float16,63,0.018292799592018127
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,2,128,1,float16,float16,127,0.022551999986171724
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,2,128,1,float16,float16,7,0.014694400131702423
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,2,128,1,float16,float16,255,0.03086079955101013
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,2,128,1,float16,float16,511,0.07301599979400634
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,4,128,1,float16,float16,1,0.023907199501991272
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,2,128,1,float16,float16,1023,0.117249596118927
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,4,128,1,float16,float16,3,0.02374400049448013
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,4,128,1,float16,float16,7,0.02396959960460663
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,4,128,1,float16,float16,31,0.02385119944810867
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,4,128,1,float16,float16,7,0.014504000544548035
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,4,128,1,float16,float16,15,0.023948800563812257
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,4,128,1,float16,float16,63,0.02459840029478073
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,4,128,1,float16,float16,127,0.030670401453971863
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,4,128,1,float16,float16,255,0.04813440144062042
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,4,128,1,float16,float16,511,0.10198080539703369
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,8,128,1,float16,float16,1,0.03115200102329254
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,4,128,1,float16,float16,1023,0.1673472046852112
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,4,128,1,float16,float16,2047,0.1133072018623352
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,8,128,1,float16,float16,7,0.031174400448799135
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,8,128,1,float16,float16,3,0.03140319883823395
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,128,8,128,1,float16,float16,7,0.016361600160598753
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,4,128,1,float16,float16,2047,0.28646719455718994
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,8,128,1,float16,float16,63,0.03288320004940033
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,8,128,1,float16,float16,15,0.031718400120735166
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,8,128,1,float16,float16,127,0.048816001415252684
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,8,128,1,float16,float16,255,0.07043520212173462
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,8,128,1,float16,float16,511,0.15899039506912233
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,1,128,1,float16,float16,1,0.014254400134086609
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,1,128,1,float16,float16,3,0.014144000411033631
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,1,128,1,float16,float16,15,0.014164799451828003
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,8,128,1,float16,float16,1023,0.26785280704498293
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,1,128,1,float16,float16,7,0.014286400377750396
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,1,128,1,float16,float16,31,0.014252799749374389
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,1,128,1,float16,float16,63,0.014262400567531586
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,1,128,1,float16,float16,511,0.023979200422763823
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,1,128,1,float16,float16,255,0.021718400716781616
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,1,128,1,float16,float16,1023,0.028244799375534056
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,1,128,1,float16,float16,127,0.01687680035829544
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,8,128,1,float16,float16,2047,0.4676623821258545
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,2,128,1,float16,float16,1,0.014044800400733947
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,1,128,1,float16,float16,2047,0.029025599360466003
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,2,128,1,float16,float16,3,0.01393119990825653
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,2,128,1,float16,float16,7,0.014120000600814819
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,2,128,1,float16,float16,31,0.014168000221252442
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,2,128,1,float16,float16,63,0.014375999569892883
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,2,128,1,float16,float16,127,0.016708800196647645
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,2,128,1,float16,float16,255,0.021678400039672852
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,2,128,1,float16,float16,511,0.02333119958639145
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,2,128,1,float16,float16,1023,0.027451199293136597
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,4,128,1,float16,float16,1,0.013998399674892425
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,4,128,1,float16,float16,3,0.014043200016021728
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,4,128,1,float16,float16,7,0.014017599821090698
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,4,128,1,float16,float16,15,0.014108799397945404
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,4,128,1,float16,float16,31,0.01395999938249588
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,4,128,1,float16,float16,63,0.014187200367450714
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,4,128,1,float16,float16,127,0.016705599427223206
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,4,128,1,float16,float16,255,0.021452799439430237
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,4,128,1,float16,float16,511,0.022998400032520294
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,4,128,1,float16,float16,1023,0.027220800518989563
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,4,128,1,float16,float16,2047,0.02815360128879547
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,8,128,1,float16,float16,1,0.01401119977235794
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,8,128,1,float16,float16,3,0.014116799831390381
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,8,128,1,float16,float16,7,0.013887999951839447
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,8,128,1,float16,float16,15,0.014051200449466705
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,8,128,1,float16,float16,31,0.014073599874973298
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,8,128,1,float16,float16,63,0.013972799479961395
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,8,128,1,float16,float16,127,0.016731199622154237
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,8,128,1,float16,float16,255,0.021558399498462676
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,8,128,1,float16,float16,511,0.023068800568580627
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,8,128,1,float16,float16,1023,0.027665600180625916
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,8,128,1,float16,float16,2047,0.028652799129486085
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,1,128,1,float16,float16,1,0.014215999841690063
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,1,128,1,float16,float16,3,0.014312000572681427
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,1,128,1,float16,float16,7,0.014259199798107147
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,1,128,1,float16,float16,15,0.01433439999818802
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,1,128,1,float16,float16,31,0.014350399374961853
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,1,128,1,float16,float16,127,0.016948799788951873
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,1,128,1,float16,float16,255,0.021849599480628968
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,2,128,1,float16,float16,2047,0.19646559953689574
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,1,128,1,float16,float16,511,0.024425600469112397
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,1,128,1,float16,float16,1023,0.028491199016571045
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,2,128,1,float16,float16,1,0.014078399538993836
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,2,128,1,float16,float16,3,0.014187200367450714
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,1,128,1,float16,float16,2047,0.029232001304626463
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,2,128,1,float16,float16,15,0.014097599685192109
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,2,128,1,float16,float16,63,0.014207999408245086
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,2,128,1,float16,float16,31,0.014129599928855896
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,2,128,1,float16,float16,127,0.01675039976835251
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,2,128,1,float16,float16,255,0.02165919989347458
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,2,128,1,float16,float16,511,0.02336000055074692
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,2,128,1,float16,float16,1023,0.027982398867607117
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,2,128,1,float16,float16,2047,0.028431999683380126
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,4,128,1,float16,float16,1,0.01422400027513504
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,96,8,128,1,float16,float16,31,0.03171359896659851
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,4,128,1,float16,float16,3,0.014004799723625182
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,4,128,1,float16,float16,31,0.014049600064754485
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,4,128,1,float16,float16,15,0.013935999572277069
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,4,128,1,float16,float16,63,0.01422239989042282
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,4,128,1,float16,float16,127,0.01675360053777695
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,4,128,1,float16,float16,255,0.02162559926509857
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,4,128,1,float16,float16,511,0.023455999791622162
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,4,128,1,float16,float16,1023,0.02775680124759674
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,4,128,1,float16,float16,2047,0.02892960011959076
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,8,128,1,float16,float16,1,0.014220799505710601
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,8,128,1,float16,float16,3,0.01420000046491623
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,8,128,1,float16,float16,7,0.014262400567531586
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,8,128,1,float16,float16,15,0.014305600523948669
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,8,128,1,float16,float16,31,0.014254400134086609
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,2,128,1,float16,float16,15,0.01436000019311905
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,8,128,1,float16,float16,63,0.01438560038805008
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,96,2,128,1,float16,float16,2047,0.027977600693702698
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,8,128,1,float16,float16,127,0.016976000368595125
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,8,128,1,float16,float16,255,0.02179519981145859
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,8,128,1,float16,float16,511,0.02380799949169159
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,8,128,1,float16,float16,1023,0.028288000822067262
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,8,128,1,float16,float16,2047,0.03743839859962463
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,1,128,1,float16,float16,1,0.0152879998087883
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,1,128,1,float16,float16,3,0.015465599298477174
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,1,128,1,float16,float16,7,0.01517920047044754
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,1,128,1,float16,float16,15,0.01534239947795868
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,1,128,1,float16,float16,31,0.015539200603961944
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,1,128,1,float16,float16,63,0.01525920033454895
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,1,128,1,float16,float16,127,0.018027199804782866
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,1,128,1,float16,float16,255,0.02311840057373047
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,1,128,1,float16,float16,511,0.033272001147270205
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,1,128,1,float16,float16,1023,0.04384639859199524
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,2,128,1,float16,float16,1,0.015049600601196289
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,1,128,1,float16,float16,2047,0.060139197111129764
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,2,128,1,float16,float16,3,0.015238399803638458
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,2,128,1,float16,float16,7,0.015006400644779205
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,2,128,1,float16,float16,15,0.01549919992685318
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,2,128,1,float16,float16,31,0.015328000485897064
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,1,128,1,float16,float16,63,0.014441600441932679
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,2,128,1,float16,float16,63,0.015265600383281707
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,2,128,1,float16,float16,127,0.018294399976730345
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,2,128,1,float16,float16,255,0.022966399788856506
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,2,128,1,float16,float16,511,0.05062400102615357
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,2,128,1,float16,float16,1023,0.07597280144691468
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,4,128,1,float16,float16,1,0.017080000042915343
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,4,128,1,float16,float16,3,0.01698399931192398
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,4,128,1,float16,float16,15,0.017233599722385407
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,4,128,1,float16,float16,7,0.017217600345611574
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,2,128,1,float16,float16,2047,0.11610720157623292
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,4,128,1,float16,float16,31,0.017179200053215028
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,4,128,1,float16,float16,63,0.01757120043039322
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,4,128,1,float16,float16,127,0.021862399578094483
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,4,128,1,float16,float16,255,0.030273601412773132
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,4,128,1,float16,float16,511,0.0712112009525299
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,8,128,1,float16,float16,1,0.023107199370861052
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,4,128,1,float16,float16,1023,0.11499840021133423
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,4,128,1,float16,float16,2047,0.1953279972076416
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,8,128,1,float16,float16,7,0.023161600530147552
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,4,128,1,float16,float16,7,0.01422560065984726
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,8,128,1,float16,float16,15,0.02316959947347641
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,8,128,1,float16,float16,63,0.02377920001745224
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,8,128,1,float16,float16,127,0.029836800694465638
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,8,128,1,float16,float16,31,0.02330079972743988
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,8,128,1,float16,float16,255,0.04642719924449921
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,8,128,1,float16,float16,511,0.0993839979171753
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,1,128,1,float16,float16,1,0.014396800100803376
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,1,128,1,float16,float16,3,0.01435679942369461
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,8,128,1,float16,float16,1023,0.1652575969696045
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,1,128,1,float16,float16,15,0.014425599575042724
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,1,128,1,float16,float16,7,0.014473600685596466
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,8,128,1,float16,float16,2047,0.28480479717254636
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,1,128,1,float16,float16,63,0.014630399644374847
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,1,128,1,float16,float16,255,0.022019200026988983
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,1,128,1,float16,float16,2047,0.029972800612449647
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,1,128,1,float16,float16,1023,0.029097598791122437
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,2,128,1,float16,float16,1,0.01419679969549179
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,2,128,1,float16,float16,3,0.014419199526309967
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,2,128,1,float16,float16,7,0.014244799315929414
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,2,128,1,float16,float16,15,0.014228799939155578
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,2,128,1,float16,float16,31,0.014287999272346497
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,2,128,1,float16,float16,63,0.014643199741840363
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,2,128,1,float16,float16,127,0.016969600319862367
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,2,128,1,float16,float16,255,0.022012799978256226
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,2,128,1,float16,float16,511,0.0239439994096756
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,2,128,1,float16,float16,1023,0.028395199775695802
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,2,128,1,float16,float16,2047,0.029462400078773498
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,4,128,1,float16,float16,1,0.014347200095653535
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,4,128,1,float16,float16,3,0.014214399456977844
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,4,128,1,float16,float16,15,0.014252799749374389
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,4,128,1,float16,float16,31,0.014294399321079254
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,4,128,1,float16,float16,63,0.014414399862289429
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,4,128,1,float16,float16,127,0.016897599399089813
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,4,128,1,float16,float16,255,0.021695999801158904
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,4,128,1,float16,float16,511,0.02402399927377701
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,4,128,1,float16,float16,1023,0.028297600150108338
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,96,2,128,1,float16,float16,7,0.01408800035715103
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,4,128,1,float16,float16,2047,0.03709119856357575
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,8,128,1,float16,float16,1,0.01433439999818802
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,8,128,1,float16,float16,3,0.0142752006649971
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,8,128,1,float16,float16,7,0.014318400621414184
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,8,128,1,float16,float16,15,0.014259199798107147
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,8,128,1,float16,float16,31,0.014379200339317322
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,8,128,1,float16,float16,63,0.014419199526309967
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,8,128,1,float16,float16,127,0.016969600319862367
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,96,8,128,1,float16,float16,3,0.02280319929122925
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,8,128,1,float16,float16,255,0.02197919934988022
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,8,128,1,float16,float16,511,0.028596800565719605
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,8,128,1,float16,float16,1023,0.03784320056438446
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,1,128,1,float16,float16,1,0.019568000733852387
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,8,128,1,float16,float16,2047,0.05429440140724182
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,1,128,1,float16,float16,3,0.01971199959516525
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,1,128,1,float16,float16,15,0.019926400482654573
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,1,128,1,float16,float16,31,0.019921599328517912
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,1,128,1,float16,float16,31,0.01443679928779602
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,1,128,1,float16,float16,63,0.020057600736618043
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,1,128,1,float16,float16,127,0.017123199999332428
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,1,128,1,float16,float16,127,0.02431199997663498
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,1,128,1,float16,float16,255,0.03296000063419342
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,1,128,1,float16,float16,511,0.024940800666809083
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,1,128,1,float16,float16,511,0.07734400033950806
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,2,128,1,float16,float16,1,0.025310400128364562
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,1,128,1,float16,float16,1023,0.12057759761810302
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,2,128,1,float16,float16,3,0.025251200795173644
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,2,128,1,float16,float16,15,0.025668799877166748
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,2,128,1,float16,float16,7,0.025323200225830077
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,2,128,1,float16,float16,63,0.026070401072502136
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,2,128,1,float16,float16,31,0.02552480101585388
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,2,128,1,float16,float16,127,0.033036801218986514
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,2,128,1,float16,float16,255,0.052249598503112796
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,2,128,1,float16,float16,511,0.1054703950881958
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,4,128,1,float16,float16,1,0.03263840079307556
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,4,128,1,float16,float16,3,0.032652801275253295
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,2,128,1,float16,float16,1023,0.1698464035987854
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,96,4,128,1,float16,float16,7,0.014435200393199921
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,4,128,1,float16,float16,7,0.032743999361991884
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,4,128,1,float16,float16,31,0.03314720094203949
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,4,128,1,float16,float16,63,0.034790399670600894
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,4,128,1,float16,float16,127,0.05098080039024353
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,4,128,1,float16,float16,255,0.07369599938392639
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,8,128,1,float16,float16,1,0.050281602144241336
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,4,128,1,float16,float16,511,0.1617087960243225
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,8,128,1,float16,float16,3,0.05003520250320435
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,4,128,1,float16,float16,1023,0.27179999351501466
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,8,128,1,float16,float16,7,0.05046719908714294
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,8,128,1,float16,float16,15,0.05065280199050903
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,8,128,1,float16,float16,31,0.05166400074958801
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,8,128,1,float16,float16,63,0.059716802835464475
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,8,128,1,float16,float16,127,0.08180639743804932
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,8,128,1,float16,float16,255,0.12341120243072509
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,96,1,128,1,float16,float16,1,0.02858879864215851
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,96,1,128,1,float16,float16,3,0.02881760001182556
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,96,1,128,1,float16,float16,7,0.029147198796272276
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,96,1,128,1,float16,float16,15,0.028998398780822755
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,96,1,128,1,float16,float16,31,0.029124799370765685
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,1,128,1,float16,float16,7,0.019657599925994872
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,96,1,128,1,float16,float16,63,0.030088001489639284
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,8,128,1,float16,float16,1023,0.5205183982849121
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,96,1,128,1,float16,float16,127,0.03883680105209351
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,96,2,128,1,float16,float16,1,0.03561919927597046
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,96,2,128,1,float16,float16,3,0.035692799091339114
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,96,2,128,1,float16,float16,15,0.03582560122013092
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,96,2,128,1,float16,float16,31,0.03605599999427796
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,96,2,128,1,float16,float16,7,0.03550559878349304
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,96,2,128,1,float16,float16,63,0.042126399278640744
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,96,2,128,1,float16,float16,127,0.05584480166435242
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,96,4,128,1,float16,float16,1,0.052502399682998656
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,96,2,128,1,float16,float16,255,0.0794431984424591
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,96,4,128,1,float16,float16,3,0.052671998739242554
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,96,4,128,1,float16,float16,7,0.052779197692871094
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,96,4,128,1,float16,float16,15,0.053243201971054074
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,96,4,128,1,float16,float16,31,0.05482879877090454
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,96,4,128,1,float16,float16,63,0.06446239948272706
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,96,4,128,1,float16,float16,127,0.08496639728546143
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,96,8,128,1,float16,float16,1,0.08937919735908509
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,96,8,128,1,float16,float16,3,0.08967040181159973
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,96,4,128,1,float16,float16,255,0.12891680002212524
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,96,8,128,1,float16,float16,7,0.08973439931869506
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,96,8,128,1,float16,float16,15,0.09046880006790162
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,96,8,128,1,float16,float16,31,0.09593759775161743
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,96,8,128,1,float16,float16,63,0.10238239765167237
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,96,8,128,1,float16,float16,127,0.14186559915542601
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,96,1,128,1,float16,float16,1,0.044838398694992065
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,96,8,128,1,float16,float16,255,0.22627520561218262
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,96,1,128,1,float16,float16,3,0.04612640142440796
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,96,1,128,1,float16,float16,15,0.0466048002243042
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,96,1,128,1,float16,float16,7,0.04715839922428131
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,96,1,128,1,float16,float16,63,0.057571202516555786
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,96,1,128,1,float16,float16,127,0.06784800291061402
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,96,2,128,1,float16,float16,3,0.06107360124588013
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,96,2,128,1,float16,float16,7,0.062412798404693604
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,96,2,128,1,float16,float16,15,0.06350240111351013
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,96,2,128,1,float16,float16,31,0.06813120245933532
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,96,2,128,1,float16,float16,63,0.07256479859352112
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,96,2,128,1,float16,float16,127,0.09435999989509583
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,96,4,128,1,float16,float16,1,0.09767680168151856
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,8,128,1,float16,float16,511,0.29669599533081054
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,96,4,128,1,float16,float16,3,0.09874079823493957
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,96,4,128,1,float16,float16,7,0.10040960311889649
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,96,4,128,1,float16,float16,15,0.10352159738540649
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,96,4,128,1,float16,float16,31,0.1049504041671753
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,96,4,128,1,float16,float16,63,0.11033920049667359
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,96,1,128,1,float16,float16,255,0.061475199460983274
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,96,8,128,1,float16,float16,1,0.16943360567092897
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,96,4,128,1,float16,float16,127,0.1516047954559326
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,96,8,128,1,float16,float16,3,0.17029600143432616
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,96,8,128,1,float16,float16,7,0.17239359617233277
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,96,8,128,1,float16,float16,15,0.17577600479125977
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,96,8,128,1,float16,float16,63,0.18710880279541015
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,1,128,1,float16,float16,1,0.014611199498176575
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,1,128,1,float16,float16,7,0.014585599303245544
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,96,8,128,1,float16,float16,127,0.2672463893890381
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,1,128,1,float16,float16,15,0.014483200013637542
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,1,128,1,float16,float16,31,0.01459839940071106
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,1,128,1,float16,float16,63,0.0146479994058609
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,1,128,1,float16,float16,127,0.017156800627708434
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,1,128,1,float16,float16,255,0.022044800221920013
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,96,4,128,1,float16,float16,15,0.032892799377441405
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,1,128,1,float16,float16,1023,0.02970399856567383
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,1,128,1,float16,float16,511,0.024977600574493407
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,2,128,1,float16,float16,1,0.01435679942369461
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,2,128,1,float16,float16,3,0.014209599792957306
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,2,128,1,float16,float16,7,0.0141184002161026
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,2,128,1,float16,float16,15,0.014201599359512328
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,2,128,1,float16,float16,31,0.014241600036621093
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,2,128,1,float16,float16,63,0.014399999380111694
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,2,128,1,float16,float16,127,0.016915200650691985
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,2,128,1,float16,float16,255,0.02181279957294464
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,2,128,1,float16,float16,511,0.02449599951505661
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,96,1,128,1,float16,float16,31,0.051888000965118405
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,96,2,128,1,float16,float16,1,0.06041600108146668
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,2,128,1,float16,float16,1023,0.028918400406837463
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,2,128,1,float16,float16,2047,0.03812800049781799
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,4,128,1,float16,float16,1,0.014185599982738495
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,4,128,1,float16,float16,3,0.014193600416183472
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,4,128,1,float16,float16,15,0.014372800290584565
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,4,128,1,float16,float16,63,0.014374400675296783
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,4,128,1,float16,float16,127,0.01706079989671707
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,4,128,1,float16,float16,255,0.021854400634765625
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,4,128,1,float16,float16,511,0.028948798775672913
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,4,128,1,float16,float16,1023,0.0380623996257782
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,8,128,1,float16,float16,1,0.014273600280284881
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,4,128,1,float16,float16,2047,0.05472480058670044
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,8,128,1,float16,float16,3,0.014276799559593201
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,8,128,1,float16,float16,7,0.014424000680446625
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,8,128,1,float16,float16,15,0.014428800344467163
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,8,128,1,float16,float16,31,0.014630399644374847
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,8,128,1,float16,float16,63,0.014683200418949128
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,8,128,1,float16,float16,127,0.0174127995967865
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,8,128,1,float16,float16,255,0.022172799706459044
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,8,128,1,float16,float16,511,0.04646559953689575
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,8,128,1,float16,float16,1023,0.0711408019065857
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,8,128,1,float16,float16,2047,0.11125760078430176
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,96,1,128,1,float16,float16,1,0.07578240036964416
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,96,1,128,1,float16,float16,3,0.07553920149803162
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,96,1,128,1,float16,float16,7,0.07728639841079712
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,96,8,128,1,float16,float16,31,0.177183997631073
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,96,1,128,1,float16,float16,15,0.07656800150871276
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,1,128,1,float16,float16,3,0.014580799639225006
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,96,1,128,1,float16,float16,31,0.07856000065803528
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,96,2,128,1,float16,float16,1,0.10721759796142578
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,96,2,128,1,float16,float16,3,0.10848480463027954
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,96,2,128,1,float16,float16,7,0.10907039642333985
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,96,2,128,1,float16,float16,15,0.10981919765472412
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,96,2,128,1,float16,float16,31,0.11199840307235717
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,96,2,128,1,float16,float16,63,0.1170464038848877
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,1,128,1,float16,float16,2047,0.03081279993057251
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,96,4,128,1,float16,float16,1,0.17653919458389283
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,96,4,128,1,float16,float16,3,0.17817920446395874
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,96,4,128,1,float16,float16,7,0.17965279817581176
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,96,4,128,1,float16,float16,15,0.18155839443206787
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,96,4,128,1,float16,float16,31,0.18389439582824707
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,96,4,128,1,float16,float16,63,0.1946463942527771
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,96,8,128,1,float16,float16,1,0.3192528009414673
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,4,128,1,float16,float16,7,0.014315199851989747
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,96,8,128,1,float16,float16,7,0.32262558937072755
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,96,8,128,1,float16,float16,3,0.31967360973358155
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,96,4,128,1,float16,float16,31,0.014297600090503692
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,96,8,128,1,float16,float16,15,0.32811200618743896
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,96,8,128,1,float16,float16,31,0.33182721138000487
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,96,1,128,1,float16,float16,1,0.12151999473571777
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,96,1,128,1,float16,float16,3,0.12216960191726685
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,96,1,128,1,float16,float16,7,0.12428959608078002
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,96,1,128,1,float16,float16,15,0.12644000053405763
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,96,8,128,1,float16,float16,63,0.34789440631866453
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,96,1,128,1,float16,float16,31,0.12847039699554444
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,96,2,128,1,float16,float16,1,0.18662400245666505
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,96,2,128,1,float16,float16,3,0.18963999748229982
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,96,2,128,1,float16,float16,7,0.19166719913482666
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,96,2,128,1,float16,float16,15,0.1921776056289673
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,96,2,128,1,float16,float16,31,0.19644800424575806
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,96,4,128,1,float16,float16,1,0.3290127992630005
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,96,4,128,1,float16,float16,3,0.3312608003616333
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,96,4,128,1,float16,float16,7,0.3340687990188599
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,96,4,128,1,float16,float16,15,0.33903679847717283
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,96,4,128,1,float16,float16,31,0.34365599155426024
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,1,128,1,float16,float16,1,0.014564800262451171
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,96,8,128,1,float16,float16,1,0.6170032024383545
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,96,8,128,1,float16,float16,3,0.6170527935028076
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,96,1,128,1,float16,float16,63,0.08554080128669739
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,1,128,1,float16,float16,3,0.014553600549697876
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,1,128,1,float16,float16,7,0.014454400539398194
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,1,128,1,float16,float16,63,0.014812800288200378
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,96,8,128,1,float16,float16,7,0.6223328113555908
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,1,128,1,float16,float16,15,0.014484800398349762
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,1,128,1,float16,float16,31,0.014593599736690522
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,96,8,128,1,float16,float16,15,0.6292143821716308
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,1,128,1,float16,float16,255,0.02231999933719635
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,96,8,128,1,float16,float16,31,0.6392144203186035
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,1,128,1,float16,float16,1023,0.03107520043849945
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,1,128,1,float16,float16,511,0.026822400093078614
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,2,128,1,float16,float16,1,0.01446399986743927
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,2,128,1,float16,float16,3,0.014523200690746307
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,2,128,1,float16,float16,15,0.014519999921321868
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,2,128,1,float16,float16,7,0.014528000354766845
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,2,128,1,float16,float16,31,0.014550399780273438
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,2,128,1,float16,float16,63,0.014723199605941772
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,2,128,1,float16,float16,127,0.017211200296878816
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,2,128,1,float16,float16,511,0.030374398827552794
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,4,128,1,float16,float16,1,0.01451839953660965
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,2,128,1,float16,float16,2047,0.05620959997177124
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,4,128,1,float16,float16,3,0.014632000029087067
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,4,128,1,float16,float16,7,0.014531199634075165
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,4,128,1,float16,float16,31,0.014707200229167938
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,4,128,1,float16,float16,63,0.014747199416160584
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,4,128,1,float16,float16,127,0.017444799840450286
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,4,128,1,float16,float16,255,0.02229759991168976
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,4,128,1,float16,float16,511,0.04756639897823334
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,4,128,1,float16,float16,1023,0.07301920056343078
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,8,128,1,float16,float16,1,0.016195200383663177
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,4,128,1,float16,float16,2047,0.11203839778900146
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,8,128,1,float16,float16,3,0.016313600540161132
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,8,128,1,float16,float16,7,0.01652639955282211
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,8,128,1,float16,float16,31,0.016752000153064727
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,8,128,1,float16,float16,63,0.017105600237846373
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,8,128,1,float16,float16,127,0.021278400719165803
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,8,128,1,float16,float16,255,0.029465600848197937
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,8,128,1,float16,float16,511,0.06931359767913818
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,8,128,1,float16,float16,1023,0.11337759494781494
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,8,128,1,float16,float16,2047,0.19334080219268798
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,1,128,1,float16,float16,1,0.01594720035791397
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,1,128,1,float16,float16,127,0.017209599912166595
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,1,128,1,float16,float16,3,0.016014400124549865
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,1,128,1,float16,float16,7,0.015846399962902068
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,1,128,1,float16,float16,15,0.015982399880886077
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,1,128,1,float16,float16,31,0.015910400450229643
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,1,128,1,float16,float16,63,0.01622239947319031
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,1,128,1,float16,float16,2047,0.04035040140151978
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,1,128,1,float16,float16,127,0.018639999628067016
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,1,128,1,float16,float16,255,0.023790399730205535
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,1,128,1,float16,float16,511,0.053142398595809937
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,1,128,1,float16,float16,1023,0.07934719920158387
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,1,128,1,float16,float16,2047,0.11858880519866943
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,2,128,1,float16,float16,255,0.021958400309085847
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,2,128,1,float16,float16,1023,0.039761599898338315
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,2,128,1,float16,float16,1,0.01753759980201721
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,2,128,1,float16,float16,7,0.017718400061130523
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,2,128,1,float16,float16,3,0.017827199399471284
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,1,128,1,float16,float16,4095,0.1981104016304016
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,2,128,1,float16,float16,15,0.017841599881649017
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,4,128,1,float16,float16,15,0.014688000082969666
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,2,128,1,float16,float16,63,0.01818079948425293
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,2,128,1,float16,float16,127,0.022521600127220154
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,2,128,1,float16,float16,255,0.030750399827957152
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,2,128,1,float16,float16,511,0.07254719734191895
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,2,128,1,float16,float16,1023,0.1164944052696228
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,4,128,1,float16,float16,1,0.023603199422359465
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,2,128,1,float16,float16,2047,0.195249605178833
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,4,128,1,float16,float16,3,0.023676800727844238
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,4,128,1,float16,float16,15,0.023683199286460878
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,4,128,1,float16,float16,7,0.023713600635528565
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,4,128,1,float16,float16,63,0.024145600199699403
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,4,128,1,float16,float16,31,0.023793600499629974
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,2,128,1,float16,float16,4095,0.3565632104873657
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,4,128,1,float16,float16,127,0.03071039915084839
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,96,8,128,1,float16,float16,15,0.01639840006828308
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,4,128,1,float16,float16,255,0.04711039960384369
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,8,128,1,float16,float16,1,0.03110400140285492
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,4,128,1,float16,float16,511,0.10043840408325196
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,8,128,1,float16,float16,3,0.03120799958705902
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,4,128,1,float16,float16,1023,0.16667360067367554
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,8,128,1,float16,float16,7,0.03121280074119568
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,8,128,1,float16,float16,15,0.03135519921779632
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,4,128,1,float16,float16,2047,0.28700480461120603
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,8,128,1,float16,float16,31,0.031769600510597226
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,8,128,1,float16,float16,127,0.04745599925518036
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,8,128,1,float16,float16,63,0.03272640109062195
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,8,128,1,float16,float16,255,0.07018079757690429
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,4,128,1,float16,float16,4095,0.5260191917419433
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,1,128,1,float16,float16,1,0.013764800131320953
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,8,128,1,float16,float16,511,0.15810400247573853
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,8,128,1,float16,float16,1023,0.26831679344177245
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,1,128,1,float16,float16,7,0.013940800726413728
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,1,128,1,float16,float16,3,0.014027200639247894
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,1,128,1,float16,float16,15,0.014368000626564025
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,1,128,1,float16,float16,63,0.013998399674892425
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,1,128,1,float16,float16,127,0.016569599509239197
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,1,128,1,float16,float16,511,0.02332960069179535
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,8,128,1,float16,float16,2047,0.4672192096710205
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,1,128,1,float16,float16,1023,0.02754560112953186
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,1,128,1,float16,float16,2047,0.028355199098587035
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,2,128,1,float16,float16,3,0.014209599792957306
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,2,128,1,float16,float16,1,0.014068800210952758
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,2,128,1,float16,float16,7,0.01419840008020401
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,2,128,1,float16,float16,15,0.01403679996728897
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,2,128,1,float16,float16,31,0.014262400567531586
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,8,128,1,float16,float16,4095,0.8648624420166016
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,2,128,1,float16,float16,63,0.014131200313568116
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,2,128,1,float16,float16,127,0.016806399822235106
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,2,128,1,float16,float16,511,0.022912000119686127
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,2,128,1,float16,float16,1023,0.027350398898124694
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,2,128,1,float16,float16,255,0.021499200165271758
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,2,128,1,float16,float16,2047,0.027790400385856628
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,4,128,1,float16,float16,1,0.01401119977235794
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,4,128,1,float16,float16,3,0.014032000303268432
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,4,128,1,float16,float16,7,0.013857600092887879
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,4,128,1,float16,float16,15,0.013993600010871887
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,64,2,128,1,float16,float16,31,0.0178384006023407
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,4,128,1,float16,float16,31,0.013952000439167023
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,4,128,1,float16,float16,63,0.0141744002699852
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,4,128,1,float16,float16,127,0.016681599617004394
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,4,128,1,float16,float16,255,0.02149759978055954
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,4,128,1,float16,float16,511,0.023004800081253052
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,4,128,1,float16,float16,1023,0.027079999446868896
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,4,128,1,float16,float16,2047,0.028065600991249086
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,4,128,1,float16,float16,4095,0.029287999868392943
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,8,128,1,float16,float16,3,0.014168000221252442
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,8,128,1,float16,float16,7,0.014137600362300873
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,8,128,1,float16,float16,15,0.014155200123786927
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,8,128,1,float16,float16,31,0.014201599359512328
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,8,128,1,float16,float16,63,0.01427839994430542
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,8,128,1,float16,float16,127,0.016569599509239197
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,8,128,1,float16,float16,255,0.021686400473117828
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,8,128,1,float16,float16,511,0.022643199563026427
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,8,128,1,float16,float16,1023,0.02720479965209961
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,8,128,1,float16,float16,2047,0.02813119888305664
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,8,128,1,float16,float16,4095,0.03708159923553467
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,1,128,1,float16,float16,1,0.014078399538993836
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,1,128,1,float16,float16,3,0.01406240016222
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,1,128,1,float16,float16,15,0.014139199256896972
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,1,128,1,float16,float16,31,0.014201599359512328
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,1,128,1,float16,float16,63,0.014286400377750396
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,1,128,1,float16,float16,127,0.016782400012016297
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,1,128,1,float16,float16,255,0.021675199270248413
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,1,128,1,float16,float16,511,0.023524799942970277
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,1,128,1,float16,float16,255,0.021908800303936004
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,1,128,1,float16,float16,1023,0.027799999713897704
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,1,128,1,float16,float16,2047,0.02863520085811615
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,1,128,1,float16,float16,4095,0.029788801074028017
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,1,128,1,float16,float16,4095,0.02916640043258667
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,2,128,1,float16,float16,1,0.014407999813556671
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,2,128,1,float16,float16,3,0.01404000073671341
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,2,128,1,float16,float16,15,0.014368000626564025
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,2,128,1,float16,float16,31,0.014127999544143677
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,2,128,1,float16,float16,7,0.014187200367450714
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,2,128,1,float16,float16,63,0.014451199769973755
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,2,128,1,float16,float16,127,0.016915200650691985
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,2,128,1,float16,float16,255,0.021704000234603883
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,2,128,1,float16,float16,511,0.023324799537658692
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,2,128,1,float16,float16,1023,0.027403199672698976
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,2,128,1,float16,float16,2047,0.028366398811340333
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,2,128,1,float16,float16,4095,0.029684799909591674
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,4,128,1,float16,float16,1,0.013963200151920319
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,4,128,1,float16,float16,3,0.01414559930562973
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,2,128,1,float16,float16,4095,0.028972798585891725
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,4,128,1,float16,float16,15,0.01406719982624054
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,4,128,1,float16,float16,7,0.014159999787807465
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,4,128,1,float16,float16,31,0.014203199744224548
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,4,128,1,float16,float16,127,0.016763199865818024
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,4,128,1,float16,float16,255,0.02184319943189621
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,4,128,1,float16,float16,511,0.02333440035581589
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,4,128,1,float16,float16,1023,0.02757920026779175
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,4,128,1,float16,float16,2047,0.02874560058116913
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,4,128,1,float16,float16,4095,0.037703999876976015
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,8,128,1,float16,float16,1,0.014094400405883788
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,8,128,1,float16,float16,3,0.01438400000333786
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,8,128,1,float16,float16,15,0.014107200503349304
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,8,128,1,float16,float16,7,0.014103999733924866
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,8,128,1,float16,float16,63,0.014166399836540222
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,8,128,1,float16,float16,127,0.016705599427223206
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,8,128,1,float16,float16,255,0.021649600565433504
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,8,128,1,float16,float16,511,0.02354079931974411
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,8,128,1,float16,float16,1,0.013875199854373932
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,8,128,1,float16,float16,2047,0.03668479919433594
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,1,128,1,float16,float16,1,0.015028800070285796
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,8,128,1,float16,float16,4095,0.0540448009967804
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,1,128,1,float16,float16,3,0.015054400265216827
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,1,128,1,float16,float16,7,0.015038399398326874
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,1,128,1,float16,float16,15,0.015132799744606018
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,1,128,1,float16,float16,31,0.015089599788188935
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,1,128,1,float16,float16,63,0.01534239947795868
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,1,128,1,float16,float16,127,0.017713600397109987
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,1,128,1,float16,float16,255,0.022779199481010436
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,1,128,1,float16,float16,511,0.03210560083389282
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,1,128,1,float16,float16,1023,0.04176959991455078
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,1,128,1,float16,float16,7,0.014135999977588654
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,2,128,1,float16,float16,1,0.014910399913787842
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,1,128,1,float16,float16,2047,0.0586143970489502
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,2,128,1,float16,float16,3,0.015033599734306336
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,1,128,1,float16,float16,4095,0.08572480082511902
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,2,128,1,float16,float16,7,0.015222400426864624
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,2,128,1,float16,float16,15,0.015196800231933594
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,2,128,1,float16,float16,63,0.015412800014019012
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,2,128,1,float16,float16,31,0.015243199467658997
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,2,128,1,float16,float16,127,0.017951999604701997
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,2,128,1,float16,float16,255,0.02292640060186386
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,64,1,128,1,float16,float16,31,0.014246399700641631
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,2,128,1,float16,float16,511,0.049358400702476504
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,2,128,1,float16,float16,1023,0.07392320036888123
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,4,128,1,float16,float16,1,0.017047999799251555
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,2,128,1,float16,float16,2047,0.1143407940864563
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,4,128,1,float16,float16,3,0.016982400417327882
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,4,128,1,float16,float16,7,0.01695840060710907
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,2,128,1,float16,float16,4095,0.19457919597625734
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,4,128,1,float16,float16,15,0.017110399901866913
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,4,128,1,float16,float16,31,0.017158399522304534
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,4,128,1,float16,float16,127,0.02176959961652756
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,4,128,1,float16,float16,63,0.017481599748134614
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,4,128,1,float16,float16,255,0.029870399832725526
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,4,128,1,float16,float16,511,0.07105439901351929
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,4,128,1,float16,float16,1023,0.11459200382232666
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,4,128,1,float16,float16,2047,0.1949023962020874
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,8,128,1,float16,float16,1,0.022945599257946016
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,8,128,1,float16,float16,3,0.022924800217151643
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,8,128,1,float16,float16,7,0.023012800514698027
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,4,128,1,float16,float16,63,0.014102399349212646
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,8,128,1,float16,float16,31,0.02329760044813156
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,4,128,1,float16,float16,4095,0.35474400520324706
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,8,128,1,float16,float16,63,0.023396800458431243
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,8,128,1,float16,float16,15,0.022987200319766997
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,8,128,1,float16,float16,127,0.02983039915561676
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,8,128,1,float16,float16,255,0.047886401414871216
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,8,128,1,float16,float16,31,0.014484800398349762
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,8,128,1,float16,float16,511,0.09902719855308532
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,8,128,1,float16,float16,1023,0.1649824023246765
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,1,128,1,float16,float16,1,0.014428800344467163
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,1,128,1,float16,float16,3,0.014081600308418273
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,1,128,1,float16,float16,15,0.014459200203418732
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,8,128,1,float16,float16,2047,0.28455519676208496
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,1,128,1,float16,float16,7,0.014233599603176116
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,1,128,1,float16,float16,31,0.01430879980325699
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,64,8,128,1,float16,float16,1023,0.027816000580787658
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,1,128,1,float16,float16,255,0.021772800385951994
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,1,128,1,float16,float16,127,0.017052799463272095
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,1,128,1,float16,float16,511,0.02380799949169159
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,1,128,1,float16,float16,1023,0.028020799160003662
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,1,128,1,float16,float16,2047,0.02890399992465973
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,2,128,1,float16,float16,1,0.014068800210952758
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,2,128,1,float16,float16,3,0.014185599982738495
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,2,128,1,float16,float16,15,0.014183999598026275
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,1,128,1,float16,float16,4095,0.03067359924316406
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,64,8,128,1,float16,float16,4095,0.5248239994049072
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,2,128,1,float16,float16,31,0.014256000518798828
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,2,128,1,float16,float16,63,0.014241600036621093
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,2,128,1,float16,float16,127,0.016777600347995757
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,2,128,1,float16,float16,255,0.02157599925994873
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,2,128,1,float16,float16,1023,0.027809599041938783
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,2,128,1,float16,float16,2047,0.028977599740028382
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,4,128,1,float16,float16,1,0.014035199582576752
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,4,128,1,float16,float16,3,0.014182400703430176
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,2,128,1,float16,float16,4095,0.03827680051326752
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,4,128,1,float16,float16,15,0.014260800182819366
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,4,128,1,float16,float16,63,0.014232000708580017
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,4,128,1,float16,float16,127,0.016945600509643555
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,4,128,1,float16,float16,255,0.02179519981145859
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,4,128,1,float16,float16,511,0.023817600309848787
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,4,128,1,float16,float16,1023,0.028228801488876343
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,4,128,1,float16,float16,2047,0.0371071994304657
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,4,128,1,float16,float16,4095,0.05479840040206909
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,8,128,1,float16,float16,1,0.014193600416183472
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,8,128,1,float16,float16,3,0.014416000247001648
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,8,128,1,float16,float16,7,0.014350399374961853
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,8,128,1,float16,float16,15,0.01414719969034195
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,8,128,1,float16,float16,31,0.014486399292945863
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,8,128,1,float16,float16,63,0.014382399618625641
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,8,128,1,float16,float16,127,0.017075200378894807
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,8,128,1,float16,float16,255,0.021836799383163453
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,8,128,1,float16,float16,511,0.028302401304244995
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,8,128,1,float16,float16,1023,0.037406399846076965
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,8,128,1,float16,float16,2047,0.05371040105819702
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,8,128,1,float16,float16,4095,0.08104000091552735
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,1,128,1,float16,float16,1,0.019145600497722626
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,1,128,1,float16,float16,7,0.01918399930000305
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,1,128,1,float16,float16,3,0.019097599387168884
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,1,128,1,float16,float16,15,0.019438399374485014
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,1,128,1,float16,float16,31,0.019257600605487823
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,1,128,1,float16,float16,63,0.01947360038757324
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,1,128,1,float16,float16,127,0.02367199957370758
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,1,128,1,float16,float16,63,0.014443199336528777
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,1,128,1,float16,float16,255,0.032153600454330446
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,1,128,1,float16,float16,511,0.07552639842033386
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,1,128,1,float16,float16,1023,0.11909760236740112
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,2,128,1,float16,float16,1,0.0248879998922348
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,2,128,1,float16,float16,7,0.014177599549293518
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,1,128,1,float16,float16,2047,0.19872159957885743
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,2,128,1,float16,float16,15,0.025059199333190917
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,2,128,1,float16,float16,3,0.025051200389862062
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,2,128,1,float16,float16,511,0.02356480062007904
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,2,128,1,float16,float16,63,0.025462400913238526
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,2,128,1,float16,float16,127,0.03219040036201477
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,2,128,1,float16,float16,255,0.05002239942550659
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,4,128,1,float16,float16,7,0.01406559944152832
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,2,128,1,float16,float16,511,0.1041856050491333
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,4,128,1,float16,float16,1,0.03229599893093109
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,64,4,128,1,float16,float16,31,0.014168000221252442
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,2,128,1,float16,float16,1023,0.16929759979248046
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,4,128,1,float16,float16,3,0.03225440084934235
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,4,128,1,float16,float16,15,0.03259679973125458
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,4,128,1,float16,float16,31,0.03285439908504486
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,4,128,1,float16,float16,63,0.0337583988904953
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,2,128,1,float16,float16,2047,0.288702392578125
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,4,128,1,float16,float16,127,0.050470399856567386
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,4,128,1,float16,float16,255,0.07422080039978027
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,4,128,1,float16,float16,511,0.16047680377960205
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,4,128,1,float16,float16,1023,0.2713711977005005
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,8,128,1,float16,float16,1,0.04999519884586334
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,8,128,1,float16,float16,7,0.050044798851013185
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,8,128,1,float16,float16,3,0.05026400089263916
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,8,128,1,float16,float16,15,0.050449597835540774
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,4,128,1,float16,float16,2047,0.468503999710083
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,8,128,1,float16,float16,31,0.05161280035972595
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,8,128,1,float16,float16,63,0.05906080007553101
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,8,128,1,float16,float16,127,0.07931839823722839
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,8,128,1,float16,float16,255,0.12316319942474366
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,1,128,1,float16,float16,1,0.027718400955200194
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,8,128,1,float16,float16,511,0.2973839998245239
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,1,128,1,float16,float16,3,0.02778559923171997
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,1,128,1,float16,float16,7,0.027809599041938783
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,8,128,1,float16,float16,1023,0.5179247856140137
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,1,128,1,float16,float16,15,0.0279664009809494
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,1,128,1,float16,float16,31,0.028177601099014283
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,1,128,1,float16,float16,63,0.028679999709129333
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,1,128,1,float16,float16,127,0.03573760092258453
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,1,128,1,float16,float16,255,0.058430397510528566
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,8,128,1,float16,float16,2047,0.915617561340332
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,2,128,1,float16,float16,7,0.025169599056243896
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,1,128,1,float16,float16,511,0.10958880186080933
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,2,128,1,float16,float16,31,0.02524479925632477
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,2,128,1,float16,float16,3,0.03457440137863159
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,2,128,1,float16,float16,15,0.035180801153182985
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,2,128,1,float16,float16,7,0.03492799997329712
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,2,128,1,float16,float16,63,0.037780800461769105
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,2,128,1,float16,float16,127,0.05345919728279114
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,2,128,1,float16,float16,255,0.07624639868736267
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,4,128,1,float16,float16,1,0.05220000147819519
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,64,4,128,1,float16,float16,7,0.03228960037231445
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,4,128,1,float16,float16,3,0.05204319953918457
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,4,128,1,float16,float16,7,0.0521232008934021
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,4,128,1,float16,float16,15,0.05265439748764038
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,4,128,1,float16,float16,31,0.053268802165985105
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,2,128,1,float16,float16,511,0.16662720441818238
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,4,128,1,float16,float16,63,0.06278719902038574
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,4,128,1,float16,float16,127,0.0830623984336853
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,4,128,1,float16,float16,255,0.12681920528411866
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,8,128,1,float16,float16,3,0.08934879899024964
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,8,128,1,float16,float16,1,0.08934239745140075
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,8,128,1,float16,float16,7,0.08924639821052552
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,8,128,1,float16,float16,15,0.09031839966773987
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,4,128,1,float16,float16,511,0.3009536027908325
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,8,128,1,float16,float16,31,0.09383360147476197
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,8,128,1,float16,float16,127,0.14025280475616456
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,64,1,128,1,float16,float16,1,0.040300801396369934
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,8,128,1,float16,float16,63,0.10204479694366456
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,8,128,1,float16,float16,255,0.22524640560150147
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,64,1,128,1,float16,float16,3,0.040212801098823546
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,64,1,128,1,float16,float16,7,0.04046719968318939
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,64,1,128,1,float16,float16,15,0.04134399890899658
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,64,1,128,1,float16,float16,31,0.04290240108966827
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,64,1,128,1,float16,float16,63,0.05122720003128052
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,64,1,128,1,float16,float16,127,0.06127039790153503
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,64,2,128,1,float16,float16,1,0.05754719972610474
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,8,128,1,float16,float16,511,0.5538320064544677
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,64,1,128,1,float16,float16,255,0.08755679726600647
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,64,2,128,1,float16,float16,7,0.0576960027217865
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,64,2,128,1,float16,float16,15,0.05817919969558716
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,64,2,128,1,float16,float16,63,0.06955360174179077
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,64,2,128,1,float16,float16,31,0.06321120262145996
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,64,2,128,1,float16,float16,127,0.09025760293006897
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,64,2,128,1,float16,float16,255,0.1346016049385071
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,64,4,128,1,float16,float16,1,0.09441760182380676
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,64,4,128,1,float16,float16,7,0.09474560022354125
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,64,4,128,1,float16,float16,3,0.0945792019367218
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,2,128,1,float16,float16,1,0.0349263995885849
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,64,4,128,1,float16,float16,15,0.098198401927948
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,64,4,128,1,float16,float16,31,0.10235519409179687
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,64,4,128,1,float16,float16,63,0.10773439407348633
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,64,2,128,1,float16,float16,31,0.035305601358413694
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,64,4,128,1,float16,float16,255,0.2342832088470459
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,64,8,128,1,float16,float16,1,0.16544959545135499
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,64,4,128,1,float16,float16,127,0.1477311968803406
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,64,8,128,1,float16,float16,3,0.16576800346374512
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,64,8,128,1,float16,float16,7,0.16712160110473634
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,64,8,128,1,float16,float16,15,0.1729904055595398
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,1,128,1,float16,float16,1,0.014254400134086609
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,64,8,128,1,float16,float16,31,0.17778079509735106
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,64,8,128,1,float16,float16,63,0.18498239517211915
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,1,128,1,float16,float16,7,0.014281600713729858
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,1,128,1,float16,float16,3,0.014241600036621093
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,1,128,1,float16,float16,63,0.014470399916172027
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,1,128,1,float16,float16,15,0.014417600631713868
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,1,128,1,float16,float16,127,0.016935999691486358
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,64,8,128,1,float16,float16,127,0.2623136043548584
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,64,8,128,1,float16,float16,255,0.4293647766113281
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,1,128,1,float16,float16,255,0.021798400580883025
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,1,128,1,float16,float16,511,0.024243199825286867
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,1,128,1,float16,float16,1023,0.02842719852924347
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,2,128,1,float16,float16,1,0.014097599685192109
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,1,128,1,float16,float16,2047,0.029763200879096986
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,2,128,1,float16,float16,3,0.014452800154685974
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,2,128,1,float16,float16,15,0.01409280002117157
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,1,128,1,float16,float16,4095,0.03924640119075775
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,2,128,1,float16,float16,63,0.014636799693107605
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,2,128,1,float16,float16,31,0.014273600280284881
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,2,128,1,float16,float16,127,0.016944000124931337
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,2,128,1,float16,float16,255,0.022006399929523468
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,2,128,1,float16,float16,511,0.024051199853420257
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,2,128,1,float16,float16,1023,0.028433600068092348
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,2,128,1,float16,float16,2047,0.03763839900493622
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,2,128,1,float16,float16,4095,0.054606401920318605
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,4,128,1,float16,float16,1,0.014135999977588654
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,4,128,1,float16,float16,3,0.01443839967250824
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,4,128,1,float16,float16,15,0.014270399510860444
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,4,128,1,float16,float16,7,0.014235199987888336
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,4,128,1,float16,float16,127,0.01706400066614151
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,4,128,1,float16,float16,63,0.014542399346828461
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,64,2,128,1,float16,float16,3,0.057265597581863406
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,4,128,1,float16,float16,1023,0.037723198533058167
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,4,128,1,float16,float16,2047,0.05482239723205566
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,4,128,1,float16,float16,4095,0.08147519826889038
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,8,128,1,float16,float16,1,0.014316800236701965
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,8,128,1,float16,float16,3,0.01435679942369461
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,8,128,1,float16,float16,15,0.014505599439144135
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,8,128,1,float16,float16,31,0.01451520025730133
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,8,128,1,float16,float16,63,0.014580799639225006
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,8,128,1,float16,float16,127,0.01736160069704056
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,8,128,1,float16,float16,255,0.022121599316596983
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,8,128,1,float16,float16,511,0.045377600193023684
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,8,128,1,float16,float16,1023,0.07043039798736572
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,8,128,1,float16,float16,2047,0.11069120168685913
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,1,128,1,float16,float16,31,0.014455999433994293
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,64,1,128,1,float16,float16,1,0.06950880289077759
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,8,128,1,float16,float16,4095,0.19084800481796266
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,2,128,1,float16,float16,7,0.01427839994430542
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,64,1,128,1,float16,float16,3,0.06984159946441651
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,64,1,128,1,float16,float16,7,0.06981279850006103
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,64,1,128,1,float16,float16,15,0.06968799829483033
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,64,1,128,1,float16,float16,31,0.07182080149650574
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,64,1,128,1,float16,float16,63,0.07586399912834167
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,64,1,128,1,float16,float16,127,0.09622719883918762
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,64,2,128,1,float16,float16,1,0.1024783968925476
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,64,2,128,1,float16,float16,3,0.10405759811401367
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,64,2,128,1,float16,float16,7,0.10497119426727294
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,4,128,1,float16,float16,31,0.014481599628925323
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,4,128,1,float16,float16,255,0.021715199947357176
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,64,2,128,1,float16,float16,63,0.1125648021697998
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,4,128,1,float16,float16,511,0.028697600960731505
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,64,4,128,1,float16,float16,1,0.17313920259475707
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,64,4,128,1,float16,float16,3,0.17433279752731323
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,64,4,128,1,float16,float16,7,0.1769711971282959
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,64,4,128,1,float16,float16,15,0.17806719541549682
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,64,8,128,1,float16,float16,7,0.014372800290584565
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,64,4,128,1,float16,float16,31,0.17967519760131836
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,64,4,128,1,float16,float16,63,0.1897328019142151
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,64,4,128,1,float16,float16,127,0.2701632022857666
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,64,8,128,1,float16,float16,1,0.31656320095062257
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,64,8,128,1,float16,float16,3,0.3168960094451904
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,64,8,128,1,float16,float16,7,0.32044799327850343
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,64,8,128,1,float16,float16,31,0.3285232067108154
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,64,8,128,1,float16,float16,15,0.3241568088531494
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,64,8,128,1,float16,float16,63,0.34202721118927004
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,64,1,128,1,float16,float16,3,0.11316959857940674
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,64,1,128,1,float16,float16,1,0.11223360300064086
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,64,1,128,1,float16,float16,7,0.11558079719543457
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,64,1,128,1,float16,float16,15,0.11560959815979004
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,64,1,128,1,float16,float16,31,0.11586560010910034
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,64,8,128,1,float16,float16,127,0.5011536121368408
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,64,2,128,1,float16,float16,1,0.18112319707870483
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,64,1,128,1,float16,float16,63,0.12208319902420044
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,64,2,128,1,float16,float16,3,0.18379679918289185
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,64,2,128,1,float16,float16,7,0.18610399961471558
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,64,2,128,1,float16,float16,15,0.18572640419006348
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,64,2,128,1,float16,float16,31,0.18839360475540162
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,64,2,128,1,float16,float16,63,0.19866080284118653
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,64,2,128,1,float16,float16,15,0.10614240169525146
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,64,2,128,1,float16,float16,31,0.10687839984893799
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,64,4,128,1,float16,float16,1,0.3244976043701172
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,64,4,128,1,float16,float16,3,0.32442560195922854
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,64,4,128,1,float16,float16,7,0.3281183958053589
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,64,4,128,1,float16,float16,15,0.3319839954376221
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,64,2,128,1,float16,float16,127,0.1560912013053894
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,64,4,128,1,float16,float16,31,0.3373248100280762
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,64,4,128,1,float16,float16,63,0.35282719135284424
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,64,8,128,1,float16,float16,1,0.6102015972137451
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,1,128,1,float16,float16,1,0.014276799559593201
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,1,128,1,float16,float16,3,0.014467200636863709
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,64,8,128,1,float16,float16,3,0.6135231971740722
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,1,128,1,float16,float16,15,0.014212800562381745
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,64,8,128,1,float16,float16,7,0.6171472072601318
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,1,128,1,float16,float16,31,0.014448000490665436
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,64,8,128,1,float16,float16,15,0.6235375881195069
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,1,128,1,float16,float16,63,0.014424000680446625
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,64,8,128,1,float16,float16,31,0.6338335990905761
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,1,128,1,float16,float16,255,0.021998399496078493
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,1,128,1,float16,float16,511,0.025832000374794006
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,64,8,128,1,float16,float16,63,0.6591599941253662
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,2,128,1,float16,float16,1,0.014392000436782838
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,1,128,1,float16,float16,1023,0.02990719974040985
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,1,128,1,float16,float16,2047,0.039273598790168764
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,2,128,1,float16,float16,3,0.014323200285434722
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,1,128,1,float16,float16,4095,0.05848479866981506
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,2,128,1,float16,float16,15,0.014444799721240997
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,2,128,1,float16,float16,31,0.01438560038805008
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,2,128,1,float16,float16,7,0.014431999623775482
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,2,128,1,float16,float16,127,0.01711679995059967
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,2,128,1,float16,float16,63,0.014420799911022186
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,2,128,1,float16,float16,255,0.021932800114154816
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,2,128,1,float16,float16,511,0.029372799396514892
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,2,128,1,float16,float16,2047,0.05539360046386719
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,4,128,1,float16,float16,3,0.0143327996134758
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,2,128,1,float16,float16,4095,0.08298400044441223
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,4,128,1,float16,float16,7,0.014511999487876893
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,4,128,1,float16,float16,15,0.014552000164985656
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,4,128,1,float16,float16,63,0.014614400267601014
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,4,128,1,float16,float16,31,0.01440960019826889
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,4,128,1,float16,float16,255,0.022086399793624877
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,4,128,1,float16,float16,511,0.046935999393463136
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,4,128,1,float16,float16,1023,0.0721231997013092
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,4,128,1,float16,float16,2047,0.11176799535751343
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,8,128,1,float16,float16,1,0.0162992000579834
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,4,128,1,float16,float16,4095,0.19205600023269653
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,8,128,1,float16,float16,3,0.016323199868202208
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,8,128,1,float16,float16,15,0.01648160070180893
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,8,128,1,float16,float16,7,0.0165120005607605
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,8,128,1,float16,float16,31,0.016673600673675536
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,8,128,1,float16,float16,63,0.016969600319862367
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,8,128,1,float16,float16,127,0.021116800606250763
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,8,128,1,float16,float16,255,0.029260799288749695
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,8,128,1,float16,float16,511,0.0688368022441864
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,8,128,1,float16,float16,1023,0.11413919925689697
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,8,128,1,float16,float16,2047,0.19276800155639648
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,1,128,1,float16,float16,1,0.015535999834537507
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,8,128,1,float16,float16,4095,0.3535151958465576
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,1,128,1,float16,float16,3,0.015699200332164764
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,1,128,1,float16,float16,7,0.01432960033416748
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,1,128,1,float16,float16,15,0.015724800527095795
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,1,128,1,float16,float16,7,0.015820799767971037
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,1,128,1,float16,float16,31,0.015751999616622925
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,1,128,1,float16,float16,127,0.017003199458122252
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,1,128,1,float16,float16,63,0.01589920073747635
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,1,128,1,float16,float16,127,0.018563200533390046
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,1,128,1,float16,float16,255,0.02338079959154129
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,1,128,1,float16,float16,511,0.04981440007686615
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,1,128,1,float16,float16,1023,0.07553759813308716
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,1,128,1,float16,float16,2047,0.11529439687728882
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,1,128,1,float16,float16,4095,0.19515039920806884
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,2,128,1,float16,float16,1,0.017481599748134614
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,2,128,1,float16,float16,3,0.017611199617385866
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,2,128,1,float16,float16,15,0.017633600533008574
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,2,128,1,float16,float16,7,0.017683200538158417
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,2,128,1,float16,float16,1023,0.0392767995595932
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,2,128,1,float16,float16,31,0.01778080016374588
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,2,128,1,float16,float16,63,0.018012799322605133
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,2,128,1,float16,float16,127,0.02231840044260025
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,2,128,1,float16,float16,255,0.03061760067939758
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,4,128,1,float16,float16,127,0.017262400686740877
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,2,128,1,float16,float16,511,0.07150080204010009
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,2,128,1,float16,float16,1023,0.11594079732894898
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,4,128,1,float16,float16,1,0.023500800132751465
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,2,128,1,float16,float16,2047,0.19479520320892335
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,4,128,1,float16,float16,3,0.02354719936847687
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,4,128,1,float16,float16,15,0.023731200397014617
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,4,128,1,float16,float16,31,0.023716799914836884
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,2,128,1,float16,float16,4095,0.3559247970581055
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,4,128,1,float16,float16,63,0.02422720044851303
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,4,128,1,float16,float16,127,0.030422401428222657
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,2,128,1,float16,float16,8191,0.6805679798126221
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,4,128,1,float16,float16,255,0.04856480062007904
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,4,128,1,float16,float16,511,0.0995423972606659
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,4,128,1,float16,float16,1023,0.16617759466171264
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,8,128,1,float16,float16,1,0.031009599566459656
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,4,128,1,float16,float16,2047,0.28684000968933104
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,8,128,1,float16,float16,3,0.031150400638580322
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,8,128,1,float16,float16,7,0.03128800094127655
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,8,128,1,float16,float16,15,0.03129920065402984
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,4,128,1,float16,float16,4095,0.5270944118499756
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,8,128,1,float16,float16,63,0.032369598746299744
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,8,128,1,float16,float16,255,0.07008479833602906
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,4,128,1,float16,float16,8191,1.0143376350402833
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,8,128,1,float16,float16,511,0.1575935959815979
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,8,128,1,float16,float16,1023,0.2678512096405029
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,64,4,128,1,float16,float16,1,0.014236800372600555
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,8,128,1,float16,float16,2047,0.4669680118560791
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,1,128,1,float16,float16,1,0.014001600444316864
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,1,128,1,float16,float16,8191,0.35745599269866946
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,1,128,1,float16,float16,7,0.013953599333763122
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,1,128,1,float16,float16,3,0.013828800618648529
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,1,128,1,float16,float16,31,0.014075200259685516
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,1,128,1,float16,float16,15,0.013976000249385834
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,1,128,1,float16,float16,63,0.013980799913406372
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,1,128,1,float16,float16,127,0.0166143998503685
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,1,128,1,float16,float16,255,0.0213359996676445
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,1,128,1,float16,float16,1023,0.027108800411224366
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,1,128,1,float16,float16,511,0.02279199957847595
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,1,128,1,float16,float16,4095,0.02844800055027008
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,1,128,1,float16,float16,2047,0.0277072012424469
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,4,128,1,float16,float16,7,0.023612800240516662
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,2,128,1,float16,float16,1,0.01406240016222
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,8,128,1,float16,float16,4095,0.8672592163085937
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,2,128,1,float16,float16,7,0.014102399349212646
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,2,128,1,float16,float16,15,0.014102399349212646
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,2,128,1,float16,float16,63,0.014239999651908874
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,2,128,1,float16,float16,127,0.016788800060749055
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,2,128,1,float16,float16,31,0.014008000493049622
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,2,128,1,float16,float16,255,0.02162559926509857
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,2,128,1,float16,float16,511,0.022782400250434875
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,2,128,1,float16,float16,1023,0.026998400688171387
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,2,128,1,float16,float16,4095,0.028604799509048463
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,2,128,1,float16,float16,2047,0.027636799216270446
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,2,128,1,float16,float16,8191,0.03107520043849945
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,4,128,1,float16,float16,1,0.01387840062379837
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,4,128,1,float16,float16,3,0.013952000439167023
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,8,128,1,float16,float16,8191,1.7018400192260743
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,4,128,1,float16,float16,15,0.013969600200653076
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,4,128,1,float16,float16,63,0.014049600064754485
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,8,128,1,float16,float16,31,0.03163360059261322
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,4,128,1,float16,float16,255,0.02149759978055954
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,32,8,128,1,float16,float16,127,0.04812000095844269
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,4,128,1,float16,float16,511,0.022412799298763275
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,4,128,1,float16,float16,1023,0.027083200216293336
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,4,128,1,float16,float16,2047,0.027372801303863527
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,4,128,1,float16,float16,4095,0.0292959988117218
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,4,128,1,float16,float16,8191,0.03743039965629578
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,8,128,1,float16,float16,1,0.01422560065984726
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,8,128,1,float16,float16,3,0.013857600092887879
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,8,128,1,float16,float16,7,0.014190399646759033
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,8,128,1,float16,float16,15,0.014046399295330048
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,8,128,1,float16,float16,31,0.014256000518798828
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,8,128,1,float16,float16,63,0.014044800400733947
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,8,128,1,float16,float16,255,0.021483199298381807
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,8,128,1,float16,float16,511,0.022623999416828154
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,8,128,1,float16,float16,1023,0.02717919945716858
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,8,128,1,float16,float16,2047,0.02805280089378357
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,8,128,1,float16,float16,4095,0.0368800014257431
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,8,128,1,float16,float16,8191,0.05389919877052307
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,1,128,1,float16,float16,1,0.014030399918556213
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,1,128,1,float16,float16,8191,0.03068319857120514
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,1,128,1,float16,float16,3,0.01408800035715103
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,2,128,1,float16,float16,3,0.013976000249385834
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,1,128,1,float16,float16,7,0.014051200449466705
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,1,128,1,float16,float16,15,0.014049600064754485
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,1,128,1,float16,float16,31,0.013953599333763122
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,1,128,1,float16,float16,63,0.014259199798107147
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,1,128,1,float16,float16,127,0.016616000235080718
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,1,128,1,float16,float16,255,0.02173919975757599
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,1,128,1,float16,float16,511,0.022993600368499754
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,1,128,1,float16,float16,1023,0.027195200324058533
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,1,128,1,float16,float16,2047,0.02794240117073059
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,1,128,1,float16,float16,8191,0.030796799063682555
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,2,128,1,float16,float16,1,0.014177599549293518
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,2,128,1,float16,float16,3,0.013891200721263885
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,2,128,1,float16,float16,7,0.014193600416183472
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,2,128,1,float16,float16,31,0.014212800562381745
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,2,128,1,float16,float16,63,0.014220799505710601
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,4,128,1,float16,float16,7,0.013953599333763122
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,2,128,1,float16,float16,127,0.01663520038127899
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,2,128,1,float16,float16,255,0.021649600565433504
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,2,128,1,float16,float16,511,0.02300799936056137
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,4,128,1,float16,float16,127,0.016492800414562227
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,2,128,1,float16,float16,1023,0.0272816002368927
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,2,128,1,float16,float16,2047,0.02799839973449707
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,2,128,1,float16,float16,4095,0.028951999545097352
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,2,128,1,float16,float16,8191,0.03813279867172241
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,4,128,1,float16,float16,1,0.013995200395584106
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,4,128,1,float16,float16,3,0.013902400434017182
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,4,128,1,float16,float16,7,0.014102399349212646
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,4,128,1,float16,float16,15,0.013982400298118591
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,4,128,1,float16,float16,31,0.01404159963130951
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,4,128,1,float16,float16,63,0.014110399782657624
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,4,128,1,float16,float16,127,0.016612799465656282
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,4,128,1,float16,float16,255,0.02146400064229965
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,4,128,1,float16,float16,511,0.02287680059671402
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,4,128,1,float16,float16,1023,0.0272816002368927
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,4,128,1,float16,float16,2047,0.02826080024242401
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,4,128,1,float16,float16,4095,0.03713119924068451
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,4,128,1,float16,float16,8191,0.05435360074043274
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,8,128,1,float16,float16,1,0.01403840035200119
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,8,128,1,float16,float16,3,0.014137600362300873
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,8,128,1,float16,float16,15,0.014078399538993836
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,8,128,1,float16,float16,7,0.01422239989042282
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,8,128,1,float16,float16,127,0.017020800709724428
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,8,128,1,float16,float16,63,0.014276799559593201
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,8,128,1,float16,float16,127,0.016894400119781494
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,8,128,1,float16,float16,255,0.02191839963197708
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,8,128,1,float16,float16,1023,0.027801600098609925
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,8,128,1,float16,float16,2047,0.0369024008512497
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,8,128,1,float16,float16,4095,0.05355200171470642
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,8,128,1,float16,float16,8191,0.08113600015640259
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,1,128,1,float16,float16,1,0.014977599680423736
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,1,128,1,float16,float16,7,0.015020799636840821
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,1,128,1,float16,float16,3,0.014971199631690978
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,1,128,1,float16,float16,15,0.01501920074224472
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,1,128,1,float16,float16,31,0.015134400129318238
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,1,128,1,float16,float16,63,0.015203200280666351
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,1,128,1,float16,float16,127,0.017768000066280366
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,1,128,1,float16,float16,255,0.022487999498844148
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,1,128,1,float16,float16,511,0.030504000186920167
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,1,128,1,float16,float16,1023,0.039971199631690976
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,1,128,1,float16,float16,2047,0.05583840012550354
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,1,128,1,float16,float16,4095,0.08377280235290527
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,1,128,1,float16,float16,4095,0.028984001278877257
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,2,128,1,float16,float16,1,0.015052799880504609
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,2,128,1,float16,float16,3,0.015062400698661804
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,2,128,1,float16,float16,7,0.015110400319099427
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,2,128,1,float16,float16,15,0.013956800103187561
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,2,128,1,float16,float16,15,0.015124799311161041
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,2,128,1,float16,float16,31,0.015105600655078887
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,2,128,1,float16,float16,127,0.017843200266361235
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,2,128,1,float16,float16,255,0.02277279943227768
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,32,4,128,1,float16,float16,31,0.014604799449443817
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,2,128,1,float16,float16,511,0.04746400117874146
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,2,128,1,float16,float16,1023,0.07256479859352112
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,2,128,1,float16,float16,2047,0.11308959722518921
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,2,128,1,float16,float16,4095,0.19296959638595582
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,4,128,1,float16,float16,1,0.01687999963760376
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,8,128,1,float16,float16,31,0.014262400567531586
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,2,128,1,float16,float16,8191,0.356279993057251
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,4,128,1,float16,float16,3,0.016873599588871004
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,4,128,1,float16,float16,7,0.017103999853134155
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,32,8,128,1,float16,float16,511,0.023260800540447234
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,4,128,1,float16,float16,15,0.016927999258041383
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,4,128,1,float16,float16,63,0.01761920005083084
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,4,128,1,float16,float16,127,0.021747200191020964
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,4,128,1,float16,float16,255,0.02991519868373871
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,4,128,1,float16,float16,1023,0.11432000398635864
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,4,128,1,float16,float16,2047,0.19421119689941407
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,8,128,1,float16,float16,1,0.022835199534893037
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,4,128,1,float16,float16,4095,0.35371999740600585
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,8,128,1,float16,float16,3,0.02290560007095337
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,8,128,1,float16,float16,7,0.022753599286079406
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,8,128,1,float16,float16,15,0.02314720004796982
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,8,128,1,float16,float16,31,0.02327840030193329
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,8,128,1,float16,float16,63,0.023521600663661955
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,8,128,1,float16,float16,127,0.029636800289154053
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,8,128,1,float16,float16,255,0.0457392007112503
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,1,128,1,float16,float16,8191,0.1371008038520813
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,8,128,1,float16,float16,511,0.09889280200004577
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,8,128,1,float16,float16,1023,0.16427520513534546
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,2,128,1,float16,float16,63,0.015273599326610566
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,1,128,1,float16,float16,1,0.014151999354362487
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,8,128,1,float16,float16,2047,0.2838768005371094
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,1,128,1,float16,float16,3,0.014140799641609192
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,1,128,1,float16,float16,7,0.0141184002161026
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,8,128,1,float16,float16,4095,0.5229504108428955
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,1,128,1,float16,float16,15,0.01417119950056076
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,1,128,1,float16,float16,31,0.014318400621414184
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,1,128,1,float16,float16,63,0.014179199934005737
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,1,128,1,float16,float16,127,0.016859200596809388
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,1,128,1,float16,float16,255,0.021803200244903564
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,1,128,1,float16,float16,511,0.023044799268245698
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,1,128,1,float16,float16,1023,0.027606400847434997
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,1,128,1,float16,float16,8191,0.03872799873352051
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,4,128,1,float16,float16,31,0.01706559956073761
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,2,128,1,float16,float16,1,0.013873599469661713
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,2,128,1,float16,float16,3,0.01401599943637848
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,2,128,1,float16,float16,7,0.013864000141620637
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,2,128,1,float16,float16,15,0.014131200313568116
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,2,128,1,float16,float16,63,0.014206400513648987
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,2,128,1,float16,float16,31,0.013979199528694152
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,4,128,1,float16,float16,511,0.07025439739227295
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,2,128,1,float16,float16,127,0.016734400391578676
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,2,128,1,float16,float16,255,0.0216048002243042
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,2,128,1,float16,float16,1023,0.02774080038070679
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,2,128,1,float16,float16,511,0.02327360063791275
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,2,128,1,float16,float16,2047,0.028579199314117433
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,2,128,1,float16,float16,4095,0.03773919939994812
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,2,128,1,float16,float16,8191,0.054763197898864746
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,4,128,1,float16,float16,1,0.014054399728775025
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,4,128,1,float16,float16,7,0.014227199554443359
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,4,128,1,float16,float16,15,0.014103999733924866
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,4,128,1,float16,float16,31,0.014204800128936768
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,4,128,1,float16,float16,63,0.014459200203418732
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,4,128,1,float16,float16,127,0.01690399944782257
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,4,128,1,float16,float16,8191,0.6809247970581055
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,4,128,1,float16,float16,255,0.02178560048341751
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,4,128,1,float16,float16,1023,0.027932798862457274
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,4,128,1,float16,float16,4095,0.05482720136642456
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,4,128,1,float16,float16,8191,0.08233280181884765
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,8,128,1,float16,float16,1,0.01406559944152832
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,8,128,1,float16,float16,3,0.014105600118637086
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,8,128,1,float16,float16,7,0.014281600713729858
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,8,128,1,float16,float16,31,0.014271999895572662
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,8,128,1,float16,float16,63,0.014428800344467163
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,8,128,1,float16,float16,127,0.017003199458122252
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,8,128,1,float16,float16,255,0.021935999393463135
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,1,128,1,float16,float16,2047,0.028326401114463808
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,8,128,1,float16,float16,511,0.028376001119613647
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,8,128,1,float16,float16,1023,0.03771359920501709
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,32,8,128,1,float16,float16,8191,1.0132623672485352
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,8,128,1,float16,float16,4095,0.0813215970993042
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,1,128,1,float16,float16,1,0.01879200041294098
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,8,128,1,float16,float16,8191,0.13457440137863158
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,1,128,1,float16,float16,3,0.018795199692249298
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,1,128,1,float16,float16,7,0.01875839978456497
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,1,128,1,float16,float16,15,0.018991999328136444
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,1,128,1,float16,float16,63,0.019289599359035493
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,1,128,1,float16,float16,31,0.01900479942560196
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,1,128,1,float16,float16,127,0.02351839989423752
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,1,128,1,float16,float16,255,0.03201279938220978
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,1,128,1,float16,float16,511,0.0737999975681305
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,4,128,1,float16,float16,3,0.014094400405883788
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,1,128,1,float16,float16,1023,0.11824480295181275
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,1,128,1,float16,float16,2047,0.19790879487991334
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,4,128,1,float16,float16,511,0.023707200586795808
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,2,128,1,float16,float16,1,0.024747200310230255
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,1,128,1,float16,float16,4095,0.3562432050704956
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,2,128,1,float16,float16,3,0.024740800261497498
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,4,128,1,float16,float16,2047,0.03672960102558136
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,2,128,1,float16,float16,7,0.02489439994096756
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,2,128,1,float16,float16,31,0.02503199875354767
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,2,128,1,float16,float16,15,0.024670399725437164
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,2,128,1,float16,float16,63,0.025284799933433532
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,2,128,1,float16,float16,127,0.031774398684501645
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,2,128,1,float16,float16,255,0.048635199666023254
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,2,128,1,float16,float16,1023,0.16903200149536132
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,2,128,1,float16,float16,511,0.10236159563064576
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,2,128,1,float16,float16,2047,0.2882512092590332
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,4,128,1,float16,float16,1,0.03223679959774017
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,4,128,1,float16,float16,3,0.03212960064411163
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,8,128,1,float16,float16,15,0.014129599928855896
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,4,128,1,float16,float16,15,0.0324752002954483
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,4,128,1,float16,float16,7,0.03252640068531036
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,4,128,1,float16,float16,63,0.033667200803756715
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,2,128,1,float16,float16,4095,0.5280511856079102
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,4,128,1,float16,float16,127,0.048716801404953006
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,4,128,1,float16,float16,255,0.07353119850158692
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,4,128,1,float16,float16,511,0.16015520095825195
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,4,128,1,float16,float16,1023,0.27058560848236085
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,8,128,1,float16,float16,1,0.05005599856376648
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,8,128,1,float16,float16,2047,0.0536191999912262
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,8,128,1,float16,float16,7,0.05011839866638183
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,4,128,1,float16,float16,2047,0.46890082359313967
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,8,128,1,float16,float16,15,0.050620800256729125
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,8,128,1,float16,float16,31,0.05100160241127014
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,8,128,1,float16,float16,127,0.07929599881172181
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,8,128,1,float16,float16,63,0.0597760021686554
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,4,128,1,float16,float16,4095,0.8707903861999512
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,8,128,1,float16,float16,255,0.12190400362014771
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,8,128,1,float16,float16,511,0.2951488018035889
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,1,128,1,float16,float16,1,0.02677919864654541
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,8,128,1,float16,float16,1023,0.5200367927551269
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,1,128,1,float16,float16,3,0.026700800657272337
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,1,128,1,float16,float16,7,0.026548799872398377
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,1,128,1,float16,float16,15,0.026761600375175477
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,8,128,1,float16,float16,2047,0.9163488388061524
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,1,128,1,float16,float16,63,0.02722240090370178
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,1,128,1,float16,float16,31,0.02683840095996857
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,1,128,1,float16,float16,255,0.051523202657699586
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,1,128,1,float16,float16,127,0.03434720039367676
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,1,128,1,float16,float16,511,0.10718719959259033
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,2,128,1,float16,float16,1,0.034134399890899655
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,1,128,1,float16,float16,1023,0.17161920070648193
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,2,128,1,float16,float16,3,0.03402880132198334
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,2,128,1,float16,float16,7,0.03432160019874573
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,2,128,1,float16,float16,31,0.03470079898834229
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,2,128,1,float16,float16,15,0.034411200881004335
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,4,128,1,float16,float16,31,0.03267520070075989
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,2,128,1,float16,float16,127,0.05144320130348205
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,2,128,1,float16,float16,63,0.036257600784301756
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,2,128,1,float16,float16,255,0.07346720099449158
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,2,128,1,float16,float16,511,0.16348960399627685
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,4,128,1,float16,float16,1,0.05183519721031189
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,4,128,1,float16,float16,3,0.05177119970321655
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,2,128,1,float16,float16,1023,0.2737855911254883
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,4,128,1,float16,float16,31,0.05271040201187134
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,4,128,1,float16,float16,15,0.05238879919052124
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,32,1,128,1,float16,float16,4095,0.029467201232910155
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,4,128,1,float16,float16,63,0.06228479743003845
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,4,128,1,float16,float16,127,0.08111199736595154
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,8,128,1,float16,float16,3,0.04975680112838745
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,8,128,1,float16,float16,3,0.08882399797439575
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,8,128,1,float16,float16,1,0.08891839981079101
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,4,128,1,float16,float16,511,0.2998847961425781
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,8,128,1,float16,float16,7,0.08928160071372986
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,8,128,1,float16,float16,15,0.08968160152435303
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,8,128,1,float16,float16,63,0.09978560209274293
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,8,128,1,float16,float16,31,0.09348639845848083
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,8,128,1,float16,float16,127,0.138153600692749
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,4,128,1,float16,float16,1023,0.52259521484375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,1,128,1,float16,float16,1,0.03902879953384399
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,8,128,1,float16,float16,255,0.22296481132507323
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,1,128,1,float16,float16,3,0.039208000898361205
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,1,128,1,float16,float16,7,0.03914079964160919
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,1,128,1,float16,float16,15,0.0392304003238678
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,8,128,1,float16,float16,511,0.5521455764770508
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,1,128,1,float16,float16,31,0.03961600065231323
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,1,128,1,float16,float16,63,0.04289920032024384
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,1,128,1,float16,float16,127,0.058736002445220946
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,1,128,1,float16,float16,255,0.07881439924240112
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,2,128,1,float16,float16,1,0.05627520084381103
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,8,128,1,float16,float16,1023,0.9747759819030761
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,2,128,1,float16,float16,3,0.056441599130630495
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,1,128,1,float16,float16,511,0.17416160106658934
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,2,128,1,float16,float16,7,0.05647680163383484
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,32,8,128,1,float16,float16,4095,1.7240255355834961
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,2,128,1,float16,float16,31,0.05764639973640442
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,2,128,1,float16,float16,63,0.06655840277671814
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,2,128,1,float16,float16,15,0.056574398279190065
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,4,128,1,float16,float16,1,0.09373440146446228
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,4,128,1,float16,float16,3,0.09355360269546509
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,2,128,1,float16,float16,255,0.13010400533676147
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,4,128,1,float16,float16,7,0.09385280013084411
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,4,128,1,float16,float16,15,0.09423360228538513
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,4,128,1,float16,float16,7,0.051759999990463254
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,4,128,1,float16,float16,31,0.09857439994812012
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,2,128,1,float16,float16,511,0.3151968002319336
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,4,128,1,float16,float16,63,0.10561280250549317
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,4,128,1,float16,float16,127,0.14384640455245973
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,32,4,128,1,float16,float16,255,0.12480640411376953
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,4,128,1,float16,float16,255,0.2303407907485962
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,8,128,1,float16,float16,3,0.16427839994430543
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,8,128,1,float16,float16,15,0.16841599941253663
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,8,128,1,float16,float16,1,0.16703519821166993
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,8,128,1,float16,float16,7,0.16476800441741943
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,8,128,1,float16,float16,31,0.17547199726104737
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,4,128,1,float16,float16,511,0.5600416183471679
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,8,128,1,float16,float16,63,0.1824895977973938
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,8,128,1,float16,float16,127,0.26013119220733644
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,1,128,1,float16,float16,7,0.014203199744224548
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,1,128,1,float16,float16,15,0.014340800046920777
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,8,128,1,float16,float16,255,0.43097600936889646
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,1,128,1,float16,float16,63,0.01438560038805008
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,1,128,1,float16,float16,127,0.01693439930677414
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,1,128,1,float16,float16,255,0.021851199865341186
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,1,128,1,float16,float16,1023,0.028100800514221192
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,1,128,1,float16,float16,511,0.02348320037126541
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,1,128,1,float16,float16,2047,0.028963199257850646
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,8,128,1,float16,float16,511,1.0798895835876465
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,1,128,1,float16,float16,8191,0.05496479868888855
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,2,128,1,float16,float16,3,0.01430400013923645
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,2,128,1,float16,float16,1,0.014129599928855896
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,2,128,1,float16,float16,7,0.014239999651908874
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,2,128,1,float16,float16,15,0.014028799533843995
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,2,128,1,float16,float16,31,0.014361600577831268
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,2,128,1,float16,float16,63,0.01430400013923645
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,2,128,1,float16,float16,127,0.016990399360656737
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,2,128,1,float16,float16,511,0.023705600202083586
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,2,128,1,float16,float16,255,0.02171040028333664
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,32,2,128,1,float16,float16,127,0.08678560256958008
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,2,128,1,float16,float16,2047,0.03744960129261017
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,2,128,1,float16,float16,8191,0.08288800120353698
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,4,128,1,float16,float16,1,0.014251199364662171
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,4,128,1,float16,float16,3,0.014214399456977844
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,4,128,1,float16,float16,7,0.0142752006649971
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,4,128,1,float16,float16,15,0.014204800128936768
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,4,128,1,float16,float16,31,0.014289599657058717
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,1,128,1,float16,float16,1,0.014033600687980652
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,4,128,1,float16,float16,63,0.014510400593280792
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,4,128,1,float16,float16,127,0.017097599804401398
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,4,128,1,float16,float16,255,0.021827200055122377
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,4,128,1,float16,float16,511,0.028308799862861632
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,4,128,1,float16,float16,1023,0.037750399112701415
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,1,128,1,float16,float16,3,0.01414559930562973
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,4,128,1,float16,float16,2047,0.05417760014533997
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,4,128,1,float16,float16,4095,0.08142079710960388
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,8,128,1,float16,float16,1,0.014475199580192565
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,8,128,1,float16,float16,7,0.014457599818706512
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,4,128,1,float16,float16,8191,0.13449759483337403
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,8,128,1,float16,float16,3,0.014369599521160126
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,8,128,1,float16,float16,15,0.014496000111103058
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,8,128,1,float16,float16,63,0.014588800072669984
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,8,128,1,float16,float16,127,0.01725119948387146
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,8,128,1,float16,float16,31,0.014556799829006196
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,8,128,1,float16,float16,255,0.02197760045528412
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,8,128,1,float16,float16,511,0.04524799883365631
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,1,128,1,float16,float16,4095,0.0381168007850647
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,8,128,1,float16,float16,2047,0.1101040005683899
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,8,128,1,float16,float16,1023,0.07034239768981934
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,8,128,1,float16,float16,4095,0.1905679941177368
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,32,1,128,1,float16,float16,1,0.05767040252685547
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,32,1,128,1,float16,float16,3,0.05788959860801697
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,8,128,1,float16,float16,8191,0.35299360752105713
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,32,1,128,1,float16,float16,7,0.0580672025680542
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,32,1,128,1,float16,float16,15,0.05839359760284424
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,32,1,128,1,float16,float16,31,0.06172639727592468
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,2,128,1,float16,float16,1023,0.02812480032444
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,32,1,128,1,float16,float16,63,0.07036960124969482
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,32,1,128,1,float16,float16,127,0.08935520052909851
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,32,1,128,1,float16,float16,255,0.132368004322052
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,2,128,1,float16,float16,4095,0.054467201232910156
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,32,2,128,1,float16,float16,3,0.09494079947471619
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,32,2,128,1,float16,float16,1,0.09473599791526795
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,32,2,128,1,float16,float16,7,0.0948527991771698
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,32,2,128,1,float16,float16,15,0.09638400077819824
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,32,2,128,1,float16,float16,31,0.10337120294570923
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,32,2,128,1,float16,float16,63,0.10793759822845458
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,32,4,128,1,float16,float16,1,0.16612800359725952
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,32,4,128,1,float16,float16,3,0.16681760549545288
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,32,4,128,1,float16,float16,7,0.167684805393219
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,32,2,128,1,float16,float16,255,0.23550400733947754
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,32,4,128,1,float16,float16,15,0.17423199415206908
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,32,4,128,1,float16,float16,31,0.17767679691314697
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,32,4,128,1,float16,float16,63,0.18506239652633666
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,32,4,128,1,float16,float16,127,0.26254079341888426
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,32,8,128,1,float16,float16,1,0.30995199680328367
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,32,8,128,1,float16,float16,7,0.3165343999862671
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,32,8,128,1,float16,float16,3,0.31095199584960936
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,32,8,128,1,float16,float16,15,0.32053279876708984
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,32,4,128,1,float16,float16,255,0.4384928226470947
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,32,8,128,1,float16,float16,31,0.3261008024215698
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,32,1,128,1,float16,float16,31,0.01419519931077957
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,32,1,128,1,float16,float16,1,0.10350719690322877
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,32,8,128,1,float16,float16,63,0.3368128061294556
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,32,1,128,1,float16,float16,3,0.10443520545959473
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,32,1,128,1,float16,float16,7,0.10550559759140014
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,32,1,128,1,float16,float16,15,0.105513596534729
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,32,1,128,1,float16,float16,31,0.1070639967918396
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,32,8,128,1,float16,float16,127,0.49248318672180175
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,32,1,128,1,float16,float16,63,0.1138416051864624
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,32,1,128,1,float16,float16,127,0.15241119861602784
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,32,2,128,1,float16,float16,1,0.17473440170288085
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,32,2,128,1,float16,float16,3,0.17631520032882692
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,32,2,128,1,float16,float16,7,0.17933919429779052
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,32,2,128,1,float16,float16,15,0.17958240509033202
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,32,2,128,1,float16,float16,31,0.18129279613494872
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,32,4,128,1,float16,float16,3,0.3199183940887451
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,32,4,128,1,float16,float16,1,0.3192415952682495
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,32,4,128,1,float16,float16,7,0.32268319129943845
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,32,4,128,1,float16,float16,15,0.32710719108581543
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,32,4,128,1,float16,float16,31,0.3307375907897949
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,32,4,128,1,float16,float16,63,0.34266560077667235
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,32,2,128,1,float16,float16,127,0.1483183979988098
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,32,4,128,1,float16,float16,127,0.5021503925323486
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,32,8,128,1,float16,float16,1,0.6069471836090088
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,32,8,128,1,float16,float16,3,0.6076960086822509
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,32,8,128,1,float16,float16,7,0.6162240028381347
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,32,8,128,1,float16,float16,15,0.623633623123169
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,32,8,128,1,float16,float16,31,0.630128002166748
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,1,128,1,float16,float16,1,0.01419840008020401
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,1,128,1,float16,float16,3,0.014241600036621093
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,32,8,128,1,float16,float16,63,0.648364782333374
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,1,128,1,float16,float16,15,0.014452800154685974
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,1,128,1,float16,float16,31,0.014239999651908874
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,1,128,1,float16,float16,127,0.016888000071048737
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,1,128,1,float16,float16,255,0.021639999747276307
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,32,8,128,1,float16,float16,255,0.823038387298584
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,1,128,1,float16,float16,511,0.024427199363708497
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,32,8,128,1,float16,float16,127,0.9567520141601562
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,32,2,128,1,float16,float16,63,0.19062399864196777
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,1,128,1,float16,float16,2047,0.037510401010513304
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,1,128,1,float16,float16,1023,0.028838399052619933
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,32,2,128,1,float16,float16,127,0.2699712038040161
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,1,128,1,float16,float16,4095,0.05617759823799133
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,2,128,1,float16,float16,1,0.01430879980325699
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,2,128,1,float16,float16,3,0.014393599331378936
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,2,128,1,float16,float16,31,0.014401599764823914
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,2,128,1,float16,float16,7,0.01435679942369461
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,2,128,1,float16,float16,63,0.014523200690746307
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,2,128,1,float16,float16,255,0.021849599480628968
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,2,128,1,float16,float16,511,0.028865599632263185
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,2,128,1,float16,float16,1023,0.03796800076961517
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,2,128,1,float16,float16,4095,0.08175039887428284
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,2,128,1,float16,float16,2047,0.05548480153083801
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,4,128,1,float16,float16,1,0.01446239948272705
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,2,128,1,float16,float16,8191,0.13443679809570314
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,4,128,1,float16,float16,3,0.014468799531459808
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,4,128,1,float16,float16,7,0.014596800506114959
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,4,128,1,float16,float16,31,0.014679999649524688
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,4,128,1,float16,float16,63,0.01470080018043518
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,4,128,1,float16,float16,127,0.017323200404644013
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,4,128,1,float16,float16,255,0.022273600101470947
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,4,128,1,float16,float16,511,0.045716801285743715
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,1,128,1,float16,float16,7,0.014180800318717957
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,4,128,1,float16,float16,1023,0.07021920084953308
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,1,128,1,float16,float16,63,0.014574399590492249
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,4,128,1,float16,float16,2047,0.11077120304107665
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,8,128,1,float16,float16,3,0.01626880019903183
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,4,128,1,float16,float16,4095,0.1906335949897766
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,8,128,1,float16,float16,1,0.016388800740242005
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,8,128,1,float16,float16,7,0.01643040031194687
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,8,128,1,float16,float16,31,0.016649599373340606
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,8,128,1,float16,float16,15,0.01650079935789108
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,8,128,1,float16,float16,63,0.016873599588871004
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,8,128,1,float16,float16,127,0.02104319930076599
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,8,128,1,float16,float16,255,0.029230400919914246
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,8,128,1,float16,float16,511,0.06860640048980712
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,8,128,1,float16,float16,1023,0.11294879913330078
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,8,128,1,float16,float16,2047,0.1928671956062317
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,1,128,1,float16,float16,8191,0.08240320086479187
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,1,128,1,float16,float16,1,0.01555200070142746
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,8,128,1,float16,float16,4095,0.3523087978363037
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,1,128,1,float16,float16,3,0.015643200278282164
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,2,128,1,float16,float16,15,0.014660799503326416
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,1,128,1,float16,float16,7,0.015937599539756774
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,2,128,1,float16,float16,127,0.017054399847984313
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,1,128,1,float16,float16,31,0.015744000673294067
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,1,128,1,float16,float16,63,0.016129599511623384
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,1,128,1,float16,float16,15,0.01563999950885773
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,1,128,1,float16,float16,127,0.01855040043592453
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,1,128,1,float16,float16,255,0.02348800003528595
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,1,128,1,float16,float16,511,0.04871520102024078
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,8,128,1,float16,float16,8191,0.6805056095123291
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,1,128,1,float16,float16,1023,0.07417600154876709
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,1,128,1,float16,float16,4095,0.19381920099258423
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,2,128,1,float16,float16,3,0.01775840073823929
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,1,128,1,float16,float16,2047,0.11354559659957886
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,2,128,1,float16,float16,15,0.017772799730300902
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,1,128,1,float16,float16,8191,0.35715200901031496
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,2,128,1,float16,float16,7,0.017715199291706084
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,2,128,1,float16,float16,31,0.01770720034837723
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,2,128,1,float16,float16,63,0.018145599961280824
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,2,128,1,float16,float16,255,0.030526399612426758
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,2,128,1,float16,float16,127,0.022249600291252135
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,2,128,1,float16,float16,511,0.07147200107574463
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,4,128,1,float16,float16,15,0.014510400593280792
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,1,128,1,float16,float16,16383,0.6797200202941894
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,2,128,1,float16,float16,1023,0.11545920372009277
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,4,128,1,float16,float16,1,0.023444800078868865
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,2,128,1,float16,float16,2047,0.19473119974136352
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,4,128,1,float16,float16,3,0.023511999845504762
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,2,128,1,float16,float16,4095,0.3542815923690796
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,4,128,1,float16,float16,15,0.023643200099468232
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,4,128,1,float16,float16,7,0.023401600122451783
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,4,128,1,float16,float16,63,0.024208000302314757
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,4,128,1,float16,float16,31,0.023766399919986726
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,4,128,1,float16,float16,127,0.03061760067939758
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,2,128,1,float16,float16,8191,0.6801119804382324
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,4,128,1,float16,float16,255,0.0472896009683609
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,4,128,1,float16,float16,511,0.0997600018978119
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,4,128,1,float16,float16,1023,0.16630079746246337
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,4,128,1,float16,float16,2047,0.28631680011749266
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,32,4,128,1,float16,float16,8191,0.3554224014282227
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,8,128,1,float16,float16,1,0.03107680082321167
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,2,128,1,float16,float16,16383,1.3281439781188964
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,4,128,1,float16,float16,4095,0.5252079963684082
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,8,128,1,float16,float16,3,0.03125919997692108
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,8,128,1,float16,float16,7,0.031222400069236756
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,8,128,1,float16,float16,15,0.03138880133628845
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,8,128,1,float16,float16,31,0.031723201274871826
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,8,128,1,float16,float16,127,0.04652160108089447
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,4,128,1,float16,float16,8191,1.0139887809753418
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,8,128,1,float16,float16,255,0.07154240012168885
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,8,128,1,float16,float16,511,0.15783840417861938
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,8,128,1,float16,float16,1023,0.2664383888244629
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,2,128,1,float16,float16,1,0.017423999309539796
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,1,128,1,float16,float16,1,0.013846400380134582
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,8,128,1,float16,float16,2047,0.466864013671875
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,1,128,1,float16,float16,3,0.013823999464511872
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,1,128,1,float16,float16,15,0.013939200341701508
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,1,128,1,float16,float16,7,0.013929599523544311
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,1,128,1,float16,float16,31,0.013915200531482697
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,1,128,1,float16,float16,127,0.016510400176048278
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,4,128,1,float16,float16,16383,1.9830303192138672
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,1,128,1,float16,float16,63,0.0141184002161026
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,1,128,1,float16,float16,255,0.02133280038833618
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,8,128,1,float16,float16,4095,0.8666239738464355
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,1,128,1,float16,float16,1023,0.026846399903297423
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,1,128,1,float16,float16,2047,0.027347201108932497
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,1,128,1,float16,float16,511,0.022759999334812164
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,1,128,1,float16,float16,8191,0.030092799663543703
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,1,128,1,float16,float16,16383,0.034283199906349184
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,1,128,1,float16,float16,4095,0.02898559868335724
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,2,128,1,float16,float16,1,0.014144000411033631
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,2,128,1,float16,float16,7,0.013868799805641175
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,2,128,1,float16,float16,3,0.013868799805641175
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,2,128,1,float16,float16,15,0.01414719969034195
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,2,128,1,float16,float16,63,0.014071999490261078
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,2,128,1,float16,float16,31,0.014280000329017639
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,2,128,1,float16,float16,127,0.016540800034999848
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,2,128,1,float16,float16,255,0.021777600049972534
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,8,128,1,float16,float16,8191,1.683465576171875
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,2,128,1,float16,float16,1023,0.026940798759460448
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,2,128,1,float16,float16,2047,0.02715040147304535
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,2,128,1,float16,float16,4095,0.028596800565719605
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,4,128,1,float16,float16,1,0.014033600687980652
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,4,128,1,float16,float16,3,0.014051200449466705
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,2,128,1,float16,float16,8191,0.03181760013103485
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,2,128,1,float16,float16,16383,0.040463998913764954
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,4,128,1,float16,float16,15,0.01406559944152832
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,4,128,1,float16,float16,31,0.014587199687957764
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,4,128,1,float16,float16,63,0.014268800616264343
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,4,128,1,float16,float16,127,0.016651199758052827
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,4,128,1,float16,float16,255,0.02220800071954727
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,4,128,1,float16,float16,511,0.02282399982213974
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,4,128,1,float16,float16,1023,0.027105599641799927
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,4,128,1,float16,float16,2047,0.02818399965763092
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,4,128,1,float16,float16,4095,0.029110398888587952
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,8,128,1,float16,float16,63,0.032471999526023865
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,4,128,1,float16,float16,8191,0.03837119936943054
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,8,128,1,float16,float16,1,0.014227199554443359
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,4,128,1,float16,float16,16383,0.05678240060806274
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,8,128,1,float16,float16,15,0.014148800075054169
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,8,128,1,float16,float16,7,0.014169600605964661
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,8,128,1,float16,float16,31,0.014364799857139588
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,8,128,1,float16,float16,63,0.014103999733924866
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,8,128,1,float16,float16,127,0.0168272003531456
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,8,128,1,float16,float16,255,0.022019200026988983
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,8,128,1,float16,float16,1023,0.027375999093055724
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,8,128,1,float16,float16,511,0.02263039946556091
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,8,128,1,float16,float16,4095,0.03731200098991394
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,8,128,1,float16,float16,2047,0.028648000955581666
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,8,128,1,float16,float16,8191,0.054104000329971313
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,1,128,1,float16,float16,1,0.014097599685192109
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,8,128,1,float16,float16,16383,0.0825984001159668
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,16,8,128,1,float16,float16,16383,3.4381439208984377
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,1,128,1,float16,float16,7,0.014212800562381745
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,1,128,1,float16,float16,3,0.013945600390434265
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,1,128,1,float16,float16,15,0.014204800128936768
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,1,128,1,float16,float16,63,0.014425599575042724
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,1,128,1,float16,float16,127,0.01669120043516159
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,1,128,1,float16,float16,31,0.015196800231933594
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,1,128,1,float16,float16,255,0.02159679979085922
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,1,128,1,float16,float16,511,0.023496000468730925
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,1,128,1,float16,float16,1023,0.027049601078033447
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,1,128,1,float16,float16,2047,0.029444798827171326
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,1,128,1,float16,float16,4095,0.0296640008687973
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,1,128,1,float16,float16,8191,0.030884799361228944
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,2,128,1,float16,float16,1,0.014416000247001648
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,1,128,1,float16,float16,16383,0.03912799954414368
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,2,128,1,float16,float16,3,0.014001600444316864
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,2,128,1,float16,float16,511,0.022484800219535826
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,2,128,1,float16,float16,31,0.014095999300479889
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,2,128,1,float16,float16,15,0.014638400077819825
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,2,128,1,float16,float16,63,0.014185599982738495
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,2,128,1,float16,float16,255,0.021617600321769716
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,2,128,1,float16,float16,127,0.016711999475955964
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,2,128,1,float16,float16,2047,0.02736000120639801
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,2,128,1,float16,float16,4095,0.02889440059661865
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,2,128,1,float16,float16,8191,0.03762879967689514
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,4,128,1,float16,float16,7,0.013982400298118591
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,2,128,1,float16,float16,16383,0.05228959918022156
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,4,128,1,float16,float16,1,0.013967999815940857
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,4,128,1,float16,float16,15,0.01387999951839447
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,4,128,1,float16,float16,7,0.013923199474811554
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,4,128,1,float16,float16,31,0.013911999762058258
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,4,128,1,float16,float16,127,0.01656160056591034
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,4,128,1,float16,float16,63,0.014044800400733947
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,4,128,1,float16,float16,255,0.02146400064229965
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,4,128,1,float16,float16,511,0.022708800435066224
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,16,8,128,1,float16,float16,3,0.014295999705791474
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,4,128,1,float16,float16,1023,0.027105599641799927
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,4,128,1,float16,float16,2047,0.028177601099014283
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,4,128,1,float16,float16,8191,0.05421919822692871
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,8,128,1,float16,float16,1,0.014179199934005737
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,4,128,1,float16,float16,16383,0.07853919863700867
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,8,128,1,float16,float16,3,0.01412159949541092
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,8,128,1,float16,float16,7,0.014140799641609192
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,8,128,1,float16,float16,31,0.014268800616264343
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,8,128,1,float16,float16,63,0.014350399374961853
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,8,128,1,float16,float16,127,0.016862399876117706
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,8,128,1,float16,float16,255,0.021804800629615782
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,8,128,1,float16,float16,511,0.02346239984035492
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,8,128,1,float16,float16,1023,0.02775520086288452
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,8,128,1,float16,float16,2047,0.03659679889678955
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,8,128,1,float16,float16,4095,0.05347999930381775
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,2,128,1,float16,float16,7,0.014483200013637542
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,8,128,1,float16,float16,8191,0.08122720122337342
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,1,128,1,float16,float16,1,0.014643199741840363
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,2,128,1,float16,float16,511,0.023628799617290495
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,1,128,1,float16,float16,3,0.014777599275112152
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,1,128,1,float16,float16,15,0.01488800048828125
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,1,128,1,float16,float16,7,0.015068799257278442
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,1,128,1,float16,float16,31,0.014892800152301789
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,1,128,1,float16,float16,63,0.015188799798488617
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,1,128,1,float16,float16,127,0.01738079935312271
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,1,128,1,float16,float16,511,0.029425600171089174
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,1,128,1,float16,float16,255,0.02255679965019226
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,1,128,1,float16,float16,1023,0.038339200615882876
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,1,128,1,float16,float16,2047,0.05571680068969727
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,1,128,1,float16,float16,4095,0.08256800174713134
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,4,128,1,float16,float16,4095,0.037303999066352844
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,2,128,1,float16,float16,1,0.015105600655078887
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,2,128,1,float16,float16,3,0.015046399831771851
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,2,128,1,float16,float16,7,0.01510079950094223
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,1,128,1,float16,float16,8191,0.13691359758377075
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,1,128,1,float16,float16,16383,0.3530512094497681
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,2,128,1,float16,float16,15,0.0151296004652977
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,8,128,1,float16,float16,15,0.014323200285434722
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,2,128,1,float16,float16,31,0.015177600085735321
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,2,128,1,float16,float16,127,0.017788800597190856
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,2,128,1,float16,float16,255,0.022878399491310118
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,2,128,1,float16,float16,63,0.015113599598407745
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,2,128,1,float16,float16,511,0.04673280119895935
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,2,128,1,float16,float16,1023,0.07109439969062806
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,2,128,1,float16,float16,2047,0.112009596824646
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,2,128,1,float16,float16,4095,0.1921679973602295
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,4,128,1,float16,float16,3,0.016927999258041383
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,4,128,1,float16,float16,7,0.017136000096797943
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,2,128,1,float16,float16,8191,0.3576575994491577
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,4,128,1,float16,float16,15,0.017000000178813934
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,2,128,1,float16,float16,16383,0.6778656005859375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,4,128,1,float16,float16,63,0.01717599928379059
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,4,128,1,float16,float16,31,0.017182399332523347
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,4,128,1,float16,float16,255,0.02987839877605438
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,4,128,1,float16,float16,127,0.02166240066289902
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,4,128,1,float16,float16,511,0.07065759897232056
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,2,128,1,float16,float16,1023,0.02707839906215668
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,4,128,1,float16,float16,1023,0.1139631986618042
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,4,128,1,float16,float16,2047,0.19530080556869506
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,4,128,1,float16,float16,3,0.01406559944152832
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,8,128,1,float16,float16,3,0.022942399978637694
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,4,128,1,float16,float16,4095,0.3559344053268433
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,8,128,1,float16,float16,7,0.02284960001707077
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,8,128,1,float16,float16,31,0.02314720004796982
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,8,128,1,float16,float16,15,0.02298559993505478
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,8,128,1,float16,float16,63,0.023446400463581086
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,4,128,1,float16,float16,8191,0.6800096035003662
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,8,128,1,float16,float16,127,0.02975200116634369
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,8,128,1,float16,float16,255,0.045817598700523376
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,8,128,1,float16,float16,511,0.09867200255393982
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,8,128,1,float16,float16,1023,0.16474560499191285
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,8,128,1,float16,float16,2047,0.2844048023223877
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,4,128,1,float16,float16,16383,1.3263808250427247
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,8,128,1,float16,float16,4095,0.5231455802917481
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,1,128,1,float16,float16,1,0.01395999938249588
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,1,128,1,float16,float16,7,0.013918399810791016
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,1,128,1,float16,float16,3,0.013804799318313599
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,1,128,1,float16,float16,15,0.013988800346851349
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,1,128,1,float16,float16,31,0.01393280029296875
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,4,128,1,float16,float16,1,0.016888000071048737
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,1,128,1,float16,float16,63,0.014076800644397735
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,1,128,1,float16,float16,255,0.02154559940099716
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,1,128,1,float16,float16,1023,0.027289599180221558
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,8,128,1,float16,float16,8191,1.0127152442932128
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,1,128,1,float16,float16,511,0.02290239930152893
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,1,128,1,float16,float16,2047,0.028279998898506166
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,1,128,1,float16,float16,4095,0.029316800832748412
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,1,128,1,float16,float16,16383,0.05318400263786316
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,1,128,1,float16,float16,8191,0.03809120059013367
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,2,128,1,float16,float16,1,0.01398559957742691
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,2,128,1,float16,float16,7,0.013872000575065612
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,2,128,1,float16,float16,15,0.01403679996728897
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,2,128,1,float16,float16,31,0.013972799479961395
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,2,128,1,float16,float16,63,0.014151999354362487
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,16,8,128,1,float16,float16,16383,0.21393120288848877
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,2,128,1,float16,float16,127,0.016622400283813475
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,2,128,1,float16,float16,255,0.021481600403785706
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,2,128,1,float16,float16,511,0.02324160039424896
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,2,128,1,float16,float16,1023,0.027291199564933775
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,8,128,1,float16,float16,16383,1.9856576919555664
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,2,128,1,float16,float16,4095,0.03732799887657166
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,2,128,1,float16,float16,2047,0.028360000252723692
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,16,8,128,1,float16,float16,1,0.022935999929904936
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,2,128,1,float16,float16,8191,0.054232001304626465
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,4,128,1,float16,float16,7,0.014273600280284881
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,4,128,1,float16,float16,3,0.014129599928855896
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,2,128,1,float16,float16,16383,0.0795408010482788
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,4,128,1,float16,float16,31,0.014366400241851807
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,4,128,1,float16,float16,15,0.014220799505710601
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,4,128,1,float16,float16,63,0.014318400621414184
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,4,128,1,float16,float16,255,0.021913599967956544
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,4,128,1,float16,float16,127,0.01703200042247772
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,4,128,1,float16,float16,1023,0.02786400020122528
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,4,128,1,float16,float16,2047,0.03659360110759735
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,4,128,1,float16,float16,4095,0.05486559867858887
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,4,128,1,float16,float16,8191,0.08128160238265991
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,4,128,1,float16,float16,16383,0.21340639591217042
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,8,128,1,float16,float16,3,0.014388799667358398
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,8,128,1,float16,float16,7,0.014271999895572662
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,8,128,1,float16,float16,15,0.014478400349617004
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,8,128,1,float16,float16,31,0.01427839994430542
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,8,128,1,float16,float16,63,0.014590400457382201
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,8,128,1,float16,float16,127,0.017057600617408752
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,8,128,1,float16,float16,255,0.02200479954481125
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,8,128,1,float16,float16,511,0.028196799755096435
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,8,128,1,float16,float16,1023,0.03711999952793121
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,8,128,1,float16,float16,2047,0.053487998247146604
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,1,128,1,float16,float16,127,0.016491200029850005
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,8,128,1,float16,float16,4095,0.08091679811477662
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,1,128,1,float16,float16,1,0.018632000684738158
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,8,128,1,float16,float16,8191,0.1348207950592041
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,1,128,1,float16,float16,3,0.018680000305175783
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,8,128,1,float16,float16,16383,0.3500880002975464
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,1,128,1,float16,float16,7,0.018692800402641298
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,1,128,1,float16,float16,63,0.019169600307941438
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,2,128,1,float16,float16,3,0.014043200016021728
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,1,128,1,float16,float16,127,0.023489600419998168
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,1,128,1,float16,float16,255,0.03158079981803894
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,1,128,1,float16,float16,511,0.07402080297470093
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,1,128,1,float16,float16,1023,0.11678719520568848
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,1,128,1,float16,float16,2047,0.19722720384597778
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,4,128,1,float16,float16,1,0.014215999841690063
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,1,128,1,float16,float16,4095,0.3554111957550049
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,2,128,1,float16,float16,1,0.024519999325275422
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,2,128,1,float16,float16,7,0.024721600115299225
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,2,128,1,float16,float16,3,0.024691200256347655
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,2,128,1,float16,float16,15,0.024692800641059876
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,2,128,1,float16,float16,63,0.02523840069770813
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,2,128,1,float16,float16,31,0.024910399317741395
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,2,128,1,float16,float16,127,0.031483200192451474
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,1,128,1,float16,float16,8191,0.6833327770233154
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,2,128,1,float16,float16,255,0.04791840016841888
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,2,128,1,float16,float16,511,0.1018831968307495
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,8,128,1,float16,float16,1,0.01454080045223236
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,2,128,1,float16,float16,1023,0.1690559983253479
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,2,128,1,float16,float16,2047,0.2863584041595459
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,4,128,1,float16,float16,1,0.03223679959774017
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,4,128,1,float16,float16,3,0.03203519880771637
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,4,128,1,float16,float16,7,0.0323199987411499
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,4,128,1,float16,float16,15,0.032308799028396604
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,2,128,1,float16,float16,4095,0.5273920059204101
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,4,128,1,float16,float16,31,0.03271200060844422
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,4,128,1,float16,float16,63,0.03325439989566803
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,4,128,1,float16,float16,127,0.04940159916877747
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,4,128,1,float16,float16,255,0.07212160229682922
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,4,128,1,float16,float16,511,0.15777280330657958
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,1,128,1,float16,float16,15,0.0186256006360054
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,2,128,1,float16,float16,8191,1.015878391265869
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,1,128,1,float16,float16,31,0.0188400000333786
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,4,128,1,float16,float16,1023,0.2703824043273926
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,8,128,1,float16,float16,1,0.0499455988407135
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,8,128,1,float16,float16,3,0.04986880123615265
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,8,128,1,float16,float16,7,0.050361597537994386
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,4,128,1,float16,float16,2047,0.46921281814575194
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,8,128,1,float16,float16,15,0.05018240213394165
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,8,128,1,float16,float16,31,0.051614397764205934
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,8,128,1,float16,float16,63,0.059062397480010985
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,8,128,1,float16,float16,255,0.1213871955871582
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,8,128,1,float16,float16,127,0.08000479936599732
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,4,128,1,float16,float16,4095,0.8655856132507325
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,8,128,1,float16,float16,511,0.29501121044158934
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,16,4,128,1,float16,float16,511,0.023555199801921844
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,1,128,1,float16,float16,1,0.026280000805854797
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,8,128,1,float16,float16,1023,0.517142391204834
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,1,128,1,float16,float16,3,0.02643679976463318
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,1,128,1,float16,float16,7,0.026372799277305604
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,1,128,1,float16,float16,15,0.02654080092906952
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,1,128,1,float16,float16,31,0.026814401149749756
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,4,128,1,float16,float16,8191,1.6952127456665038
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,1,128,1,float16,float16,63,0.02693440020084381
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,1,128,1,float16,float16,127,0.03352000117301941
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,1,128,1,float16,float16,255,0.050076800584793094
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,1,128,1,float16,float16,1023,0.17101919651031494
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,1,128,1,float16,float16,511,0.11076639890670777
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,1,128,1,float16,float16,2047,0.2910912036895752
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,2,128,1,float16,float16,3,0.03373279869556427
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,2,128,1,float16,float16,1,0.03400799930095673
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,2,128,1,float16,float16,7,0.03531680107116699
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,8,128,1,float16,float16,4095,1.731884765625
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,2,128,1,float16,float16,31,0.03440319895744324
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,2,128,1,float16,float16,63,0.035129600763320924
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,2,128,1,float16,float16,255,0.07462239861488343
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,2,128,1,float16,float16,511,0.16301759481430053
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,4,128,1,float16,float16,1,0.05159519910812378
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,4,128,1,float16,float16,3,0.051635199785232545
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,4,128,1,float16,float16,7,0.051712000370025636
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,2,128,1,float16,float16,2047,0.48349919319152834
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,4,128,1,float16,float16,15,0.05201759934425354
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,4,128,1,float16,float16,63,0.06041120290756226
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,4,128,1,float16,float16,31,0.05370240211486817
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,4,128,1,float16,float16,127,0.08071200251579284
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,4,128,1,float16,float16,255,0.12800159454345703
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,4,128,1,float16,float16,511,0.2975856065750122
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,8,128,1,float16,float16,8191,3.4562816619873047
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,16,8,128,1,float16,float16,2047,0.9166192054748535
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,4,128,1,float16,float16,1023,0.5352496147155762
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,8,128,1,float16,float16,1,0.09522879719734192
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,8,128,1,float16,float16,3,0.08953760266304016
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,8,128,1,float16,float16,7,0.09047039747238159
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,8,128,1,float16,float16,31,0.09331200122833253
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,8,128,1,float16,float16,63,0.10155999660491943
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,8,128,1,float16,float16,127,0.13967200517654418
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,2,128,1,float16,float16,15,0.034443199634552
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,4,128,1,float16,float16,2047,0.9164752006530762
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,2,128,1,float16,float16,127,0.05284960269927978
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,8,128,1,float16,float16,255,0.22691841125488282
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,1,128,1,float16,float16,1,0.039024001359939574
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,1,128,1,float16,float16,3,0.03904959857463837
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,1,128,1,float16,float16,7,0.03871360123157501
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,2,128,1,float16,float16,1023,0.27272961139678953
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,1,128,1,float16,float16,15,0.03940480053424835
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,1,128,1,float16,float16,31,0.03931039869785309
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,8,128,1,float16,float16,511,0.5539792060852051
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,1,128,1,float16,float16,255,0.07768800258636474
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,1,128,1,float16,float16,63,0.0404992014169693
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,8,128,1,float16,float16,1023,0.9748512268066406
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,1,128,1,float16,float16,511,0.1727839946746826
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,2,128,1,float16,float16,1,0.05605440139770508
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,1,128,1,float16,float16,1023,0.2819087982177734
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,2,128,1,float16,float16,31,0.056846398115158084
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,2,128,1,float16,float16,63,0.06541919708251953
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,2,128,1,float16,float16,127,0.08500639796257019
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,2,128,1,float16,float16,255,0.12881439924240112
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,8,128,1,float16,float16,2047,1.7755359649658202
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,2,128,1,float16,float16,511,0.3089679956436157
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,2,128,1,float16,float16,1023,0.5307263851165771
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,4,128,1,float16,float16,1,0.10056320428848267
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,4,128,1,float16,float16,3,0.09325760006904601
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,4,128,1,float16,float16,7,0.09353600144386291
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,16,8,128,1,float16,float16,15,0.09594720005989074
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,4,128,1,float16,float16,63,0.10566560029983521
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,4,128,1,float16,float16,31,0.09702240228652954
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,4,128,1,float16,float16,127,0.14525279998779297
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,4,128,1,float16,float16,255,0.23260960578918458
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,4,128,1,float16,float16,511,0.5570159912109375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,8,128,1,float16,float16,1,0.16462719440460205
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,1,128,1,float16,float16,127,0.056953597068786624
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,8,128,1,float16,float16,7,0.16508159637451172
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,8,128,1,float16,float16,3,0.16443359851837158
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,8,128,1,float16,float16,15,0.16652640104293823
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,8,128,1,float16,float16,31,0.17373119592666625
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,2,128,1,float16,float16,3,0.05616000294685364
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,8,128,1,float16,float16,63,0.18260159492492675
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,2,128,1,float16,float16,7,0.05630559921264648
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,8,128,1,float16,float16,127,0.2569040060043335
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,1,128,1,float16,float16,1,0.014273600280284881
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,1,128,1,float16,float16,3,0.014135999977588654
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,8,128,1,float16,float16,255,0.4245279788970947
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,1,128,1,float16,float16,15,0.01417279988527298
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,1,128,1,float16,float16,31,0.014124800264835358
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,1,128,1,float16,float16,63,0.014215999841690063
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,1,128,1,float16,float16,127,0.01685280054807663
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,1,128,1,float16,float16,255,0.021585600078105928
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,1,128,1,float16,float16,511,0.023211200535297394
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,1,128,1,float16,float16,1023,0.027564799785614012
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,4,128,1,float16,float16,15,0.10119999647140503
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,1,128,1,float16,float16,2047,0.02863680124282837
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,1,128,1,float16,float16,4095,0.03775199949741363
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,1,128,1,float16,float16,8191,0.05475999712944031
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,1,128,1,float16,float16,16383,0.0799727976322174
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,2,128,1,float16,float16,1,0.014059199392795563
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,2,128,1,float16,float16,7,0.014086399972438813
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,2,128,1,float16,float16,3,0.01422400027513504
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,2,128,1,float16,float16,15,0.013990400731563568
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,2,128,1,float16,float16,31,0.014326399564743042
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,2,128,1,float16,float16,63,0.014153599739074707
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,2,128,1,float16,float16,127,0.01690080016851425
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,8,128,1,float16,float16,1023,1.9425615310668944
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,2,128,1,float16,float16,255,0.02173759937286377
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,2,128,1,float16,float16,511,0.023628799617290495
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,2,128,1,float16,float16,2047,0.03700799942016601
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,2,128,1,float16,float16,1023,0.028867200016975403
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,2,128,1,float16,float16,8191,0.08202720284461976
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,4,128,1,float16,float16,1,0.01419679969549179
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,2,128,1,float16,float16,16383,0.2199023962020874
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,4,128,1,float16,float16,3,0.014164799451828003
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,4,128,1,float16,float16,15,0.014348800480365752
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,4,128,1,float16,float16,7,0.014510400593280792
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,4,128,1,float16,float16,31,0.014361600577831268
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,4,128,1,float16,float16,63,0.014779199659824372
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,4,128,1,float16,float16,127,0.01701440066099167
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,4,128,1,float16,float16,1023,0.985812759399414
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,4,128,1,float16,float16,511,0.02818399965763092
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,4,128,1,float16,float16,255,0.02221119999885559
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,4,128,1,float16,float16,1023,0.038104000687599185
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,4,128,1,float16,float16,2047,0.053681600093841556
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,4,128,1,float16,float16,4095,0.08268160223960877
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,1,128,1,float16,float16,7,0.01422560065984726
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,4,128,1,float16,float16,8191,0.13549760580062867
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,8,128,1,float16,float16,3,0.014591999351978302
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,8,128,1,float16,float16,1,0.014585599303245544
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,8,128,1,float16,float16,15,0.014560000598430633
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,4,128,1,float16,float16,16383,0.3515952110290527
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,8,128,1,float16,float16,7,0.014420799911022186
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,8,128,1,float16,float16,31,0.014595200121402741
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,8,128,1,float16,float16,255,0.02210880070924759
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,8,128,1,float16,float16,511,0.046035200357437134
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,8,128,1,float16,float16,511,1.0812496185302733
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,8,128,1,float16,float16,1023,0.07045919895172119
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,8,128,1,float16,float16,2047,0.11208640336990357
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,8,128,1,float16,float16,4095,0.1914463996887207
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,8,128,1,float16,float16,8191,0.3539680004119873
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,1,128,1,float16,float16,1,0.056724798679351804
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,1,128,1,float16,float16,7,0.0570527970790863
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,1,128,1,float16,float16,3,0.05679200291633606
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,1,128,1,float16,float16,15,0.05748800039291382
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,1,128,1,float16,float16,31,0.05809599757194519
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,8,128,1,float16,float16,16383,0.6876239776611328
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,2,128,1,float16,float16,4095,0.05391200184822083
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,1,128,1,float16,float16,63,0.06754720211029053
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,1,128,1,float16,float16,127,0.08753439784049988
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,2,128,1,float16,float16,1,0.09397600293159485
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,2,128,1,float16,float16,3,0.0940064013004303
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,1,128,1,float16,float16,511,0.3187983989715576
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,2,128,1,float16,float16,7,0.09431359767913819
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,2,128,1,float16,float16,15,0.09459199905395507
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,16,2,128,1,float16,float16,15,0.05666239857673645
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,2,128,1,float16,float16,31,0.10089759826660157
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,2,128,1,float16,float16,63,0.1062608003616333
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,2,128,1,float16,float16,127,0.14436960220336914
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,2,128,1,float16,float16,255,0.2312096118927002
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,4,128,1,float16,float16,1,0.16607040166854858
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,4,128,1,float16,float16,3,0.16589759588241576
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,4,128,1,float16,float16,7,0.16681439876556398
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,4,128,1,float16,float16,15,0.16916160583496093
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,4,128,1,float16,float16,31,0.17595200538635253
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,2,128,1,float16,float16,511,0.5688943862915039
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,8,128,1,float16,float16,63,0.014620800316333771
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,16,8,128,1,float16,float16,127,0.017206400632858276
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,4,128,1,float16,float16,63,0.1832576036453247
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,4,128,1,float16,float16,127,0.2599119901657104
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,8,128,1,float16,float16,1,0.3113584041595459
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,8,128,1,float16,float16,3,0.30971360206604004
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,4,128,1,float16,float16,255,0.4316448211669922
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,8,128,1,float16,float16,7,0.3132272005081177
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,8,128,1,float16,float16,15,0.32273919582366944
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,8,128,1,float16,float16,31,0.32647199630737306
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,8,128,1,float16,float16,63,0.3354543924331665
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,16,1,128,1,float16,float16,1,0.09548159837722778
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,8,128,1,float16,float16,127,0.48666558265686033
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,16,1,128,1,float16,float16,7,0.09518880248069764
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,4,128,1,float16,float16,511,1.0883456230163575
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,16,1,128,1,float16,float16,15,0.09820960164070129
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,16,1,128,1,float16,float16,31,0.10298399925231934
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,16,1,128,1,float16,float16,63,0.10838559865951539
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,8,128,1,float16,float16,255,0.8237983703613281
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,16,1,128,1,float16,float16,127,0.14814079999923707
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,16,1,128,1,float16,float16,255,0.23069279193878173
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,16,2,128,1,float16,float16,3,0.16681439876556398
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,1,128,1,float16,float16,255,0.13112000226974488
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,16,2,128,1,float16,float16,15,0.17609920501708984
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,16,2,128,1,float16,float16,63,0.18641599416732788
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,16,2,128,1,float16,float16,31,0.17984960079193116
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,16,2,128,1,float16,float16,127,0.262719988822937
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,16,4,128,1,float16,float16,1,0.31174240112304685
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,16,2,128,1,float16,float16,255,0.4436672210693359
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,16,4,128,1,float16,float16,3,0.3161151885986328
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,16,4,128,1,float16,float16,7,0.31887519359588623
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,16,8,128,1,float16,float16,511,2.1338079452514647
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,16,4,128,1,float16,float16,15,0.3246767997741699
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,16,4,128,1,float16,float16,63,0.33909759521484373
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,16,4,128,1,float16,float16,127,0.5079504013061523
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,16,8,128,1,float16,float16,1,0.6068336009979248
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,16,1,128,1,float16,float16,3,0.09519680142402649
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,16,4,128,1,float16,float16,255,0.8313072204589844
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,16,8,128,1,float16,float16,3,0.6167823791503906
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,16,8,128,1,float16,float16,7,0.612668800354004
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,16,8,128,1,float16,float16,15,0.6220047950744629
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,16,2,128,1,float16,float16,1,0.16764320135116578
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,16,8,128,1,float16,float16,31,0.6465487957000733
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,1,128,1,float16,float16,1,0.014123199880123139
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,16,2,128,1,float16,float16,7,0.17222239971160888
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,1,128,1,float16,float16,3,0.01446399986743927
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,1,128,1,float16,float16,7,0.014126400649547576
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,1,128,1,float16,float16,15,0.014220799505710601
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,16,8,128,1,float16,float16,63,0.6607215881347657
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,1,128,1,float16,float16,31,0.01451520025730133
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,1,128,1,float16,float16,127,0.01703200042247772
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,1,128,1,float16,float16,511,0.02420320063829422
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,1,128,1,float16,float16,255,0.02194399982690811
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,16,8,128,1,float16,float16,127,0.9503007888793945
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,1,128,1,float16,float16,2047,0.03797119855880737
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,1,128,1,float16,float16,4095,0.05586240291595459
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,1,128,1,float16,float16,8191,0.0851472020149231
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,2,128,1,float16,float16,1,0.01451520025730133
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,2,128,1,float16,float16,7,0.014539200067520141
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,2,128,1,float16,float16,3,0.01485760062932968
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,16,4,128,1,float16,float16,31,0.32692480087280273
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,1,128,1,float16,float16,16383,0.2168351888656616
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,2,128,1,float16,float16,15,0.014511999487876893
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,2,128,1,float16,float16,31,0.014814400672912597
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,2,128,1,float16,float16,127,0.017272000014781953
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,2,128,1,float16,float16,511,0.02900800108909607
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,2,128,1,float16,float16,1023,0.03800959885120392
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,16,8,128,1,float16,float16,255,1.6536079406738282
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,2,128,1,float16,float16,2047,0.05607519745826721
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,2,128,1,float16,float16,4095,0.081905597448349
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,4,128,1,float16,float16,1,0.014475199580192565
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,2,128,1,float16,float16,8191,0.14998079538345338
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,4,128,1,float16,float16,3,0.014448000490665436
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,4,128,1,float16,float16,7,0.0160848006606102
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,4,128,1,float16,float16,15,0.014510400593280792
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,4,128,1,float16,float16,63,0.014696000516414643
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,4,128,1,float16,float16,31,0.015320000052452088
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,2,128,1,float16,float16,16383,0.3568943977355957
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,4,128,1,float16,float16,127,0.0174575999379158
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,4,128,1,float16,float16,255,0.023350399732589722
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,4,128,1,float16,float16,511,0.04686880111694336
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,4,128,1,float16,float16,2047,0.11665920019149781
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,4,128,1,float16,float16,4095,0.19321440458297728
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,1,128,1,float16,float16,63,0.014267200231552124
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,8,128,1,float16,float16,1,0.016103999316692354
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,8,128,1,float16,float16,3,0.016329599916934966
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,4,128,1,float16,float16,8191,0.36696159839630127
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,1,128,1,float16,float16,1023,0.028305599093437196
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,8,128,1,float16,float16,7,0.01626719981431961
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,8,128,1,float16,float16,31,0.016641600430011748
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,8,128,1,float16,float16,63,0.01685599982738495
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,8,128,1,float16,float16,15,0.016782400012016297
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,8,128,1,float16,float16,127,0.021294400095939636
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,8,128,1,float16,float16,255,0.029425600171089174
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,4,128,1,float16,float16,16383,0.6849840164184571
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,8,128,1,float16,float16,1023,0.11308319568634033
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,8,128,1,float16,float16,511,0.0700655996799469
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,8,128,1,float16,float16,2047,0.19290879964828492
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,8,128,1,float16,float16,4095,0.3579312086105347
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,1,128,1,float16,float16,3,0.015707199275493623
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,2,128,1,float16,float16,63,0.014379200339317322
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,2,128,1,float16,float16,255,0.022180800139904023
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,1,128,1,float16,float16,15,0.01568479984998703
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,1,128,1,float16,float16,7,0.015835200250148774
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,1,128,1,float16,float16,31,0.015531200170516967
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,1,128,1,float16,float16,63,0.015939199924468996
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,1,128,1,float16,float16,127,0.01841920018196106
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,1,128,1,float16,float16,255,0.02348479926586151
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,1,128,1,float16,float16,511,0.04729920029640198
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,8,128,1,float16,float16,8191,0.6808784008026123
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,1,128,1,float16,float16,1023,0.07256960272789001
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,1,128,1,float16,float16,2047,0.11277920007705688
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,2,128,1,float16,float16,1,0.01736160069704056
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,2,128,1,float16,float16,3,0.017523199319839478
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,1,128,1,float16,float16,4095,0.19287840127944947
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,2,128,1,float16,float16,7,0.017576000094413756
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,1,128,1,float16,float16,8191,0.3553647994995117
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,2,128,1,float16,float16,15,0.017844800651073457
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,2,128,1,float16,float16,31,0.01759359985589981
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,2,128,1,float16,float16,63,0.017985600233078002
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,8,128,1,float16,float16,16383,1.3401200294494628
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,2,128,1,float16,float16,127,0.022368000447750093
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,16,4,128,1,float16,float16,1023,0.07114719748497009
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,2,128,1,float16,float16,511,0.07189599871635437
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,2,128,1,float16,float16,1023,0.11713759899139405
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,2,128,1,float16,float16,2047,0.1950767993927002
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,2,128,1,float16,float16,4095,0.3542975902557373
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,1,128,1,float16,float16,32767,1.3267583847045898
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,4,128,1,float16,float16,1,0.023395200073719025
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,2,128,1,float16,float16,8191,0.6807024002075195
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,4,128,1,float16,float16,3,0.023263999819755556
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,4,128,1,float16,float16,7,0.023503999412059783
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,4,128,1,float16,float16,15,0.023556800186634065
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,4,128,1,float16,float16,31,0.02377600073814392
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,4,128,1,float16,float16,63,0.02401760071516037
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,4,128,1,float16,float16,127,0.03059200048446655
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,4,128,1,float16,float16,255,0.047809600830078125
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,2,128,1,float16,float16,16383,1.3256671905517579
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,1,128,1,float16,float16,1,0.015758399665355683
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,4,128,1,float16,float16,511,0.10024640560150147
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,4,128,1,float16,float16,1023,0.1668287992477417
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,4,128,1,float16,float16,2047,0.2865839958190918
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,4,128,1,float16,float16,4095,0.5246831893920898
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,1,128,1,float16,float16,1,0.013903999328613281
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,1,128,1,float16,float16,3,0.013964800536632538
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,1,128,1,float16,float16,7,0.013939200341701508
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,1,128,1,float16,float16,16383,0.6806591987609864
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,2,128,1,float16,float16,255,0.030556800961494445
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,4,128,1,float16,float16,8191,1.0135663986206054
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,1,128,1,float16,float16,15,0.013991999626159667
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,2,128,1,float16,float16,32767,2.614067268371582
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,1,128,1,float16,float16,63,0.014144000411033631
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,1,128,1,float16,float16,255,0.021214400231838227
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,1,128,1,float16,float16,1023,0.026555201411247252
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,1,128,1,float16,float16,2047,0.026712000370025635
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,1,128,1,float16,float16,127,0.01648000031709671
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,1,128,1,float16,float16,511,0.022336000204086305
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,1,128,1,float16,float16,4095,0.028068798780441283
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,1,128,1,float16,float16,8191,0.02980000078678131
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,2,128,1,float16,float16,1,0.013948799669742584
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,1,128,1,float16,float16,16383,0.03377119898796081
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,2,128,1,float16,float16,3,0.013955199718475341
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,2,128,1,float16,float16,7,0.013971200585365296
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,2,128,1,float16,float16,31,0.013916799426078796
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,2,128,1,float16,float16,63,0.014075200259685516
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,2,128,1,float16,float16,127,0.01660960018634796
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,2,128,1,float16,float16,255,0.02152319997549057
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,2,128,1,float16,float16,511,0.022172799706459044
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,2,128,1,float16,float16,1023,0.0265855997800827
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,2,128,1,float16,float16,2047,0.026910400390625
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,4,128,1,float16,float16,16383,1.9972448348999023
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,2,128,1,float16,float16,4095,0.028310400247573853
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,2,128,1,float16,float16,8191,0.030164799094200133
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,2,128,1,float16,float16,32767,0.05708000063896179
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,2,128,1,float16,float16,16383,0.04085760116577149
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,4,128,1,float16,float16,3,0.014028799533843995
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,4,128,1,float16,float16,7,0.014343999326229095
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,4,128,1,float16,float16,15,0.01403679996728897
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,4,128,1,float16,float16,63,0.014212800562381745
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,4,128,1,float16,float16,31,0.014369599521160126
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,4,128,1,float16,float16,127,0.01675360053777695
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,4,128,1,float16,float16,255,0.02205760031938553
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,4,128,1,float16,float16,511,0.022228799760341644
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,4,128,1,float16,float16,2047,0.02747359871864319
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,4,128,1,float16,float16,1023,0.02732959985733032
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,4,128,1,float16,float16,4095,0.0286080002784729
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,4,128,1,float16,float16,8191,0.03837119936943054
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,4,128,1,float16,float16,16383,0.05505759716033935
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,1,128,1,float16,float16,1,0.0137472003698349
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,1,128,1,float16,float16,3,0.013838399946689606
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,4,128,1,float16,float16,32767,0.08427680134773255
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,1,128,1,float16,float16,7,0.013891200721263885
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,1,128,1,float16,float16,15,0.013982400298118591
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,1,128,1,float16,float16,31,0.01401119977235794
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,1,128,1,float16,float16,127,0.016809600591659545
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,1,128,1,float16,float16,255,0.021587200462818146
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,1,128,1,float16,float16,511,0.022313599288463593
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,1,128,1,float16,float16,1023,0.026526400446891786
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,8,4,128,1,float16,float16,32767,3.9732208251953125
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,1,128,1,float16,float16,2047,0.02728480100631714
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,1,128,1,float16,float16,31,0.01408800035715103
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,1,128,1,float16,float16,8191,0.03054400086402893
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,1,128,1,float16,float16,16383,0.0370608001947403
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,1,128,1,float16,float16,4095,0.02911199927330017
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,2,128,1,float16,float16,1,0.014099200069904328
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,1,128,1,float16,float16,32767,0.05397440195083618
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,2,128,1,float16,float16,3,0.01427839994430542
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,2,128,1,float16,float16,7,0.014047999680042268
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,1,128,1,float16,float16,32767,0.04307039976119995
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,2,128,1,float16,float16,15,0.013963200151920319
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,2,128,1,float16,float16,31,0.014393599331378936
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,2,128,1,float16,float16,15,0.013691200315952301
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,2,128,1,float16,float16,255,0.021531200408935545
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,2,128,1,float16,float16,127,0.016771200299263
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,2,128,1,float16,float16,2047,0.02744320034980774
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,2,128,1,float16,float16,511,0.023310400545597076
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,2,128,1,float16,float16,4095,0.028832000494003297
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,2,128,1,float16,float16,8191,0.03758879899978638
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,2,128,1,float16,float16,1023,0.026617598533630372
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,4,128,1,float16,float16,1,0.013911999762058258
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,2,128,1,float16,float16,16383,0.05209439992904663
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,4,128,1,float16,float16,3,0.01406719982624054
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,4,128,1,float16,float16,15,0.013944000005722046
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,4,128,1,float16,float16,7,0.013926400244235993
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,4,128,1,float16,float16,31,0.014099200069904328
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,2,128,1,float16,float16,32767,0.08223360180854797
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,4,128,1,float16,float16,127,0.016769599914550782
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,4,128,1,float16,float16,255,0.02144159972667694
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,8,4,128,1,float16,float16,1,0.013956800103187561
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,4,128,1,float16,float16,1023,0.02707360088825226
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,4,128,1,float16,float16,2047,0.028244799375534056
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,4,128,1,float16,float16,511,0.02336000055074692
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,4,128,1,float16,float16,4095,0.03686720132827759
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,4,128,1,float16,float16,8191,0.05432479977607727
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,1,128,1,float16,float16,1,0.014795200526714325
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,4,128,1,float16,float16,16383,0.07876960039138795
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,1,128,1,float16,float16,3,0.014822399616241455
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,1,128,1,float16,float16,7,0.014841599762439728
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,1,128,1,float16,float16,15,0.014875200390815736
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,1,128,1,float16,float16,63,0.01494240015745163
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,4,128,1,float16,float16,32767,0.1315824031829834
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,1,128,1,float16,float16,127,0.017574399709701538
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,1,128,1,float16,float16,255,0.02244960069656372
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,1,128,1,float16,float16,511,0.029548799991607665
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,1,128,1,float16,float16,1023,0.03835520148277283
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,1,128,1,float16,float16,2047,0.055027198791503903
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,1,128,1,float16,float16,8191,0.13645119667053224
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,1,128,1,float16,float16,4095,0.08363999724388123
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,2,128,1,float16,float16,1,0.015361599624156952
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,1,128,1,float16,float16,16383,0.3521199941635132
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,2,128,1,float16,float16,3,0.015371200442314149
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,2,128,1,float16,float16,7,0.014920000731945039
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,2,128,1,float16,float16,15,0.015347200632095336
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,2,128,1,float16,float16,31,0.015089599788188935
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,2,128,1,float16,float16,63,0.015481600165367126
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,2,128,1,float16,float16,127,0.017923200130462648
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,2,128,1,float16,float16,255,0.023207999765872955
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,2,128,1,float16,float16,511,0.0462336003780365
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,2,128,1,float16,float16,1023,0.07161120176315308
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,2,128,1,float16,float16,2047,0.11162879467010497
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,2,128,1,float16,float16,63,0.014244799315929414
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,2,128,1,float16,float16,4095,0.19211679697036743
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,2,128,1,float16,float16,8191,0.3545488119125366
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,4,128,1,float16,float16,1,0.016806399822235106
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,4,128,1,float16,float16,63,0.014073599874973298
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,4,128,1,float16,float16,7,0.01687999963760376
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,4,128,1,float16,float16,15,0.017047999799251555
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,2,128,1,float16,float16,16383,0.676800012588501
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,4,128,1,float16,float16,31,0.017102399468421937
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,4,128,1,float16,float16,127,0.021617600321769716
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,4,128,1,float16,float16,63,0.017524799704551695
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,4,128,1,float16,float16,255,0.029782399535179138
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,1,128,1,float16,float16,31,0.015043200552463531
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,4,128,1,float16,float16,511,0.06991519927978515
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,4,128,1,float16,float16,1023,0.11352479457855225
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,4,128,1,float16,float16,2047,0.19380639791488646
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,2,128,1,float16,float16,32767,1.3258336067199707
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,4,128,1,float16,float16,4095,0.35446240901947024
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,1,128,1,float16,float16,1,0.01414559930562973
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,8,1,128,1,float16,float16,63,0.014219200611114502
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,4,128,1,float16,float16,8191,0.6794320106506347
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,1,128,1,float16,float16,3,0.014140799641609192
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,1,128,1,float16,float16,7,0.01371839940547943
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,1,128,1,float16,float16,63,0.013974399864673614
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,1,128,1,float16,float16,31,0.014214399456977844
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,1,128,1,float16,float16,15,0.014047999680042268
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,1,128,1,float16,float16,127,0.016553600132465363
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,1,128,1,float16,float16,255,0.021771200001239777
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,1,128,1,float16,float16,1023,0.026719999313354493
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,1,128,1,float16,float16,2047,0.028038400411605834
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,1,128,1,float16,float16,511,0.022551999986171724
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,1,128,1,float16,float16,4095,0.028836798667907716
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,1,128,1,float16,float16,8191,0.03805440068244934
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,4,128,1,float16,float16,16383,1.3212287902832032
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,2,128,1,float16,float16,1,0.014315199851989747
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,1,128,1,float16,float16,16383,0.05302240252494812
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,2,128,1,float16,float16,7,0.01430879980325699
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,2,128,1,float16,float16,3,0.014017599821090698
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,2,128,1,float16,float16,15,0.0139055997133255
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,2,128,1,float16,float16,31,0.014238399267196656
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,2,128,1,float16,float16,127,0.016612799465656282
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,2,128,1,float16,float16,255,0.021836799383163453
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,2,128,1,float16,float16,63,0.014164799451828003
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,2,128,1,float16,float16,511,0.022577600181102754
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,2,128,1,float16,float16,2047,0.02815839946269989
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,2,128,1,float16,float16,4095,0.036908799409866334
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,4,128,1,float16,float16,3,0.01698399931192398
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,2,128,1,float16,float16,8191,0.054529601335525514
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,2,128,1,float16,float16,16383,0.0785152018070221
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,2,128,1,float16,float16,32767,0.12770240306854247
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,4,128,1,float16,float16,1,0.013966399431228637
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,4,128,1,float16,float16,3,0.014207999408245086
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,4,128,1,float16,float16,7,0.014083200693130493
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,4,128,1,float16,float16,15,0.01420000046491623
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,4,128,1,float16,float16,63,0.01430400013923645
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,4,128,1,float16,float16,31,0.01409280002117157
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,4,128,1,float16,float16,255,0.021822400391101837
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,4,128,1,float16,float16,511,0.023369599878787995
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,4,128,1,float16,float16,32767,2.6189184188842773
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,4,128,1,float16,float16,2047,0.03654240071773529
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,4,128,1,float16,float16,4095,0.05401279926300049
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,4,128,1,float16,float16,8191,0.08196799755096436
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,4,128,1,float16,float16,16383,0.21266400814056396
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,1,128,1,float16,float16,1,0.018505600094795228
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,1,128,1,float16,float16,3,0.018438400328159334
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,4,128,1,float16,float16,32767,0.3969631910324097
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,1,128,1,float16,float16,7,0.018592000007629395
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,1,128,1,float16,float16,15,0.018699200451374055
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,1,128,1,float16,float16,31,0.018831999599933626
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,1,128,1,float16,float16,63,0.01908160001039505
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,1,128,1,float16,float16,127,0.023316800594329834
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,1,128,1,float16,float16,255,0.03136639893054962
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,8,1,128,1,float16,float16,32767,0.670582389831543
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,1,128,1,float16,float16,511,0.07376959919929504
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,1,128,1,float16,float16,1023,0.1166767954826355
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,1,128,1,float16,float16,2047,0.19741599559783934
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,1,128,1,float16,float16,32767,0.07896959781646729
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,2,128,1,float16,float16,1,0.024534399807453155
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,1,128,1,float16,float16,4095,0.35743041038513185
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,2,128,1,float16,float16,3,0.024697600305080412
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,2,128,1,float16,float16,1023,0.02733440101146698
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,2,128,1,float16,float16,15,0.024751999974250795
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,1,128,1,float16,float16,8191,0.6823135852813721
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,2,128,1,float16,float16,7,0.024668799340724946
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,2,128,1,float16,float16,31,0.024886399507522583
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,2,128,1,float16,float16,63,0.02528960108757019
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,2,128,1,float16,float16,127,0.03124319911003113
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,2,128,1,float16,float16,511,0.10188159942626954
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,2,128,1,float16,float16,255,0.048686400055885315
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,2,128,1,float16,float16,1023,0.16792479753494263
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,4,128,1,float16,float16,127,0.016846400499343873
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,4,128,1,float16,float16,1,0.03213280141353607
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,8,4,128,1,float16,float16,1023,0.02762399911880493
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,2,128,1,float16,float16,2047,0.28743839263916016
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,4,128,1,float16,float16,3,0.03224320113658905
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,1,128,1,float16,float16,16383,1.3270992279052733
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,4,128,1,float16,float16,7,0.03220320045948029
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,2,128,1,float16,float16,4095,0.5294991970062256
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,4,128,1,float16,float16,31,0.03276320099830628
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,4,128,1,float16,float16,15,0.03237600028514862
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,4,128,1,float16,float16,63,0.03346560001373291
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,4,128,1,float16,float16,255,0.0724016010761261
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,4,128,1,float16,float16,511,0.1588912010192871
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,2,128,1,float16,float16,8191,1.020524787902832
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,4,128,1,float16,float16,1023,0.2686784029006958
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,1,128,1,float16,float16,1,0.02608320116996765
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,4,128,1,float16,float16,2047,0.4690720081329346
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,1,128,1,float16,float16,3,0.026254400610923767
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,1,128,1,float16,float16,15,0.026481598615646362
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,1,128,1,float16,float16,7,0.026311999559402464
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,1,128,1,float16,float16,31,0.026625600457191468
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,1,128,1,float16,float16,63,0.026907199621200563
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,4,128,1,float16,float16,4095,0.8703071594238281
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,1,128,1,float16,float16,127,0.03314239978790283
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,2,128,1,float16,float16,16383,1.9962160110473632
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,1,128,1,float16,float16,255,0.05024799704551697
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,1,128,1,float16,float16,511,0.10645920038223267
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,1,128,1,float16,float16,1023,0.17072800397872925
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,1,128,1,float16,float16,2047,0.29332480430603025
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,2,128,1,float16,float16,1,0.034030398726463316
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,4,128,1,float16,float16,8191,1.7019968032836914
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,2,128,1,float16,float16,7,0.03422400057315826
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,2,128,1,float16,float16,15,0.034283199906349184
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,1,128,1,float16,float16,4095,0.5313263893127441
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,2,128,1,float16,float16,31,0.03490560054779053
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,2,128,1,float16,float16,127,0.051534402370452884
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,2,128,1,float16,float16,255,0.0733680009841919
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,2,128,1,float16,float16,511,0.16509759426116943
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,4,128,1,float16,float16,127,0.048075199127197266
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,2,128,1,float16,float16,1023,0.27422080039978025
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,4,128,1,float16,float16,1,0.05185440182685852
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,2,128,1,float16,float16,2047,0.47055840492248535
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,4,128,1,float16,float16,3,0.05193600058555603
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,4,128,1,float16,float16,31,0.05285279750823975
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,4,128,1,float16,float16,15,0.052116799354553225
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,8,4,128,1,float16,float16,16383,3.4617359161376955
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,4,128,1,float16,float16,63,0.060147202014923094
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,4,128,1,float16,float16,127,0.08054400086402894
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,4,128,1,float16,float16,255,0.1255295991897583
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,2,128,1,float16,float16,4095,0.8821871757507325
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,1,128,1,float16,float16,1,0.03858720064163208
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,4,128,1,float16,float16,511,0.31956160068511963
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,1,128,1,float16,float16,3,0.03851360082626343
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,4,128,1,float16,float16,1023,0.5193808078765869
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,1,128,1,float16,float16,7,0.04036639928817749
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,1,128,1,float16,float16,15,0.038848000764846805
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,1,128,1,float16,float16,63,0.041863998770713805
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,4,128,1,float16,float16,2047,0.9199680328369141
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,1,128,1,float16,float16,255,0.07902079820632935
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,2,128,1,float16,float16,3,0.03389280140399933
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,1,128,1,float16,float16,511,0.17197920083999635
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,1,128,1,float16,float16,1023,0.28734400272369387
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,2,128,1,float16,float16,1,0.05612800121307373
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,2,128,1,float16,float16,63,0.03559199869632721
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,1,128,1,float16,float16,2047,0.47886881828308103
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,4,128,1,float16,float16,4095,1.7452816009521483
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,2,128,1,float16,float16,15,0.0563759982585907
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,2,128,1,float16,float16,7,0.05628479719161987
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,2,128,1,float16,float16,3,0.056739199161529544
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,2,128,1,float16,float16,31,0.056627202033996585
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,2,128,1,float16,float16,127,0.08492640256881714
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,2,128,1,float16,float16,63,0.06852480173110961
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,8,4,128,1,float16,float16,7,0.051830399036407473
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,2,128,1,float16,float16,255,0.12959359884262084
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,4,128,1,float16,float16,1,0.09690560102462768
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,4,128,1,float16,float16,3,0.09359359741210938
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,2,128,1,float16,float16,511,0.3136575937271118
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,4,128,1,float16,float16,7,0.09392480254173279
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,4,128,1,float16,float16,15,0.09407520294189453
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,4,128,1,float16,float16,31,0.09842399954795837
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,4,128,1,float16,float16,63,0.10445599555969239
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,4,128,1,float16,float16,127,0.14206720590591432
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,2,128,1,float16,float16,2047,0.9283887863159179
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,4,128,1,float16,float16,255,0.23358399868011476
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,1,128,1,float16,float16,1,0.014248000085353851
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,1,128,1,float16,float16,3,0.014297600090503692
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,4,128,1,float16,float16,511,0.5583151817321778
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,1,128,1,float16,float16,31,0.03885760009288788
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,1,128,1,float16,float16,7,0.0143327996134758
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,1,128,1,float16,float16,31,0.01409280002117157
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,1,128,1,float16,float16,15,0.014163200557231904
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,1,128,1,float16,float16,127,0.055980801582336426
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,1,128,1,float16,float16,127,0.016531200706958772
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,1,128,1,float16,float16,511,0.02279680073261261
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,1,128,1,float16,float16,255,0.021828800439834595
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,1,128,1,float16,float16,1023,0.027340799570083618
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,4,128,1,float16,float16,1023,0.9791199684143066
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,1,128,1,float16,float16,2047,0.028116801381111146
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,1,128,1,float16,float16,8191,0.05434240102767944
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,1,128,1,float16,float16,4095,0.03730719983577728
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,1,128,1,float16,float16,16383,0.07886880040168762
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,2,128,1,float16,float16,1,0.014239999651908874
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,2,128,1,float16,float16,3,0.014473600685596466
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,1,128,1,float16,float16,32767,0.12937120199203492
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,2,128,1,float16,float16,15,0.014283199608325959
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,2,128,1,float16,float16,7,0.013990400731563568
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,2,128,1,float16,float16,31,0.014407999813556671
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,2,128,1,float16,float16,63,0.01427839994430542
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,2,128,1,float16,float16,127,0.016979199647903443
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,2,128,1,float16,float16,511,0.023395200073719025
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,2,128,1,float16,float16,255,0.021617600321769716
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,2,128,1,float16,float16,1023,0.027657601237297057
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,2,128,1,float16,float16,2047,0.036657598614692685
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,4,128,1,float16,float16,2047,1.7763744354248048
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,4,128,1,float16,float16,1,0.014644800126552582
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,2,128,1,float16,float16,16383,0.21264801025390626
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,4,128,1,float16,float16,3,0.014662399888038635
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,4,128,1,float16,float16,7,0.014192000031471252
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,2,128,1,float16,float16,32767,0.39371039867401125
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,4,128,1,float16,float16,15,0.014616000652313232
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,4,128,1,float16,float16,31,0.014236800372600555
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,4,128,1,float16,float16,63,0.014390400052070618
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,8,2,128,1,float16,float16,1023,0.5288047790527344
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,4,128,1,float16,float16,255,0.021774399280548095
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,4,128,1,float16,float16,127,0.017566399276256563
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,4,128,1,float16,float16,511,0.027955201268196107
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,4,128,1,float16,float16,2047,0.05438079833984375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,4,128,1,float16,float16,1023,0.03714720010757446
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,4,128,1,float16,float16,4095,0.08336319923400878
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,4,128,1,float16,float16,8191,0.1339151978492737
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,4,128,1,float16,float16,16383,0.35197761058807375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,8,1,128,1,float16,float16,1,0.05773599743843079
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,8,1,128,1,float16,float16,3,0.05631359815597534
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,8,1,128,1,float16,float16,31,0.05708320140838623
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,8,1,128,1,float16,float16,15,0.05756000280380249
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,4,128,1,float16,float16,32767,0.6703296184539795
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,8,1,128,1,float16,float16,63,0.06582239866256714
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,8,1,128,1,float16,float16,255,0.1285055994987488
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,8,1,128,1,float16,float16,127,0.08685759902000427
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,8,1,128,1,float16,float16,511,0.3156991958618164
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,8,2,128,1,float16,float16,1,0.09469119906425476
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,8,2,128,1,float16,float16,3,0.09363200068473816
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,8,1,128,1,float16,float16,1023,0.5351583957672119
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,8,2,128,1,float16,float16,15,0.09412639737129211
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,2,128,1,float16,float16,4095,0.05345439910888672
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,8,2,128,1,float16,float16,31,0.09850879907608032
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,8,2,128,1,float16,float16,63,0.10540319681167602
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,8,2,128,1,float16,float16,127,0.1430367946624756
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,2,128,1,float16,float16,8191,0.08183519840240479
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,8,2,128,1,float16,float16,255,0.23328959941864014
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,8,4,128,1,float16,float16,1,0.16584960222244263
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,8,4,128,1,float16,float16,7,0.1659168004989624
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,8,2,128,1,float16,float16,511,0.5655136108398438
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,8,4,128,1,float16,float16,3,0.16562240123748778
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,8,4,128,1,float16,float16,31,0.1750480055809021
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,8,4,128,1,float16,float16,15,0.1676800012588501
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,8,4,128,1,float16,float16,63,0.1833791971206665
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,8,4,128,1,float16,float16,127,0.2578927993774414
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,8,1,128,1,float16,float16,7,0.056302398443222046
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,8,1,128,1,float16,float16,1,0.09381759762763978
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,8,1,128,1,float16,float16,63,0.014459200203418732
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,8,1,128,1,float16,float16,3,0.09391040205955506
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,8,4,128,1,float16,float16,255,0.4305552005767822
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,8,1,128,1,float16,float16,7,0.09408000111579895
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,8,1,128,1,float16,float16,31,0.09953119754791259
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,8,1,128,1,float16,float16,15,0.09478719830513001
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,8,1,128,1,float16,float16,63,0.10693279504776002
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,8,1,128,1,float16,float16,127,0.14343199729919434
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,8,4,128,1,float16,float16,511,1.088798427581787
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,8,1,128,1,float16,float16,255,0.22707839012145997
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,8,2,128,1,float16,float16,1,0.16571520566940307
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,8,2,128,1,float16,float16,7,0.0949504017829895
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,8,2,128,1,float16,float16,3,0.1657055974006653
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,8,2,128,1,float16,float16,7,0.16680320501327514
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,8,2,128,1,float16,float16,15,0.1712272047996521
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,8,1,128,1,float16,float16,511,0.589355182647705
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,8,2,128,1,float16,float16,63,0.18448640108108522
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,8,2,128,1,float16,float16,31,0.17641119956970214
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,8,2,128,1,float16,float16,127,0.2595983982086182
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,8,4,128,1,float16,float16,1,0.31143999099731445
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,8,4,128,1,float16,float16,3,0.3106816053390503
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,8,2,128,1,float16,float16,255,0.43877601623535156
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,8,4,128,1,float16,float16,1023,1.935468864440918
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,8,4,128,1,float16,float16,7,0.31648960113525393
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,8,4,128,1,float16,float16,15,0.32327680587768554
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,8,4,128,1,float16,float16,31,0.3262880086898804
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,8,4,128,1,float16,float16,63,0.34135520458221436
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,1,128,1,float16,float16,1,0.014207999408245086
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,8,2,128,1,float16,float16,511,1.111836814880371
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,8,4,128,1,float16,float16,127,0.5035615921020508
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,1,128,1,float16,float16,3,0.014052799344062806
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,1,128,1,float16,float16,7,0.014236800372600555
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,1,128,1,float16,float16,63,0.014159999787807465
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,1,128,1,float16,float16,127,0.01695999950170517
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,8,2,128,1,float16,float16,1023,0.9860624313354492
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,1,128,1,float16,float16,31,0.014707200229167938
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,8,4,128,1,float16,float16,255,0.829798412322998
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,1,128,1,float16,float16,511,0.023556800186634065
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,1,128,1,float16,float16,255,0.021774399280548095
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,1,128,1,float16,float16,2047,0.03711679875850678
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,1,128,1,float16,float16,1023,0.027852800488471986
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,1,128,1,float16,float16,4095,0.0563215970993042
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,1,128,1,float16,float16,8191,0.08112800121307373
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,2,128,1,float16,float16,1,0.014326399564743042
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,2,128,1,float16,float16,3,0.014502400159835815
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,1,128,1,float16,float16,16383,0.2137455940246582
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,2,128,1,float16,float16,7,0.014238399267196656
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,2,128,1,float16,float16,15,0.014337599277496338
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,2,128,1,float16,float16,31,0.014422400295734406
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,2,128,1,float16,float16,127,0.01695999950170517
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,2,128,1,float16,float16,63,0.014486399292945863
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,2,128,1,float16,float16,255,0.02207999974489212
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,1,128,1,float16,float16,32767,0.40195841789245607
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,2,128,1,float16,float16,511,0.02884959876537323
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,2,128,1,float16,float16,1023,0.037145599722862244
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,2,128,1,float16,float16,2047,0.05396639704704285
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,4,128,1,float16,float16,1,0.014419199526309967
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,2,128,1,float16,float16,4095,0.08214719891548157
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,2,128,1,float16,float16,8191,0.13656480312347413
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,4,128,1,float16,float16,3,0.014387199282646179
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,4,128,1,float16,float16,7,0.0146479994058609
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,4,128,1,float16,float16,31,0.014630399644374847
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,4,128,1,float16,float16,15,0.014630399644374847
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,8,4,128,1,float16,float16,511,2.1470176696777346
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,2,128,1,float16,float16,16383,0.35592639446258545
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,4,128,1,float16,float16,63,0.014867199957370758
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,4,128,1,float16,float16,255,0.022307200729846953
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,2,128,1,float16,float16,32767,0.669975996017456
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,4,128,1,float16,float16,1023,0.0709168016910553
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,4,128,1,float16,float16,511,0.04754399955272674
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,4,128,1,float16,float16,2047,0.11107039451599121
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,4,128,1,float16,float16,4095,0.19279359579086303
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,1,0.01586720049381256
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,4,128,1,float16,float16,8191,0.35441439151763915
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,3,0.015855999290943147
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,15,0.015729600191116334
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,31,0.015726399421691895
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,63,0.015907199680805208
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,127,0.018483200669288637
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,255,0.023403200507164
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,511,0.04938560128211975
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,1,128,1,float16,float16,15,0.01419840008020401
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,1023,0.07348639965057373
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,4095,0.19443680047988893
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,2047,0.11436480283737183
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,4,128,1,float16,float16,32767,1.3422975540161133
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,8191,0.35824320316314695
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,1,0.017553600668907165
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,3,0.01762399971485138
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,7,0.01759839951992035
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,15,0.017574399709701538
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,16383,0.6850416183471679
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,31,0.01764959990978241
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,127,0.022129599750041962
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,63,0.017907199263572694
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,255,0.030364799499511718
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,4,128,1,float16,float16,127,0.01729599982500076
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,511,0.07131519913673401
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,1023,0.11508159637451172
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,32767,1.3286352157592773
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,2047,0.19428960084915162
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,4095,0.3540911912918091
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,1,0.013575999438762665
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,3,0.013652800023555756
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,7,0.015910400450229643
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,7,0.013643200695514678
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,15,0.01403840035200119
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,8191,0.6807007789611816
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,31,0.013699199259281158
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,63,0.013947199285030364
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,255,0.02125760018825531
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,127,0.01659200042486191
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,511,0.02377920001745224
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,1023,0.02751680016517639
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,4095,0.028278398513793945
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,2047,0.027905601263046264
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,8191,0.03062080144882202
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,16383,0.0356687992811203
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,32767,0.04468800127506256
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,65535,2.6420207977294923
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,16383,1.324351978302002
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,65535,0.06263840198516846
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,15,0.014127999544143677
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,7,0.013897599279880523
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,3,0.013911999762058258
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,31,0.014142400026321411
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,127,0.01658719927072525
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,63,0.014110399782657624
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,255,0.02144480049610138
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,511,0.023747199773788454
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,2047,0.028152000904083253
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,1023,0.027584001421928406
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,8191,0.03161759972572327
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,4095,0.02861439883708954
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,16383,0.040387201309204104
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,32767,2.6209808349609376
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,32767,0.05788959860801697
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,1,0.013687999546527862
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,65535,0.08590720295906067
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,15,0.013761599361896516
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,7,0.013859200477600097
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,63,0.013889600336551667
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,3,0.013950400054454803
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,127,0.016672000288963318
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,255,0.021222400665283202
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,1023,0.027718400955200194
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,511,0.023662400245666505
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,2047,0.027926400303840637
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,4095,0.028683200478553772
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,8191,0.03145920038223267
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,16383,0.04030559957027435
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,32767,0.05780799984931946
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,1,0.013911999762058258
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,3,0.013872000575065612
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,7,0.01393440067768097
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,15,0.014107200503349304
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,65535,0.08534719944000244
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,8,4,128,1,float16,float16,16383,0.6839583873748779
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,31,0.014017599821090698
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,63,0.014144000411033631
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,511,0.02361920028924942
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,127,0.016710400581359863
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,1023,0.028043198585510253
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,255,0.021537600457668303
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,2047,0.028384000062942505
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,1,0.013995200395584106
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,4095,0.029236799478530882
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,8191,0.03971199989318848
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,32767,0.08157119750976563
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,3,0.014918400347232819
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,1,0.014865599572658539
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,16383,0.05422239899635315
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,65535,0.13538880348205568
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,65535,5.2010047912597654
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,7,0.01480959951877594
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,31,0.014892800152301789
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,63,0.014998400211334228
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,127,0.017735999822616578
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,511,0.02988319993019104
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,1023,0.03895840048789978
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,255,0.022368000447750093
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,2047,0.05564640164375305
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,4095,0.08399199843406677
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,8191,0.1381119966506958
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,1,0.015118399262428283
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,3,0.014865599572658539
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,7,0.015014399588108063
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,31,0.015065599977970124
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,16383,0.3593103885650635
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,63,0.015139199793338776
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,255,0.022657600045204163
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,127,0.017791999876499175
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,32767,0.6795216083526612
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,511,0.04767520129680634
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,31,0.01408960074186325
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,1023,0.07306560277938842
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,2047,0.11374720335006713
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,4095,0.19284160137176515
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,65535,1.3269904136657715
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,8191,0.358188796043396
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,1,0.013964800536632538
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,3,0.013935999572277069
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,7,0.013889600336551667
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,15,0.013916799426078796
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,31,0.01387840062379837
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,16383,0.6804543972015381
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,63,0.014056000113487243
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,127,0.016574400663375854
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,511,0.023739199340343475
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,255,0.021462400257587434
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,1023,0.027987200021743774
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,2047,0.02834399938583374
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,4095,0.02903839945793152
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,16383,0.057897597551345825
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,8191,0.03951039910316467
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,15,0.015033599734306336
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,1,0.014267200231552124
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,65535,0.1389248013496399
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,32767,0.08559520244598388
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,3,0.014078399538993836
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,7,0.014112000167369843
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,32767,1.3321375846862793
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,63,0.014283199608325959
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,127,0.016836799681186676
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,15,0.01406400054693222
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,31,0.014209599792957306
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,1023,0.02852959930896759
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,2047,0.02911359965801239
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,255,0.021740800142288207
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,4095,0.037729600071907045
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,8191,0.05685439705848694
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,16383,0.0812175989151001
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,1,128,1,float16,float16,1,0.0186256006360054
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,1,128,1,float16,float16,3,0.018592000007629395
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,65535,0.23230559825897218
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,1,128,1,float16,float16,15,0.01863519996404648
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,15,0.014987200498580933
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,1,128,1,float16,float16,31,0.018838399648666383
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,1,128,1,float16,float16,63,0.019067199528217317
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,1,128,1,float16,float16,127,0.02317280024290085
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,1,128,1,float16,float16,255,0.031339201331138614
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,1,128,1,float16,float16,511,0.07297120094299317
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,65535,2.617630386352539
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,1,128,1,float16,float16,1023,0.11648160219192505
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,1,128,1,float16,float16,2047,0.19593600034713746
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,1,128,1,float16,float16,4095,0.35652799606323243
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,2,128,1,float16,float16,1,0.024596799910068513
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,2,128,1,float16,float16,3,0.024644799530506134
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,1,128,1,float16,float16,8191,0.6869088172912597
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,2,128,1,float16,float16,7,0.024644799530506134
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,2,128,1,float16,float16,31,0.02492479979991913
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,2,128,1,float16,float16,63,0.025259199738502502
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,2,128,1,float16,float16,15,0.024694399535655977
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,2,128,1,float16,float16,127,0.0314736008644104
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,2,128,1,float16,float16,255,0.04869439899921417
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,1,128,1,float16,float16,16383,1.3258272171020509
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,2,128,1,float16,float16,511,0.10178879499435425
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,2,128,1,float16,float16,1023,0.16908960342407225
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,2,128,1,float16,float16,2047,0.2871792078018188
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,2,128,1,float16,float16,4095,0.5287375926971436
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,511,0.02407039999961853
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,4,1,128,1,float16,float16,1,0.026199999451637267
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,4,1,128,1,float16,float16,3,0.02619200050830841
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,4,1,128,1,float16,float16,7,0.026318401098251343
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,1,128,1,float16,float16,7,0.01870879977941513
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,32767,0.13165119886398316
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,4,1,128,1,float16,float16,15,0.026398399472236635
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,1,128,1,float16,float16,32767,2.6146223068237306
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,4,1,128,1,float16,float16,127,0.033225598931312564
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,4,1,128,1,float16,float16,63,0.02709600031375885
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,2,128,1,float16,float16,8191,1.0238335609436036
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,4,1,128,1,float16,float16,255,0.05781919956207275
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,4,1,128,1,float16,float16,1023,0.17157599925994874
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,4,1,128,1,float16,float16,2047,0.2920351982116699
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,4,2,128,1,float16,float16,1,0.03400959968566895
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,4,1,128,1,float16,float16,4095,0.5302271842956543
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,4,2,128,1,float16,float16,3,0.034123200178146365
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,2,128,1,float16,float16,16383,1.9915536880493163
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,4,2,128,1,float16,float16,7,0.03376320004463196
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,4,2,128,1,float16,float16,15,0.03427839875221252
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,4,2,128,1,float16,float16,31,0.03463360071182251
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,4,2,128,1,float16,float16,63,0.0352048009634018
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,4,2,128,1,float16,float16,127,0.05108000040054321
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,4,2,128,1,float16,float16,255,0.07316160202026367
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,4,1,128,1,float16,float16,8191,1.0229904174804687
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,4,2,128,1,float16,float16,511,0.16237759590148926
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,4,2,128,1,float16,float16,1023,0.274232006072998
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,4,2,128,1,float16,float16,2047,0.4769087791442871
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,4,1,128,1,float16,float16,1,0.038252800703048706
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,4,1,128,1,float16,float16,7,0.03851040005683899
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,4,1,128,1,float16,float16,3,0.038515201210975646
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,4,1,128,1,float16,float16,15,0.038577601313591
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,4,1,128,1,float16,float16,31,0.03912000060081482
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,4,1,128,1,float16,float16,63,0.03980480134487152
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,4,2,128,1,float16,float16,32767,4.011619186401367
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,4,1,128,1,float16,float16,127,0.054995197057724
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,4,1,128,1,float16,float16,511,0.11223679780960083
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,4,1,128,1,float16,float16,255,0.07644799947738648
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,4,2,128,1,float16,float16,1,0.05605599880218506
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,4,2,128,1,float16,float16,8191,1.6941263198852539
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,4,1,128,1,float16,float16,1023,0.2816272020339966
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,4,2,128,1,float16,float16,3,0.056067198514938354
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,4,1,128,1,float16,float16,2047,0.47878079414367675
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,4,2,128,1,float16,float16,31,0.05689600110054016
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,4,2,128,1,float16,float16,7,0.05935199856758118
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,4,2,128,1,float16,float16,63,0.06519039869308471
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,4,1,128,1,float16,float16,4095,0.881499195098877
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,4,2,128,1,float16,float16,255,0.13220479488372802
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,4,2,128,1,float16,float16,511,0.30841119289398194
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,4,2,128,1,float16,float16,1023,0.5306320190429688
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,1,0.01417119950056076
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,3,0.014236800372600555
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,7,0.014246399700641631
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,4,2,128,1,float16,float16,2047,0.9451824188232422
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,15,0.014217600226402283
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,4,2,128,1,float16,float16,4095,0.8729040145874023
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,31,0.014348800480365752
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,4,1,128,1,float16,float16,31,0.026782399415969847
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,127,0.016972799599170686
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,255,0.02186720073223114
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,511,0.02425280064344406
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,2047,0.029683199524879456
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,4095,0.03830240070819855
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,4,2,128,1,float16,float16,4095,1.7425807952880858
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,4,1,128,1,float16,float16,511,0.1762608051300049
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,8191,0.05780320167541504
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,16383,0.08956959843635559
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,32767,0.13572319746017455
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,1,0.01512639969587326
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,65535,0.24112000465393066
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,3,0.01401440054178238
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,4,2,128,1,float16,float16,127,0.08440799713134765
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,7,0.015132799744606018
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,31,0.014142400026321411
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,15,0.0143327996134758
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,63,0.014230400323867798
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,127,0.017448000609874725
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,1023,0.028811201453208923
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,255,0.021719999611377716
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,511,0.02476319968700409
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,2047,0.03911679983139038
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,4095,0.0546783983707428
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,8191,0.08460320234298706
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,16383,0.21644160747528077
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,4,1,128,1,float16,float16,1,0.05626879930496216
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,32767,0.40735840797424316
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,4,1,128,1,float16,float16,3,0.05622239708900452
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,4,1,128,1,float16,float16,7,0.056411200761795045
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,4,1,128,1,float16,float16,15,0.057291197776794436
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,4,1,128,1,float16,float16,31,0.05713919997215271
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,4,1,128,1,float16,float16,63,0.06571360230445862
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,4,1,128,1,float16,float16,127,0.08617920279502869
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,63,0.013982400298118591
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,4,1,128,1,float16,float16,255,0.1278048038482666
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,1023,0.028619199991226196
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,4,2,128,1,float16,float16,1,0.09358400106430054
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,4,1,128,1,float16,float16,511,0.31678879261016846
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,4,2,128,1,float16,float16,3,0.09424319863319397
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,4,2,128,1,float16,float16,7,0.09377920031547546
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,4,2,128,1,float16,float16,15,0.09411519765853882
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,4,1,128,1,float16,float16,1023,0.5416031837463379
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,4,2,128,1,float16,float16,31,0.0975600004196167
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,4,2,128,1,float16,float16,63,0.10573920011520385
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,4,2,128,1,float16,float16,127,0.14413440227508545
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,4,2,128,1,float16,float16,255,0.23278400897979737
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,4,1,128,1,float16,float16,2047,0.9313360214233398
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,4,2,128,1,float16,float16,15,0.056355202198028566
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,4,2,128,1,float16,float16,511,0.5665071964263916
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,4,1,128,1,float16,float16,1,0.09449120163917542
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,4,1,128,1,float16,float16,7,0.09402719736099244
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,4,1,128,1,float16,float16,3,0.09373440146446228
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,4,1,128,1,float16,float16,15,0.09465280175209045
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,4,1,128,1,float16,float16,63,0.10549759864807129
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,4,1,128,1,float16,float16,127,0.14338879585266112
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,4,1,128,1,float16,float16,255,0.22839040756225587
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,4,1,128,1,float16,float16,511,0.5878111839294433
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,65535,0.7634064197540283
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,4,2,128,1,float16,float16,3,0.1657696008682251
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,4,2,128,1,float16,float16,2047,1.768903923034668
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,4,2,128,1,float16,float16,7,0.16640959978103637
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,4,2,128,1,float16,float16,15,0.16901919841766358
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,4,1,128,1,float16,float16,1023,1.0072079658508302
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,4,2,128,1,float16,float16,63,0.18518400192260742
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,4,2,128,1,float16,float16,127,0.25749280452728274
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,1,0.014248000085353851
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,4,2,128,1,float16,float16,255,0.4359407901763916
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,3,0.01419519931077957
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,15,0.014115199446678162
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,7,0.014220799505710601
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,31,0.014161600172519684
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,63,0.014393599331378936
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,127,0.01688160002231598
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,511,0.0247311994433403
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,255,0.0217631995677948
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,1023,0.02901279926300049
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,2047,0.03796960115432739
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,4,2,128,1,float16,float16,511,1.1048175811767578
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,4095,0.05577279925346375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,4,1,128,1,float16,float16,31,0.09957119822502136
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,4,2,128,1,float16,float16,1023,0.9875791549682618
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,16383,0.22017440795898438
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,32767,0.40071840286254884
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,1,0.014310400187969207
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,7,0.014339199662208557
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,15,0.014206400513648987
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,31,0.014327999949455262
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,4,2,128,1,float16,float16,1,0.16641600131988527
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,4,2,128,1,float16,float16,1023,1.9609424591064453
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,65535,0.764799976348877
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,63,0.014443199336528777
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,127,0.017148800194263458
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,255,0.021876800060272216
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,1023,0.03854880034923554
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,511,0.029737600684165956
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,4095,0.08266239762306213
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,8191,0.13778400421142578
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,4,2,128,1,float16,float16,31,0.18874080181121827
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,1,0.01569119989871979
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,16383,0.3554960012435913
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,3,0.0157600000500679
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,7,0.015676799416542053
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,15,0.015787200629711153
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,31,0.015731200575828552
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,63,0.016008000075817107
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,127,0.018492799997329713
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,255,0.023379200696945192
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,511,0.0489439994096756
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,1023,0.07398880124092103
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,2047,0.11450719833374023
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,65535,1.3237312316894532
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,4095,0.19454879760742189
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,8191,0.3587248086929321
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,8191,0.08487520217895508
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,16383,0.6906832218170166
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,3,0.014271999895572662
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,3,0.01374399960041046
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,1,0.01390880048274994
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,7,0.013672000169754029
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,15,0.014059199392795563
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,31,0.013704000413417817
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,127,0.016259199380874632
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,255,0.021243199706077576
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,511,0.023636800050735474
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,2047,0.0560479998588562
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,1023,0.027959999442100526
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,32767,1.3358256340026855
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,2047,0.028016000986099243
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,4095,0.02810559868812561
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,16383,0.03591040074825287
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,8191,0.030969598889350893
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,32767,0.04534400105476379
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,65535,0.06265439987182617
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,1,0.013889600336551667
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,7,0.013924799859523773
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,3,0.01393280029296875
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,131071,0.09097599983215332
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,15,0.013916799426078796
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,31,0.013918399810791016
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,127,0.01661120057106018
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,63,0.014142400026321411
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,32767,0.6842415809631348
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,511,0.023950399458408357
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,1023,0.028299200534820556
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,4095,0.028705599904060363
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,2047,0.02810719907283783
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,16383,0.04122560024261475
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,32767,0.05884479880332947
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,65535,0.08589919805526733
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,65535,2.623606491088867
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,1,0.014662399888038635
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,7,0.014769600331783294
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,3,0.014632000029087067
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,15,0.01478080004453659
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,131071,0.14098399877548218
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,31,0.014860799908638
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,63,0.014926399290561675
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,127,0.017580799758434296
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,511,0.03033120036125183
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,255,0.02231999933719635
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,1023,0.03951680064201355
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,2047,0.055848002433776855
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,4095,0.08417919874191285
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,8191,0.13998559713363648
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,16383,0.3613471984863281
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,63,0.014187200367450714
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,1,0.014214399456977844
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,3,0.014169600605964661
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,7,0.014159999787807465
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,15,0.014206400513648987
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,32767,0.681870412826538
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,31,0.014084799587726593
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,127,0.016843199729919434
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,63,0.014095999300479889
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,255,0.02171359956264496
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,1023,0.028553599119186403
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,511,0.024132800102233887
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,2047,0.02882080078125
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,65535,1.3283328056335448
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,4095,0.02911520004272461
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,255,0.02157440036535263
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,131071,5.205508804321289
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,16383,0.05985119938850403
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,8191,0.03161439895629883
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,1,0.018515199422836304
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,32767,0.0866047978401184
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,7,0.01849920004606247
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,3,0.01857759952545166
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,63,0.018984000384807586
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,131071,0.24456639289855958
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,31,0.018769599497318268
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,255,0.031481599807739256
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,511,0.07351999878883361
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,1023,0.11710879802703858
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,2047,0.19678879976272584
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,131071,2.6207759857177733
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,4095,0.35801920890808103
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,8191,0.6838912010192871
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,16383,1.330179214477539
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,2,1,128,1,float16,float16,1,0.026100799441337585
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,2,1,128,1,float16,float16,3,0.026089599728584288
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,8191,0.04033119976520538
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,2,1,128,1,float16,float16,7,0.026233598589897156
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,2,1,128,1,float16,float16,15,0.026447999477386474
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,2,1,128,1,float16,float16,31,0.026556798815727235
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,2,1,128,1,float16,float16,63,0.02703840136528015
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,2,1,128,1,float16,float16,255,0.049691200256347656
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,2,1,128,1,float16,float16,127,0.033327999711036685
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,65535,0.1397968053817749
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,15,0.018884800374507904
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,2,1,128,1,float16,float16,511,0.1052623987197876
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,127,0.0232464000582695
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,32767,2.6254655838012697
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,2,1,128,1,float16,float16,2047,0.2893712043762207
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,2,1,128,1,float16,float16,1023,0.17177920341491698
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,2,1,128,1,float16,float16,4095,0.5308095932006835
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,2,1,128,1,float16,float16,1,0.03820320069789886
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,2,1,128,1,float16,float16,3,0.03824160099029541
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,2,1,128,1,float16,float16,7,0.038438400626182555
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,2,1,128,1,float16,float16,15,0.03861280083656311
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,2,1,128,1,float16,float16,31,0.03880159854888916
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,2,1,128,1,float16,float16,127,0.05391039848327637
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,2,1,128,1,float16,float16,63,0.03972800076007843
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,2,1,128,1,float16,float16,255,0.07758399844169617
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,2,1,128,1,float16,float16,1023,0.28036160469055177
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,2,1,128,1,float16,float16,511,0.17080639600753783
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,2,1,128,1,float16,float16,8191,1.024944019317627
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,2,1,128,1,float16,float16,2047,0.4781487941741943
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,1,0.014008000493049622
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,3,0.014177599549293518
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,15,0.01398719996213913
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,7,0.014144000411033631
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,31,0.014233599603176116
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,63,0.014159999787807465
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,127,0.01671999990940094
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,255,0.0217616006731987
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,256,1,2,1,128,1,float16,float16,16383,1.9902095794677734
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,511,0.02415039986371994
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,2047,0.029580798745155335
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,2,1,128,1,float16,float16,4095,0.8812272071838378
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,8191,0.05776159763336182
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,4095,0.03826720118522644
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,16383,0.08752800226211548
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,2,1,128,1,float16,float16,1,0.05663359761238098
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,65535,0.2391887903213501
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,65535,5.20251350402832
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,2,1,128,1,float16,float16,3,0.05664640069007874
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,512,1,2,1,128,1,float16,float16,8191,1.7228912353515624
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,131071,0.44735040664672854
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,2,1,128,1,float16,float16,7,0.05650879740715027
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,1023,0.02863999903202057
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,2,1,128,1,float16,float16,31,0.05748479962348938
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,2,1,128,1,float16,float16,15,0.05663040280342102
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,2,1,128,1,float16,float16,127,0.08540480136871338
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,32767,0.13714239597320557
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,2,1,128,1,float16,float16,63,0.06591519713401794
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,2,1,128,1,float16,float16,1,0.09392799735069275
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,2,1,128,1,float16,float16,255,0.12810879945755005
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,2,1,128,1,float16,float16,511,0.3154400110244751
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,2,1,128,1,float16,float16,3,0.09486879706382752
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,2,1,128,1,float16,float16,15,0.09454240202903748
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,2,1,128,1,float16,float16,31,0.09780799746513366
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,2,1,128,1,float16,float16,1023,0.5365583896636963
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,2,1,128,1,float16,float16,63,0.10566560029983521
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,2,1,128,1,float16,float16,255,0.2258608102798462
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,2,1,128,1,float16,float16,127,0.14404640197753907
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,1,0.014126400649547576
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,2,1,128,1,float16,float16,2047,0.9349776268005371
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,3,0.01417119950056076
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,7,0.014232000708580017
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,2,1,128,1,float16,float16,511,0.5875408172607421
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,31,0.014238399267196656
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,63,0.014446400105953217
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,15,0.01404159963130951
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,127,0.0169855996966362
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,511,0.025167998671531678
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,255,0.021859200298786165
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,1023,0.029185599088668822
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,4095,0.055615997314453124
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,8191,0.08565279841423035
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,2047,0.03820799887180328
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,2,1,128,1,float16,float16,1023,1.0094016075134278
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,32767,0.4034287929534912
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,16383,0.22106080055236815
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,2,1,128,1,float16,float16,2047,1.7656543731689454
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,65535,0.7666768074035645
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,131071,1.4935919761657714
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,2048,1,2,1,128,1,float16,float16,7,0.09453439712524414
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,generation_attention,flash_attention,1024,1,2,1,128,1,float16,float16,4095,1.7446912765502929
