framework,version,device,op_name,kernel_source,batch_size,isl,num_heads,num_key_value_heads,head_dim,beam_width,attn_dtype,kv_cache_dtype,step,latency
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,64,128,1,float16,float16,1,0.0875711977481842
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,64,128,1,float16,fp8,1,0.08561919927597046
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,64,128,1,float16,float16,3,0.08780639767646789
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,64,128,1,float16,fp8,3,0.0856544017791748
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,64,128,1,float16,float16,7,0.08798080086708068
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,64,128,1,float16,fp8,7,0.08576319813728332
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,64,128,1,float16,float16,15,0.08820800185203552
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,64,128,1,float16,fp8,15,0.08590559959411621
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,64,128,1,float16,float16,31,0.09687039852142335
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,64,128,1,float16,fp8,31,0.08636000156402587
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,64,128,1,float16,float16,63,0.10103360414505005
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,64,128,1,float16,fp8,63,0.09427040219306945
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,64,128,1,float16,float16,127,0.10831999778747559
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,64,128,1,float16,fp8,127,0.11770880222320557
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,64,128,1,float16,float16,255,0.15801440477371215
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,64,128,1,float16,fp8,255,0.15083999633789064
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,64,128,1,float16,float16,511,0.2752367973327637
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,64,128,1,float16,fp8,511,0.23561758995056153
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,64,128,1,float16,float16,1023,0.5039936065673828
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,64,128,1,float16,fp8,1023,0.375216007232666
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,64,128,1,float16,float16,2047,0.9686160087585449
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,64,128,1,float16,fp8,2047,0.6916816234588623
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,64,128,1,float16,float16,1,0.01138240024447441
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,64,128,1,float16,fp8,1,0.010515200346708298
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,64,128,1,float16,float16,3,0.01141280010342598
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,64,128,1,float16,fp8,3,0.01064639985561371
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,64,128,1,float16,float16,7,0.01141280010342598
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,64,128,1,float16,fp8,7,0.010604800283908844
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,64,128,1,float16,float16,15,0.011744000017642975
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,64,128,1,float16,fp8,15,0.01090560033917427
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,64,128,1,float16,float16,31,0.01154239997267723
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,64,128,1,float16,fp8,31,0.010729599744081497
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,64,128,1,float16,float16,63,0.011564800143241882
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,64,128,1,float16,fp8,63,0.010699199885129929
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,64,128,1,float16,float16,127,0.011747200042009354
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,64,128,1,float16,fp8,127,0.01156959980726242
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,64,128,1,float16,float16,255,0.01311040073633194
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,64,128,1,float16,fp8,255,0.012297599762678146
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,64,128,1,float16,float16,511,0.015161600708961488
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,64,128,1,float16,fp8,511,0.0146479994058609
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,64,128,1,float16,float16,1023,0.017696000635623932
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,64,128,1,float16,fp8,1023,0.016891199350357055
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,64,128,1,float16,float16,2047,0.03136799931526184
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,64,128,1,float16,fp8,2047,0.02173279970884323
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,64,128,1,float16,float16,1,0.011695999652147293
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,64,128,1,float16,fp8,1,0.010889600217342376
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,64,128,1,float16,float16,3,0.01159999966621399
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,64,128,1,float16,fp8,3,0.010769599676132202
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,64,128,1,float16,float16,7,0.011456000059843064
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,64,128,1,float16,fp8,7,0.0107744000852108
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,64,128,1,float16,float16,15,0.011459200084209442
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,64,128,1,float16,fp8,15,0.01080320030450821
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,64,128,1,float16,float16,31,0.011636800318956374
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,64,128,1,float16,fp8,31,0.01090560033917427
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,64,128,1,float16,float16,63,0.011907199770212174
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,64,128,1,float16,fp8,63,0.010942400246858597
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,64,128,1,float16,float16,127,0.012124799937009812
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,64,128,1,float16,fp8,127,0.011947199702262878
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,64,128,1,float16,float16,255,0.013419200479984284
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,64,128,1,float16,fp8,255,0.012532800436019897
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,64,128,1,float16,float16,511,0.015934400260448456
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,64,128,1,float16,fp8,511,0.016249600052833556
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,64,128,1,float16,float16,1023,0.030623999238014222
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,64,128,1,float16,fp8,1023,0.02056639939546585
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,64,128,1,float16,float16,2047,0.04535039961338043
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,64,128,1,float16,fp8,2047,0.03238399922847748
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,64,128,1,float16,float16,1,0.04953120052814484
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,64,128,1,float16,fp8,1,0.04865919947624207
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,64,128,1,float16,float16,3,0.04970400035381317
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,64,128,1,float16,fp8,3,0.04865759909152985
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,64,128,1,float16,float16,7,0.049777600169181827
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,64,128,1,float16,fp8,7,0.048607999086380006
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,64,128,1,float16,float16,15,0.04975520074367523
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,64,128,1,float16,fp8,15,0.048556798696517946
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,64,128,1,float16,float16,31,0.050160002708435056
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,64,128,1,float16,fp8,31,0.048548799753189084
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,64,128,1,float16,float16,63,0.058475202322006224
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,64,128,1,float16,fp8,63,0.04915199875831604
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,64,128,1,float16,float16,127,0.061875200271606444
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,64,128,1,float16,fp8,127,0.06569920182228088
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,64,128,1,float16,float16,255,0.09029759764671326
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,64,128,1,float16,fp8,255,0.08212479948997498
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,64,128,1,float16,float16,511,0.1468832015991211
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,64,128,1,float16,fp8,511,0.12394560575485229
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,64,128,1,float16,float16,1023,0.26251840591430664
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,64,128,1,float16,fp8,1023,0.19268800020217897
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,64,128,1,float16,float16,2047,0.4924816131591797
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,64,128,1,float16,fp8,2047,0.3465872049331665
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,64,128,1,float16,float16,1,0.014020800590515137
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,64,128,1,float16,fp8,1,0.013342399895191193
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,64,128,1,float16,float16,3,0.01404159963130951
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,64,128,1,float16,fp8,3,0.013396799564361572
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,64,128,1,float16,float16,7,0.01388320028781891
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,64,128,1,float16,fp8,7,0.013142399489879608
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,64,128,1,float16,float16,15,0.013926400244235993
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,64,128,1,float16,fp8,15,0.013195200264453888
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,64,128,1,float16,float16,31,0.014046399295330048
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,64,128,1,float16,fp8,31,0.013303999602794648
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,64,128,1,float16,float16,63,0.014244799315929414
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,64,128,1,float16,fp8,63,0.013443200290203095
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,64,128,1,float16,float16,127,0.01435679942369461
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,64,128,1,float16,fp8,127,0.015385599434375763
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,64,128,1,float16,fp8,255,0.016756799817085267
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,64,128,1,float16,float16,511,0.031140801310539246
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,64,128,1,float16,float16,1023,0.04763199985027313
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,64,128,1,float16,fp8,1023,0.034307199716567996
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,64,128,1,float16,float16,2047,0.0771503984928131
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,64,128,1,float16,fp8,2047,0.05414400100708008
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,64,128,1,float16,float16,1,0.1651535987854004
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,64,128,1,float16,float16,3,0.16519360542297362
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,64,128,1,float16,float16,7,0.16537920236587525
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,64,128,1,float16,float16,255,0.017723199725151063
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,64,128,1,float16,float16,15,0.1707376003265381
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,64,128,1,float16,float16,31,0.1821344017982483
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,64,128,1,float16,float16,63,0.18689919710159303
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,64,128,1,float16,fp8,511,0.022675199806690215
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,64,128,1,float16,float16,127,0.20014240741729736
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,64,128,1,float16,fp8,127,0.22185440063476564
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,64,128,1,float16,float16,255,0.2943311929702759
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,64,128,1,float16,fp8,255,0.28551199436187746
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,64,128,1,float16,float16,511,0.5353983879089356
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,64,128,1,float16,fp8,511,0.4462751865386963
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,64,128,1,float16,fp8,1,0.16070560216903687
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,64,128,1,float16,float16,1,0.31924800872802733
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,64,128,1,float16,fp8,1023,0.732747220993042
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,64,128,1,float16,float16,1023,0.9993295669555664
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,64,128,1,float16,fp8,63,0.17719839811325072
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,64,128,1,float16,fp8,3,0.15959199666976928
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,64,128,1,float16,fp8,15,0.16065759658813478
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,64,128,1,float16,float16,3,0.319267201423645
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,64,128,1,float16,float16,7,0.32564640045166016
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,64,128,1,float16,float16,15,0.3452735900878906
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,64,128,1,float16,fp8,15,0.31573920249938964
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,64,128,1,float16,float16,31,0.34976160526275635
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,64,128,1,float16,fp8,31,0.3291680097579956
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,64,128,1,float16,float16,63,0.3561055898666382
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,64,128,1,float16,fp8,63,0.34099040031433103
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,64,128,1,float16,float16,127,0.37984960079193114
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,64,128,1,float16,fp8,127,0.4308527946472168
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,64,128,1,float16,float16,255,0.5645952224731445
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,64,64,128,1,float16,float16,1,0.6658815860748291
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,64,128,1,float16,fp8,255,0.5556992053985595
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,64,128,1,float16,fp8,1,0.31058559417724607
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,64,64,128,1,float16,fp8,1,0.615496015548706
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,64,128,1,float16,fp8,7,0.15941439867019652
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,64,128,1,float16,fp8,31,0.16732319593429565
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,64,128,1,float16,fp8,3,0.3108432054519653
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,64,64,128,1,float16,float16,3,0.6673535823822021
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,64,64,128,1,float16,fp8,3,0.6114783763885498
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,64,64,128,1,float16,float16,7,0.6778960227966309
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,64,64,128,1,float16,fp8,7,0.6241487979888916
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,64,128,1,float16,fp8,7,0.3093008041381836
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,64,64,128,1,float16,fp8,15,0.6411647796630859
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,64,64,128,1,float16,float16,31,0.6814112186431884
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,64,64,128,1,float16,float16,15,0.6771872043609619
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,64,64,128,1,float16,float16,63,0.6984943866729736
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,64,128,1,float16,float16,3,0.019351999461650848
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,64,64,128,1,float16,fp8,63,0.6666240215301513
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,64,128,1,float16,float16,1,0.019278399646282196
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,64,128,1,float16,float16,7,0.01929119974374771
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,64,128,1,float16,fp8,3,0.018190400302410127
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,64,128,1,float16,fp8,7,0.018027199804782866
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,64,128,1,float16,fp8,15,0.017987200617790224
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,64,128,1,float16,float16,31,0.01923519968986511
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,64,128,1,float16,fp8,31,0.018054400384426118
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,64,128,1,float16,fp8,63,0.017875200510025023
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,64,128,1,float16,float16,127,0.020017600059509276
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,64,128,1,float16,fp8,127,0.021268799901008606
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,64,128,1,float16,float16,255,0.03417760133743286
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,64,128,1,float16,fp8,255,0.024875199794769286
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,64,128,1,float16,float16,511,0.048475199937820436
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,64,128,1,float16,fp8,511,0.03881439864635468
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,64,128,1,float16,float16,1023,0.0791808009147644
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,64,128,1,float16,fp8,1023,0.056796801090240476
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,64,128,1,float16,float16,2047,0.1372320055961609
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,64,128,1,float16,fp8,2047,0.09571359753608703
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,64,64,128,1,float16,float16,1,1.3366543769836425
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,64,64,128,1,float16,fp8,1,1.2568511962890625
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,64,64,128,1,float16,fp8,31,0.6460336208343506
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,64,128,1,float16,fp8,1,0.017859199643135072
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,64,64,128,1,float16,float16,127,0.7512256145477295
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,64,128,1,float16,float16,15,0.019419200718402863
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,64,128,1,float16,float16,63,0.01939679980278015
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,64,64,128,1,float16,fp8,127,0.847265625
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,64,64,128,1,float16,fp8,3,1.2572624206542968
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,64,64,128,1,float16,float16,7,1.3316080093383789
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,64,64,128,1,float16,fp8,7,1.2589232444763183
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,64,64,128,1,float16,float16,15,1.3514623641967773
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,64,64,128,1,float16,fp8,15,1.2680784225463868
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,64,64,128,1,float16,fp8,31,1.2757583618164063
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,64,64,128,1,float16,float16,31,1.34442081451416
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,64,64,128,1,float16,float16,63,1.3812463760375977
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,64,64,128,1,float16,fp8,63,1.3085616111755372
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,64,64,128,1,float16,float16,1,2.642625617980957
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,64,64,128,1,float16,float16,3,1.336464023590088
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,64,64,128,1,float16,float16,3,2.642684745788574
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,64,64,128,1,float16,fp8,3,2.4941551208496096
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,64,64,128,1,float16,float16,7,2.651063919067383
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,64,64,128,1,float16,fp8,7,2.5058895111083985
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,64,128,1,float16,float16,1,0.029284799098968507
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,64,128,1,float16,fp8,1,0.02784799933433533
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,64,128,1,float16,float16,3,0.029348799586296083
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,64,128,1,float16,fp8,3,0.028036800026893616
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,64,128,1,float16,float16,7,0.02938719987869263
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,64,128,1,float16,fp8,7,0.02804960012435913
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,64,64,128,1,float16,float16,15,2.6575679779052734
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,64,128,1,float16,fp8,15,0.02789120078086853
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,64,128,1,float16,fp8,31,0.028086400032043456
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,64,128,1,float16,float16,31,0.029691201448440552
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,64,128,1,float16,float16,63,0.030347201228141784
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,64,128,1,float16,fp8,63,0.028262400627136232
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,64,128,1,float16,float16,127,0.038140800595283506
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,64,128,1,float16,fp8,127,0.035283198952674864
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,64,128,1,float16,fp8,255,0.044803199172019956
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,64,128,1,float16,float16,255,0.0545520007610321
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,64,128,1,float16,float16,511,0.08218399882316589
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,64,128,1,float16,fp8,511,0.06772480010986329
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,64,64,128,1,float16,fp8,15,2.521241569519043
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,64,128,1,float16,float16,1023,0.1417647957801819
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,64,128,1,float16,fp8,1023,0.1016543984413147
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,64,64,128,1,float16,float16,31,2.685016059875488
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,64,128,1,float16,float16,2047,0.25553278923034667
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,64,128,1,float16,fp8,2047,0.17904160022735596
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,64,64,128,1,float16,fp8,31,2.5317663192749023
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,48,48,128,1,float16,float16,1,0.06853280067443848
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,48,48,128,1,float16,fp8,1,0.06761760115623475
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,64,64,128,1,float16,fp8,1,2.510660743713379
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,48,48,128,1,float16,float16,15,0.06935999989509582
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,48,48,128,1,float16,float16,31,0.07145439982414245
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,48,48,128,1,float16,fp8,31,0.06778240203857422
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,48,48,128,1,float16,float16,63,0.0792240023612976
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,48,48,128,1,float16,fp8,63,0.07502560019493103
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,48,48,128,1,float16,float16,127,0.08485280275344849
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,64,128,1,float16,float16,15,0.029315200448036195
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,48,48,128,1,float16,fp8,127,0.0924784004688263
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,48,48,128,1,float16,fp8,255,0.11749919652938842
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,48,48,128,1,float16,float16,511,0.2114272117614746
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,48,48,128,1,float16,fp8,511,0.1807520031929016
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,48,48,128,1,float16,float16,1023,0.3825695991516113
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,48,48,128,1,float16,float16,3,0.06852319836616516
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,48,48,128,1,float16,fp8,3,0.06734079718589783
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,48,48,128,1,float16,float16,7,0.06858720183372498
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,48,48,128,1,float16,float16,1,0.010791999846696853
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,48,48,128,1,float16,fp8,1023,0.28535358905792235
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,48,48,128,1,float16,fp8,15,0.06774880290031433
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,48,48,128,1,float16,fp8,1,0.010235200077295304
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,48,48,128,1,float16,float16,3,0.010902400314807891
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,48,48,128,1,float16,float16,7,0.010737600177526474
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,48,48,128,1,float16,fp8,3,0.010289599746465683
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,48,48,128,1,float16,float16,15,0.010911999642848969
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,48,48,128,1,float16,fp8,15,0.010313600301742554
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,48,48,128,1,float16,float16,31,0.010868799686431885
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,48,48,128,1,float16,fp8,31,0.010292799770832061
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,48,48,128,1,float16,fp8,63,0.010262399911880493
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,48,48,128,1,float16,fp8,127,0.011502400040626526
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,48,48,128,1,float16,float16,63,0.010971199721097946
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,48,48,128,1,float16,float16,255,0.012703999876976013
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,48,48,128,1,float16,fp8,255,0.012004800140857697
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,48,48,128,1,float16,float16,511,0.014500799775123595
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,48,48,128,1,float16,fp8,511,0.01430719941854477
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,48,48,128,1,float16,float16,1023,0.01677920073270798
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,48,48,128,1,float16,fp8,1023,0.018988800048828126
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,48,48,128,1,float16,float16,255,0.12339040040969848
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,48,48,128,1,float16,float16,2047,0.02900800108909607
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,48,48,128,1,float16,fp8,2047,0.02489439994096756
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,48,48,128,1,float16,float16,1,0.011088000237941742
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,48,48,128,1,float16,fp8,1,0.010513599961996078
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,48,48,128,1,float16,float16,7,0.01106560006737709
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,48,48,128,1,float16,fp8,3,0.010424000024795533
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,48,48,128,1,float16,fp8,7,0.010595200210809707
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,48,48,128,1,float16,float16,15,0.011088000237941742
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,48,48,128,1,float16,float16,31,0.011088000237941742
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,48,48,128,1,float16,fp8,31,0.01064160019159317
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,48,48,128,1,float16,float16,63,0.011121600121259689
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,48,48,128,1,float16,fp8,63,0.010734400153160096
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,48,48,128,1,float16,float16,127,0.01156800016760826
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,48,48,128,1,float16,fp8,127,0.011668799817562104
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,48,48,128,1,float16,float16,255,0.012824000418186187
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,48,48,128,1,float16,fp8,255,0.012251199781894683
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,48,48,128,1,float16,float16,511,0.014891199767589569
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,48,48,128,1,float16,fp8,511,0.01842239946126938
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,48,48,128,1,float16,fp8,7,0.06719520092010497
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,48,48,128,1,float16,float16,1023,0.02800000011920929
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,48,48,128,1,float16,fp8,1023,0.022526399791240694
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,48,48,128,1,float16,float16,2047,0.04141919910907745
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,48,48,128,1,float16,fp8,2047,0.03373120129108429
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,48,48,128,1,float16,float16,1,0.039103999733924866
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,48,48,128,1,float16,fp8,1,0.038545599579811095
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,48,48,128,1,float16,float16,3,0.03899999856948853
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,48,48,128,1,float16,fp8,3,0.03868159949779511
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,48,48,128,1,float16,float16,7,0.039052799344062805
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,48,48,128,1,float16,float16,15,0.03908160030841827
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,48,48,128,1,float16,fp8,15,0.0386352002620697
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,48,48,128,1,float16,float16,31,0.03927200138568878
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,48,48,128,1,float16,float16,127,0.011315199732780456
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,48,48,128,1,float16,fp8,31,0.03870239853858948
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,48,48,128,1,float16,fp8,63,0.038689601421356204
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,48,48,128,1,float16,float16,63,0.046009600162506104
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,48,48,128,1,float16,float16,127,0.049711999297142026
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,48,48,128,1,float16,fp8,127,0.05142239928245544
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,48,48,128,1,float16,float16,255,0.07185440063476563
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,48,48,128,1,float16,float16,3,0.011273600161075592
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,48,48,128,1,float16,fp8,255,0.06278560161590577
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,48,48,128,1,float16,fp8,511,0.09582560062408448
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,48,48,128,1,float16,fp8,15,0.010664000362157821
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,48,48,128,1,float16,float16,1023,0.1991984009742737
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,48,48,128,1,float16,fp8,1023,0.14772640466690062
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,48,48,128,1,float16,float16,2047,0.37367680072784426
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,48,48,128,1,float16,fp8,2047,0.262604808807373
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,48,48,128,1,float16,float16,1,0.013491199910640716
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,48,48,128,1,float16,fp8,1,0.012654399871826172
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,48,48,128,1,float16,float16,3,0.013512000441551208
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,48,48,128,1,float16,fp8,3,0.012630400061607362
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,48,48,128,1,float16,float16,7,0.01348000019788742
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,48,48,128,1,float16,fp8,7,0.012667199969291687
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,48,48,128,1,float16,float16,15,0.013527999818325042
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,48,48,128,1,float16,fp8,15,0.012700800597667695
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,48,48,128,1,float16,float16,31,0.013497599959373474
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,48,48,128,1,float16,fp8,31,0.012807999551296235
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,48,48,128,1,float16,float16,63,0.013451200723648072
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,48,48,128,1,float16,fp8,63,0.012854400277137756
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,48,48,128,1,float16,fp8,7,0.03866879940032959
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,48,48,128,1,float16,float16,127,0.013752000033855438
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,48,48,128,1,float16,fp8,127,0.014742399752140044
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,48,48,128,1,float16,fp8,255,0.01627359986305237
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,48,48,128,1,float16,float16,511,0.02513279914855957
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,48,48,128,1,float16,fp8,511,0.022521600127220154
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,48,48,128,1,float16,fp8,2047,0.5179984092712402
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,48,48,128,1,float16,float16,2047,0.7282048225402832
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,48,48,128,1,float16,float16,2047,0.06899999976158142
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,48,48,128,1,float16,fp8,2047,0.0548687994480133
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,48,48,128,1,float16,float16,1,0.12682080268859863
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,48,48,128,1,float16,float16,3,0.1271008014678955
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,48,48,128,1,float16,float16,511,0.11364320516586304
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,48,48,128,1,float16,fp8,3,0.12441600561141967
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,48,48,128,1,float16,float16,7,0.12708799839019774
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,48,48,128,1,float16,fp8,7,0.12436480522155761
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,48,48,128,1,float16,float16,15,0.12760640382766725
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,48,48,128,1,float16,fp8,15,0.12441439628601074
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,48,48,128,1,float16,float16,31,0.1387392044067383
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,48,48,128,1,float16,fp8,31,0.12761919498443602
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,48,48,128,1,float16,float16,63,0.1431615948677063
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,48,48,128,1,float16,fp8,63,0.13676480054855347
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,48,48,128,1,float16,float16,127,0.15312000513076782
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,48,48,128,1,float16,fp8,127,0.1716431975364685
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,48,48,128,1,float16,float16,255,0.22578558921813965
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,48,48,128,1,float16,fp8,255,0.21808319091796874
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,48,48,128,1,float16,fp8,7,0.010249599814414978
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,48,48,128,1,float16,float16,255,0.01693280041217804
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,48,48,128,1,float16,float16,511,0.40635361671447756
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,48,48,128,1,float16,float16,1023,0.040520000457763675
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,48,48,128,1,float16,float16,1,0.24352641105651857
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,48,48,128,1,float16,fp8,1,0.2386080026626587
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,48,48,128,1,float16,float16,3,0.24312639236450195
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,48,48,128,1,float16,fp8,1023,0.5456719875335694
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,48,48,128,1,float16,fp8,1,0.12459039688110352
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,48,48,128,1,float16,float16,1023,0.7431647777557373
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,48,48,128,1,float16,float16,7,0.24437921047210692
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,48,48,128,1,float16,fp8,7,0.23880159854888916
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,48,48,128,1,float16,float16,15,0.2591536045074463
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,48,48,128,1,float16,float16,31,0.26388959884643554
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,48,48,128,1,float16,fp8,15,0.24255039691925048
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,48,48,128,1,float16,fp8,31,0.25364639759063723
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,48,48,128,1,float16,float16,63,0.2728832006454468
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,48,48,128,1,float16,fp8,63,0.2619999885559082
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,48,48,128,1,float16,float16,127,0.29032800197601316
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,48,48,128,1,float16,fp8,127,0.33067998886108396
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,48,48,128,1,float16,float16,255,0.4275360107421875
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,48,48,128,1,float16,float16,1,0.48462557792663574
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,48,48,128,1,float16,fp8,255,0.42303037643432617
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,48,48,128,1,float16,fp8,1,0.4678703784942627
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,48,48,128,1,float16,fp8,3,0.471343994140625
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,48,48,128,1,float16,float16,3,0.4905248165130615
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,48,48,128,1,float16,fp8,7,0.4728384017944336
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,48,48,128,1,float16,float16,7,0.5043424129486084
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,48,48,128,1,float16,float16,15,0.5132383823394775
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,48,48,128,1,float16,fp8,1023,0.03298240005970001
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,48,48,128,1,float16,fp8,15,0.4914976119995117
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,48,48,128,1,float16,float16,31,0.5193088054656982
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,48,48,128,1,float16,fp8,31,0.501203203201294
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,48,48,128,1,float16,float16,1,0.016228799521923066
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,48,48,128,1,float16,float16,63,0.5298431873321533
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,48,48,128,1,float16,fp8,1,0.015516799688339234
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,48,48,128,1,float16,float16,3,0.016313600540161132
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,48,48,128,1,float16,float16,7,0.016390399634838106
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,48,48,128,1,float16,fp8,7,0.015324799716472626
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,48,48,128,1,float16,float16,15,0.016356800496578217
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,48,48,128,1,float16,fp8,3,0.23849279880523683
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,48,48,128,1,float16,float16,127,0.564846420288086
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,48,48,128,1,float16,float16,31,0.016339200735092162
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,48,48,128,1,float16,fp8,31,0.015563200414180755
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,48,48,128,1,float16,fp8,63,0.015467199683189391
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,48,48,128,1,float16,float16,127,0.016739200055599212
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,48,48,128,1,float16,fp8,127,0.01797440052032471
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,48,48,128,1,float16,float16,255,0.02570880055427551
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,48,48,128,1,float16,fp8,255,0.02054080069065094
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,48,48,128,1,float16,float16,511,0.03886080086231232
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,48,48,128,1,float16,float16,1023,0.062294399738311766
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,48,48,128,1,float16,fp8,1023,0.04491840004920959
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,48,48,128,1,float16,float16,2047,0.10615520477294922
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,48,48,128,1,float16,fp8,511,0.34282560348510743
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,48,48,128,1,float16,fp8,2047,0.07503520250320435
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,48,48,128,1,float16,fp8,63,0.5118624210357666
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,48,48,128,1,float16,fp8,15,0.01552959978580475
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,48,48,128,1,float16,float16,63,0.01643519997596741
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,48,48,128,1,float16,float16,1,1.0005536079406738
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,48,48,128,1,float16,fp8,1,0.9426464080810547
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,48,48,128,1,float16,float16,3,1.0094271659851075
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,48,48,128,1,float16,fp8,3,0.9516143798828125
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,48,48,128,1,float16,float16,7,1.0052080154418945
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,48,48,128,1,float16,fp8,7,0.9600576400756836
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,48,48,128,1,float16,float16,15,1.015395164489746
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,48,48,128,1,float16,fp8,15,0.9697967529296875
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,48,48,128,1,float16,float16,31,1.0169679641723632
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,48,48,128,1,float16,fp8,31,0.9792672157287597
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,48,48,128,1,float16,fp8,3,0.015548799932003022
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,48,48,128,1,float16,float16,63,1.0460415840148927
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,48,48,128,1,float16,fp8,63,1.0026448249816895
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,48,48,128,1,float16,fp8,127,0.6532559871673584
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,48,48,128,1,float16,fp8,511,0.03136639893054962
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,48,48,128,1,float16,float16,1,1.9875280380249023
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,48,48,128,1,float16,fp8,1,1.902992057800293
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,48,48,128,1,float16,float16,3,1.9921167373657227
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,48,48,128,1,float16,float16,1,0.024184000492095948
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,48,48,128,1,float16,fp8,1,0.022966399788856506
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,48,48,128,1,float16,fp8,3,1.9119247436523437
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,48,48,128,1,float16,float16,3,0.024040000140666963
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,48,48,128,1,float16,fp8,7,1.9168783187866212
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,48,48,128,1,float16,float16,7,2.0094112396240233
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,48,48,128,1,float16,fp8,3,0.022977599501609804
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,48,48,128,1,float16,float16,7,0.024057599902153014
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,48,48,128,1,float16,fp8,7,0.023193599283695222
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,48,48,128,1,float16,fp8,15,0.023094399273395537
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,48,48,128,1,float16,float16,31,0.02436479926109314
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,48,48,128,1,float16,float16,15,2.0142431259155273
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,48,48,128,1,float16,float16,63,0.02433439940214157
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,48,48,128,1,float16,fp8,127,0.028112000226974486
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,48,48,128,1,float16,float16,127,0.03158240020275116
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,48,48,128,1,float16,fp8,15,1.9413312911987304
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,48,48,128,1,float16,fp8,255,0.03521920144557953
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,48,48,128,1,float16,fp8,511,0.05279679894447327
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,48,48,128,1,float16,float16,511,0.06501759886741638
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,48,48,128,1,float16,fp8,1023,0.07973120212554932
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,48,48,128,1,float16,float16,1023,0.10921599864959716
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,48,48,128,1,float16,float16,31,2.0150848388671876
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,48,48,128,1,float16,fp8,31,1.9387056350708007
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,48,48,128,1,float16,fp8,2047,0.13760800361633302
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,48,48,128,1,float16,float16,2047,0.19538880586624147
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,40,40,128,1,float16,float16,1,0.05953279733657837
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,40,40,128,1,float16,float16,7,0.05952159762382507
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,40,40,128,1,float16,fp8,1,0.05883839726448059
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,40,40,128,1,float16,fp8,7,0.059084802865982056
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,40,40,128,1,float16,float16,15,0.05964319705963135
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,40,40,128,1,float16,fp8,15,0.05912799835205078
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,40,40,128,1,float16,float16,31,0.06142079830169678
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,40,40,128,1,float16,fp8,31,0.05902400016784668
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,40,40,128,1,float16,float16,63,0.06739360094070435
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,40,40,128,1,float16,fp8,63,0.061926400661468504
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,40,40,128,1,float16,float16,127,0.0725600004196167
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,40,40,128,1,float16,fp8,127,0.08045439720153809
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,40,40,128,1,float16,float16,255,0.10799679756164551
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,40,40,128,1,float16,float16,511,0.17899680137634277
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,48,48,128,1,float16,float16,15,0.024216000735759736
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,48,48,128,1,float16,fp8,31,0.023235200345516203
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,48,48,128,1,float16,fp8,63,0.023044799268245698
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,48,48,128,1,float16,float16,255,0.04392000138759613
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,40,40,128,1,float16,fp8,511,0.15416640043258667
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,40,40,128,1,float16,float16,1023,0.3228640079498291
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,40,40,128,1,float16,fp8,1023,0.2418207883834839
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,40,40,128,1,float16,float16,2047,0.6096303939819336
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,40,40,128,1,float16,float16,3,0.01064319983124733
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,40,40,128,1,float16,float16,7,0.010606399923563003
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,40,40,128,1,float16,fp8,2047,0.4352735996246338
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,40,40,128,1,float16,fp8,7,0.010606399923563003
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,40,40,128,1,float16,fp8,3,0.010524799674749374
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,40,40,128,1,float16,fp8,15,0.010606399923563003
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,40,40,128,1,float16,float16,3,0.05974079966545105
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,40,40,128,1,float16,fp8,31,0.010623999685049058
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,40,40,128,1,float16,float16,15,0.010580799728631973
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,40,40,128,1,float16,float16,63,0.010718400031328202
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,40,40,128,1,float16,fp8,63,0.010718400031328202
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,40,40,128,1,float16,float16,127,0.010969600081443787
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,40,40,128,1,float16,float16,255,0.012569600343704223
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,40,40,128,1,float16,fp8,255,0.012270399928092956
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,40,40,128,1,float16,float16,511,0.014430400729179383
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,40,40,128,1,float16,fp8,511,0.014212800562381745
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,40,40,128,1,float16,float16,1023,0.015878400206565856
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,40,40,128,1,float16,float16,2047,0.020203199982643128
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,40,40,128,1,float16,fp8,2047,0.018838399648666383
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,40,40,128,1,float16,float16,1,0.010585600137710571
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,40,40,128,1,float16,fp8,1,0.010582400113344192
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,40,40,128,1,float16,float16,3,0.01074879989027977
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,40,40,128,1,float16,fp8,3,0.010649599879980088
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,40,40,128,1,float16,fp8,255,0.10058879852294922
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,40,40,128,1,float16,float16,7,0.010911999642848969
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,40,40,128,1,float16,fp8,7,0.01074720025062561
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,40,40,128,1,float16,float16,15,0.01090560033917427
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,40,40,128,1,float16,fp8,15,0.010814400017261505
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,40,40,128,1,float16,float16,31,0.010955200344324113
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,40,40,128,1,float16,fp8,31,0.010830400139093399
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,40,40,128,1,float16,float16,63,0.0110895998775959
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,40,40,128,1,float16,fp8,63,0.010700800269842149
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,40,40,128,1,float16,float16,127,0.011072000116109848
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,40,40,128,1,float16,fp8,127,0.011535999923944473
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,40,40,128,1,float16,float16,255,0.012649600207805634
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,40,40,128,1,float16,fp8,255,0.012371200323104858
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,40,40,128,1,float16,float16,511,0.01462559998035431
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,40,40,128,1,float16,float16,1,0.010729599744081497
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,40,40,128,1,float16,fp8,511,0.018743999302387238
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,40,40,128,1,float16,float16,1023,0.024243199825286867
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,40,40,128,1,float16,fp8,1023,0.02300959974527359
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,40,40,128,1,float16,float16,2047,0.04019519984722138
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,40,40,128,1,float16,fp8,2047,0.033435198664665225
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,40,40,128,1,float16,fp8,1,0.010689599812030793
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,40,40,128,1,float16,fp8,1,0.034204798936843875
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,40,40,128,1,float16,float16,3,0.03457280099391937
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,40,40,128,1,float16,fp8,3,0.03440319895744324
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,40,40,128,1,float16,float16,7,0.034585601091384886
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,40,40,128,1,float16,fp8,7,0.03455359935760498
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,40,40,128,1,float16,fp8,15,0.03452799916267395
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,40,40,128,1,float16,float16,31,0.03480960130691528
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,40,40,128,1,float16,fp8,31,0.034646400809288026
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,40,40,128,1,float16,float16,31,0.010620799660682679
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,40,40,128,1,float16,fp8,3,0.05916159749031067
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,40,40,128,1,float16,float16,63,0.03707360029220581
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,40,40,128,1,float16,fp8,63,0.03424960076808929
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,40,40,128,1,float16,fp8,127,0.011454399675130844
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,40,40,128,1,float16,float16,255,0.06286720037460328
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,40,40,128,1,float16,fp8,1023,0.015422399342060088
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,40,40,128,1,float16,fp8,255,0.055244797468185426
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,40,40,128,1,float16,float16,511,0.09775840044021607
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,40,40,128,1,float16,fp8,511,0.08301759958267212
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,40,40,128,1,float16,float16,1023,0.1701151967048645
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,40,40,128,1,float16,fp8,1023,0.12747360467910768
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,40,40,128,1,float16,float16,1,0.012942400574684144
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,40,40,128,1,float16,float16,2047,0.313427209854126
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,40,40,128,1,float16,fp8,2047,0.22414720058441162
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,40,40,128,1,float16,fp8,3,0.012886400520801543
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,40,40,128,1,float16,float16,7,0.012904000282287598
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,40,40,128,1,float16,fp8,7,0.013068799674510957
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,40,40,128,1,float16,float16,15,0.01311199963092804
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,40,40,128,1,float16,fp8,15,0.01311040073633194
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,40,40,128,1,float16,float16,31,0.013171200454235078
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,40,40,128,1,float16,fp8,31,0.013035200536251068
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,40,40,128,1,float16,float16,63,0.013132800161838532
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,40,40,128,1,float16,fp8,63,0.012883199751377106
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,40,40,128,1,float16,float16,127,0.013214400410652161
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,40,40,128,1,float16,float16,1,0.03438880145549774
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,40,40,128,1,float16,fp8,127,0.014847999811172486
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,40,40,128,1,float16,float16,255,0.016502399742603303
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,40,40,128,1,float16,float16,511,0.021484799683094025
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,40,40,128,1,float16,fp8,511,0.022601599991321563
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,40,40,128,1,float16,float16,1023,0.03955360054969788
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,40,40,128,1,float16,fp8,1023,0.0319599986076355
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,40,40,128,1,float16,float16,2047,0.06285759806632996
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,40,40,128,1,float16,fp8,2047,0.053620797395706174
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,40,40,128,1,float16,fp8,127,0.04436799883842468
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,40,40,128,1,float16,float16,127,0.042955198884010316
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,40,40,128,1,float16,float16,1,0.10846079587936401
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,40,40,128,1,float16,fp8,1,0.10726560354232788
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,40,40,128,1,float16,float16,3,0.10877120494842529
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,40,40,128,1,float16,fp8,3,0.1083359956741333
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,40,40,128,1,float16,float16,7,0.10877280235290528
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,40,40,128,1,float16,fp8,7,0.10750880241394042
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,40,40,128,1,float16,fp8,15,0.10746879577636718
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,40,40,128,1,float16,float16,31,0.11676479578018188
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,40,40,128,1,float16,fp8,31,0.10892479419708252
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,40,40,128,1,float16,float16,3,0.013064000010490417
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,40,40,128,1,float16,fp8,63,0.11833120584487915
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,40,40,128,1,float16,float16,127,0.13150080442428588
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,40,40,128,1,float16,fp8,127,0.1482095956802368
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,40,40,128,1,float16,float16,255,0.19411200284957886
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,40,40,128,1,float16,fp8,255,0.18675999641418456
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,40,40,128,1,float16,float16,511,0.34171199798583984
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,40,40,128,1,float16,fp8,511,0.2905519962310791
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,40,40,128,1,float16,fp8,255,0.016441600024700166
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,40,40,128,1,float16,float16,15,0.03469919860363006
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,40,40,128,1,float16,float16,1,0.2081376075744629
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,40,40,128,1,float16,fp8,1,0.20484800338745118
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,40,40,128,1,float16,float16,3,0.20685920715332032
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,40,40,128,1,float16,fp8,1023,0.46035199165344237
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,40,40,128,1,float16,float16,1023,0.6225791931152344
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,40,40,128,1,float16,fp8,3,0.2046351909637451
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,40,40,128,1,float16,float16,7,0.20849120616912842
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,40,40,128,1,float16,fp8,7,0.20313920974731445
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,40,40,128,1,float16,float16,31,0.2243328094482422
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,40,40,128,1,float16,fp8,15,0.2044895887374878
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,40,40,128,1,float16,fp8,31,0.21765758991241455
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,40,40,128,1,float16,float16,63,0.23023359775543212
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,40,40,128,1,float16,fp8,63,0.22535040378570556
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,40,40,128,1,float16,float16,15,0.11047359704971313
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,40,40,128,1,float16,float16,127,0.24666080474853516
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,40,40,128,1,float16,fp8,127,0.2838655948638916
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,40,40,128,1,float16,fp8,1,0.012852799892425538
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,40,40,128,1,float16,float16,255,0.36343679428100584
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,40,40,128,1,float16,float16,63,0.12172800302505493
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,40,40,128,1,float16,fp8,255,0.3594320058822632
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,40,40,128,1,float16,fp8,1,0.40026559829711916
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,40,40,128,1,float16,float16,1,0.40516319274902346
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,40,40,128,1,float16,float16,3,0.40975518226623536
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,40,40,128,1,float16,fp8,3,0.3982656002044678
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,40,40,128,1,float16,fp8,7,0.3984783887863159
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,40,40,128,1,float16,float16,7,0.4175055980682373
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,40,40,128,1,float16,fp8,15,0.414734411239624
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,40,40,128,1,float16,float16,15,0.43339200019836427
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,40,40,128,1,float16,float16,31,0.4358047962188721
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,40,40,128,1,float16,float16,1,0.01584800034761429
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,40,40,128,1,float16,fp8,31,0.4244688034057617
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,40,40,128,1,float16,fp8,1,0.01558080017566681
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,40,40,128,1,float16,float16,63,0.44834561347961427
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,40,40,128,1,float16,fp8,63,0.4356800079345703
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,40,40,128,1,float16,float16,3,0.015892800688743592
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,40,40,128,1,float16,float16,15,0.015961599349975587
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,40,40,128,1,float16,float16,127,0.4839456081390381
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,40,40,128,1,float16,fp8,7,0.015542399883270264
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,40,40,128,1,float16,float16,7,0.015862399339675905
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,40,40,128,1,float16,fp8,15,0.015607999265193939
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,40,40,128,1,float16,fp8,127,0.5558271884918213
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,40,40,128,1,float16,fp8,31,0.01564960032701492
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,40,40,128,1,float16,fp8,63,0.015468800067901611
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,40,40,128,1,float16,float16,31,0.01600960046052933
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,40,40,128,1,float16,fp8,127,0.018190400302410127
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,40,40,128,1,float16,fp8,255,0.020739200711250304
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,40,40,128,1,float16,float16,127,0.01642879992723465
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,40,40,128,1,float16,float16,511,0.03517760038375854
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,40,40,128,1,float16,float16,1023,0.05615839958190918
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,40,40,128,1,float16,fp8,1023,0.0458976000547409
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,40,40,128,1,float16,float16,2047,0.09482079744338989
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,40,40,128,1,float16,fp8,2047,0.07507039904594422
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,40,40,128,1,float16,float16,15,0.21574881076812744
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,40,40,128,1,float16,float16,1,0.8452079772949219
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,40,40,128,1,float16,float16,3,0.8408143997192383
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,40,40,128,1,float16,fp8,3,0.7986991882324219
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,40,40,128,1,float16,float16,7,0.8439888000488281
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,40,40,128,1,float16,fp8,3,0.015643200278282164
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,40,40,128,1,float16,fp8,7,0.8150176048278809
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,40,40,128,1,float16,float16,63,0.01594880074262619
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,40,40,128,1,float16,float16,255,0.021289600431919097
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,40,40,128,1,float16,fp8,511,0.029767999053001405
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,40,40,128,1,float16,float16,15,0.8495360374450683
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,40,40,128,1,float16,fp8,15,0.8229536056518555
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,40,40,128,1,float16,float16,31,0.8590096473693848
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,40,40,128,1,float16,fp8,31,0.8310671806335449
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,40,40,128,1,float16,float16,63,0.8794832229614258
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,40,40,128,1,float16,fp8,63,0.8578864097595215
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,40,40,128,1,float16,fp8,1,0.7868303775787353
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,40,40,128,1,float16,float16,1,1.6659263610839843
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,40,40,128,1,float16,fp8,1,1.615724754333496
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,40,40,128,1,float16,float16,3,1.6742399215698243
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,40,40,128,1,float16,float16,1,0.021612800657749176
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,40,40,128,1,float16,fp8,3,1.6135919570922852
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,40,40,128,1,float16,fp8,1,0.021116800606250763
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,40,40,128,1,float16,fp8,3,0.02115360051393509
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,40,40,128,1,float16,float16,7,1.6715984344482422
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,40,40,128,1,float16,fp8,7,1.6328655242919923
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,40,40,128,1,float16,float16,7,0.021663999557495116
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,40,40,128,1,float16,float16,15,1.6800416946411132
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,40,40,128,1,float16,fp8,15,0.021040000021457672
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,40,40,128,1,float16,fp8,7,0.0208079993724823
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,40,40,128,1,float16,fp8,15,1.6406879425048828
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,40,40,128,1,float16,fp8,31,0.021030400693416596
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,40,40,128,1,float16,float16,31,0.02152000069618225
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,40,40,128,1,float16,float16,127,0.0235167995095253
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,40,40,128,1,float16,fp8,63,0.02126239985227585
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,40,40,128,1,float16,fp8,127,0.025281599164009093
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,40,40,128,1,float16,float16,255,0.038115200400352475
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,40,40,128,1,float16,float16,31,1.6916351318359375
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,40,40,128,1,float16,fp8,511,0.04666079878807068
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,40,40,128,1,float16,fp8,31,1.6521936416625977
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,40,40,128,1,float16,float16,1023,0.09603360295295715
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,40,40,128,1,float16,fp8,2047,0.1176416039466858
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,40,40,128,1,float16,float16,2047,0.16473599672317504
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,32,128,1,float16,float16,1,0.05045440196990967
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,32,128,1,float16,float16,3,0.05023199915885925
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,32,128,1,float16,fp8,1,0.049516800045967105
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,32,128,1,float16,fp8,3,0.04954400062561035
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,32,128,1,float16,float16,7,0.05027840137481689
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,32,128,1,float16,fp8,7,0.04943839907646179
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,32,128,1,float16,float16,15,0.05042240023612976
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,32,128,1,float16,fp8,15,0.04964799880981445
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,32,128,1,float16,float16,31,0.05114560127258301
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,32,128,1,float16,fp8,31,0.04961119890213013
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,32,128,1,float16,float16,63,0.05679519772529602
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,32,128,1,float16,fp8,63,0.04971039891242981
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,32,128,1,float16,float16,127,0.06155040264129639
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,32,128,1,float16,float16,255,0.08898239731788635
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,32,128,1,float16,fp8,255,0.08156319856643676
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,40,40,128,1,float16,float16,3,0.021617600321769716
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,32,128,1,float16,float16,511,0.14629600048065186
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,40,40,128,1,float16,float16,15,0.021550400555133818
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,32,128,1,float16,fp8,511,0.12460960149765014
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,40,40,128,1,float16,float16,63,0.021671999990940095
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,40,40,128,1,float16,fp8,255,0.031035199761390686
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,40,40,128,1,float16,float16,511,0.055486398935317996
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,32,128,1,float16,fp8,1023,0.1955135941505432
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,32,128,1,float16,float16,1,0.010395199805498124
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,40,40,128,1,float16,fp8,1023,0.069377601146698
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,32,128,1,float16,fp8,1,0.00997759997844696
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,32,128,1,float16,float16,3,0.010444799810647965
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,32,128,1,float16,float16,2047,0.4911632061004639
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,32,128,1,float16,fp8,3,0.009964799880981446
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,32,128,1,float16,fp8,2047,0.3496927976608276
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,32,128,1,float16,float16,7,0.010407999902963639
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,32,128,1,float16,fp8,7,0.009854400157928466
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,32,128,1,float16,float16,31,0.01032480001449585
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,32,128,1,float16,float16,15,0.010380800068378448
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,32,128,1,float16,fp8,31,0.00998080000281334
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,32,128,1,float16,fp8,15,0.010046400129795074
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,32,128,1,float16,float16,63,0.010264000296592713
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,32,128,1,float16,float16,127,0.010744000226259232
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,32,128,1,float16,fp8,127,0.010814400017261505
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,32,128,1,float16,float16,255,0.012228800356388092
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,32,128,1,float16,fp8,255,0.011886399984359742
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,32,128,1,float16,fp8,4095,0.6400991916656494
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,32,128,1,float16,fp8,511,0.013227200508117676
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,32,128,1,float16,fp8,1023,0.014009599387645722
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,32,128,1,float16,float16,2047,0.016846400499343873
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,32,128,1,float16,float16,1023,0.015436799824237823
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,32,128,1,float16,fp8,2047,0.016475200653076172
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,32,128,1,float16,float16,4095,0.029624000191688538
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,32,128,1,float16,fp8,4095,0.020921599864959717
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,32,128,1,float16,fp8,1,0.009996800124645234
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,32,128,1,float16,float16,1,0.010856000334024429
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,32,128,1,float16,float16,3,0.010500799864530563
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,32,128,1,float16,fp8,3,0.010460799932479859
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,32,128,1,float16,fp8,7,0.010443200170993806
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,32,128,1,float16,float16,15,0.01080320030450821
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,32,128,1,float16,fp8,15,0.010427200049161912
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,32,128,1,float16,float16,31,0.01069760024547577
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,32,128,1,float16,fp8,31,0.010124800354242324
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,32,128,1,float16,float16,63,0.010635200142860412
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,32,128,1,float16,fp8,63,0.010169599950313569
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,32,128,1,float16,float16,127,0.010883200168609618
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,32,128,1,float16,fp8,127,0.01117440015077591
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,32,128,1,float16,float16,255,0.012409599870443344
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,32,128,1,float16,float16,1023,0.2634255886077881
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,32,128,1,float16,fp8,255,0.011875200271606445
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,32,128,1,float16,fp8,511,0.014323200285434722
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,32,128,1,float16,float16,511,0.014475199580192565
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,32,128,1,float16,float16,1023,0.01671839952468872
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,32,128,1,float16,fp8,1023,0.016553600132465363
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,32,128,1,float16,float16,2047,0.029187199473381043
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,32,128,1,float16,fp8,2047,0.02126079946756363
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,32,128,1,float16,fp8,4095,0.03222079873085022
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,32,128,1,float16,float16,4095,0.04491199851036072
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,32,128,1,float16,float16,1,0.029760000109672547
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,32,128,1,float16,float16,3,0.029790401458740234
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,32,128,1,float16,fp8,3,0.028751999139785767
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,32,128,1,float16,fp8,63,0.01005759984254837
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,32,128,1,float16,fp8,7,0.02897599935531616
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,32,128,1,float16,float16,511,0.014068800210952758
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,32,128,1,float16,float16,15,0.030027198791503906
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,32,128,1,float16,float16,4095,0.9682815551757813
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,32,128,1,float16,float16,31,0.03007200062274933
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,32,128,1,float16,fp8,15,0.0286655992269516
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,32,128,1,float16,float16,63,0.030590400099754333
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,32,128,1,float16,fp8,31,0.028860801458358766
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,32,128,1,float16,fp8,127,0.03674559891223907
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,32,128,1,float16,fp8,127,0.06656640172004699
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,32,128,1,float16,float16,7,0.010569600015878677
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,32,128,1,float16,fp8,255,0.04503679871559143
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,32,128,1,float16,fp8,511,0.06694239974021912
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,32,128,1,float16,float16,511,0.08117120265960694
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,32,128,1,float16,float16,1023,0.1392016053199768
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,32,128,1,float16,fp8,1023,0.10257279872894287
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,32,128,1,float16,fp8,2047,0.18005599975585937
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,32,128,1,float16,float16,2047,0.2543152093887329
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,32,128,1,float16,fp8,1,0.028502398729324342
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,32,128,1,float16,fp8,4095,0.3256063938140869
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,32,128,1,float16,float16,1,0.01075040027499199
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,32,128,1,float16,float16,4095,0.4867584228515625
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,32,128,1,float16,float16,3,0.010769599676132202
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,32,128,1,float16,float16,7,0.029710400104522704
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,32,128,1,float16,fp8,3,0.010227199643850327
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,32,128,1,float16,float16,7,0.01077599972486496
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,32,128,1,float16,fp8,7,0.010260800272226334
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,32,128,1,float16,fp8,15,0.010372799634933472
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,32,128,1,float16,float16,15,0.010665600001811982
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,32,128,1,float16,float16,31,0.01072160005569458
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,32,128,1,float16,fp8,31,0.010425599664449692
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,32,128,1,float16,float16,63,0.01085280030965805
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,32,128,1,float16,float16,127,0.011230400204658509
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,32,128,1,float16,fp8,127,0.011182399839162827
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,32,128,1,float16,float16,255,0.012595200538635254
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,32,128,1,float16,fp8,255,0.011897599697113037
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,32,128,1,float16,fp8,63,0.028697600960731505
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,32,128,1,float16,float16,511,0.014636799693107605
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,32,128,1,float16,fp8,511,0.015558399260044098
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,32,128,1,float16,float16,127,0.03706879913806915
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,32,128,1,float16,fp8,1023,0.019934399425983428
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,32,128,1,float16,float16,2047,0.043859198689460754
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,32,128,1,float16,fp8,2047,0.03148640096187592
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,32,128,1,float16,float16,255,0.05372160077095032
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,32,128,1,float16,float16,1,0.09114080071449279
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,32,128,1,float16,fp8,1,0.09037119746208191
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,32,128,1,float16,fp8,3,0.09080960154533387
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,32,128,1,float16,float16,7,0.09209759831428528
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,32,128,1,float16,fp8,7,0.09075520038604737
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,32,128,1,float16,float16,15,0.09269279837608338
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,32,128,1,float16,float16,31,0.09685440063476562
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,32,128,1,float16,fp8,15,0.08911679983139038
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,32,128,1,float16,fp8,31,0.08958560228347778
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,32,128,1,float16,float16,63,0.10149760246276855
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,32,128,1,float16,fp8,63,0.09765920042991638
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,32,128,1,float16,fp8,1,0.010225600004196167
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,32,128,1,float16,float16,127,0.10890239477157593
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,32,128,1,float16,fp8,127,0.12174079418182374
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,32,128,1,float16,fp8,63,0.010286399722099304
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,32,128,1,float16,float16,255,0.15886720418930053
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,32,128,1,float16,fp8,255,0.15083999633789064
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,32,128,1,float16,float16,511,0.2824079990386963
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,32,128,1,float16,fp8,511,0.23498079776763917
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,32,128,1,float16,float16,1023,0.028507199883461
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,32,128,1,float16,fp8,4095,0.0494704008102417
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,32,128,1,float16,float16,1023,0.5066351890563965
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,32,128,1,float16,float16,3,0.09111840128898621
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,32,128,1,float16,float16,1,0.1720080018043518
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,32,128,1,float16,fp8,1023,0.37788639068603513
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,32,128,1,float16,float16,3,0.17293280363082886
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,32,128,1,float16,fp8,1,0.16884000301361085
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,32,128,1,float16,float16,7,0.17148159742355346
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,32,128,1,float16,fp8,7,0.16991039514541625
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,32,128,1,float16,float16,15,0.1739743947982788
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,32,128,1,float16,fp8,2047,0.690831995010376
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,32,128,1,float16,float16,2047,0.9616640090942383
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,32,128,1,float16,fp8,31,0.17541760206222534
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,32,128,1,float16,float16,31,0.18537919521331786
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,32,128,1,float16,float16,127,0.20460159778594972
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,32,128,1,float16,float16,63,0.1917631983757019
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,32,128,1,float16,fp8,63,0.18749120235443115
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,32,128,1,float16,fp8,127,0.23410239219665527
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,32,128,1,float16,float16,255,0.2973551988601685
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,32,128,1,float16,fp8,255,0.29236640930175783
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,32,128,1,float16,float16,511,0.5498432159423828
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,32,128,1,float16,fp8,511,0.45908641815185547
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,32,128,1,float16,float16,1,0.3336479902267456
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,32,128,1,float16,fp8,1,0.32944159507751464
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,32,128,1,float16,float16,3,0.3337519884109497
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,32,128,1,float16,fp8,7,0.3296591997146606
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,32,128,1,float16,fp8,3,0.33019199371337893
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,32,128,1,float16,float16,4095,0.07374399900436401
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,32,128,1,float16,fp8,15,0.3313136100769043
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,32,128,1,float16,float16,31,0.3617408037185669
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,32,128,1,float16,fp8,31,0.3505295991897583
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,32,128,1,float16,float16,63,0.37183198928833006
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,32,128,1,float16,fp8,3,0.16853439807891846
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,32,128,1,float16,fp8,63,0.3587775945663452
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,32,128,1,float16,float16,127,0.39967360496521
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,32,128,1,float16,float16,1,0.013422399759292603
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,32,128,1,float16,fp8,127,0.45771040916442873
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,32,128,1,float16,float16,3,0.013368000090122224
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,32,128,1,float16,fp8,15,0.16853760480880736
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,32,128,1,float16,float16,7,0.01342879980802536
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,32,128,1,float16,fp8,3,0.012942400574684144
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,32,128,1,float16,fp8,15,0.012966400384902954
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,32,128,1,float16,float16,31,0.01350879967212677
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,32,128,1,float16,fp8,255,0.5668399810791016
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,32,128,1,float16,fp8,31,0.012761600315570831
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,32,128,1,float16,fp8,63,0.012857599556446076
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,32,128,1,float16,float16,63,0.013524800539016724
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,32,128,1,float16,float16,127,0.01393440067768097
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,32,128,1,float16,fp8,127,0.014924800395965577
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,32,128,1,float16,float16,255,0.016918399930000307
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,32,128,1,float16,fp8,255,0.016180799901485445
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,32,128,1,float16,float16,511,0.02874560058116913
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,32,128,1,float16,fp8,511,0.022438399493694305
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,32,128,1,float16,fp8,1023,0.03325439989566803
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,32,128,1,float16,float16,2047,0.07509920001029968
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,32,128,1,float16,fp8,2047,0.052297598123550414
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,32,128,1,float16,float16,4095,0.13394240140914918
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,32,128,1,float16,fp8,4095,0.08941439986228943
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,32,128,1,float16,float16,7,0.33713440895080565
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,32,128,1,float16,float16,15,0.35506880283355713
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,32,32,128,1,float16,float16,1,0.6756192207336426
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,32,32,128,1,float16,fp8,1,0.6410655975341797
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,32,32,128,1,float16,float16,3,0.6832208156585693
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,32,32,128,1,float16,fp8,3,0.6412432193756104
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,32,128,1,float16,fp8,1,0.01281599998474121
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,32,32,128,1,float16,float16,7,0.6849264144897461
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,32,128,1,float16,float16,255,0.5739935874938965
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,32,32,128,1,float16,fp8,7,0.6541584014892579
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,32,32,128,1,float16,float16,15,0.6968656063079834
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,32,32,128,1,float16,fp8,15,0.676251220703125
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,32,32,128,1,float16,float16,31,0.7052656173706054
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,32,32,128,1,float16,fp8,31,0.6870351791381836
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,32,128,1,float16,float16,1023,0.045763200521469115
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,32,32,128,1,float16,float16,63,0.7284912109375
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,32,32,128,1,float16,fp8,63,0.7009247779846192
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,32,32,128,1,float16,float16,127,0.7813136100769043
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,32,32,128,1,float16,fp8,127,0.891875171661377
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,32,32,128,1,float16,fp8,1,1.3245023727416991
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,32,32,128,1,float16,float16,3,1.348259162902832
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,32,32,128,1,float16,fp8,3,1.318289566040039
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,32,32,128,1,float16,float16,7,1.3642016410827638
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,32,32,128,1,float16,fp8,7,1.321939182281494
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,32,128,1,float16,float16,15,0.013575999438762665
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,32,32,128,1,float16,float16,15,1.3886079788208008
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,32,32,128,1,float16,fp8,15,1.3328304290771484
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,32,128,1,float16,float16,1,0.018960000574588777
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,32,128,1,float16,float16,3,0.01868479996919632
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,32,128,1,float16,fp8,1,0.017739200592041017
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,32,128,1,float16,fp8,3,0.017844800651073457
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,32,32,128,1,float16,float16,31,1.3944928169250488
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,32,128,1,float16,fp8,7,0.01767359972000122
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,32,32,128,1,float16,fp8,31,1.3587295532226562
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,32,128,1,float16,float16,15,0.018727999925613404
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,32,128,1,float16,fp8,15,0.01804800033569336
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,32,128,1,float16,float16,31,0.018747200071811677
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,32,128,1,float16,fp8,31,0.017859199643135072
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,32,128,1,float16,float16,63,0.018910400569438934
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,32,32,128,1,float16,float16,63,1.4385087966918946
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,32,128,1,float16,fp8,63,0.017873600125312805
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,32,128,1,float16,fp8,127,0.021491199731826782
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,32,128,1,float16,fp8,255,0.02537280023097992
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,32,128,1,float16,float16,255,0.03391680121421814
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,32,128,1,float16,float16,511,0.046910399198532106
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,32,128,1,float16,fp8,511,0.03850559890270233
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,32,128,1,float16,float16,1023,0.07732959985733032
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,32,32,128,1,float16,float16,1,1.3555935859680175
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,32,128,1,float16,float16,2047,0.1357264041900635
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,32,128,1,float16,fp8,2047,0.0950767993927002
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,32,128,1,float16,float16,4095,0.2535952091217041
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,32,128,1,float16,fp8,4095,0.16945279836654664
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,24,24,128,1,float16,float16,1,0.04084640145301819
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,24,24,128,1,float16,float16,3,0.04086720049381256
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,32,128,1,float16,fp8,7,0.012787200510501862
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,24,24,128,1,float16,fp8,1,0.040689599514007566
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,24,24,128,1,float16,float16,7,0.0407696008682251
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,24,24,128,1,float16,float16,15,0.041238400340080264
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,24,24,128,1,float16,fp8,15,0.04088639914989471
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,32,128,1,float16,float16,7,0.01865919977426529
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,24,24,128,1,float16,float16,31,0.04094400107860565
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,24,24,128,1,float16,float16,63,0.04492320120334625
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,24,24,128,1,float16,fp8,63,0.04097760021686554
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,24,24,128,1,float16,float16,127,0.05019199848175049
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,24,24,128,1,float16,fp8,127,0.05324800014495849
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,24,24,128,1,float16,float16,255,0.07211199998855591
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,24,24,128,1,float16,fp8,255,0.0656607985496521
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,24,24,128,1,float16,float16,511,0.11373440027236939
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,32,128,1,float16,fp8,1023,0.056123197078704834
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,24,24,128,1,float16,fp8,511,0.09848960041999817
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,24,24,128,1,float16,float16,1023,0.20209119319915772
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,24,24,128,1,float16,fp8,1023,0.15076639652252197
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,24,24,128,1,float16,float16,2047,0.3746655941009521
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,24,24,128,1,float16,fp8,2047,0.26777920722961424
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,24,24,128,1,float16,fp8,3,0.04092639982700348
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,24,24,128,1,float16,fp8,7,0.04079839885234833
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,24,24,128,1,float16,float16,1,0.010131199657917023
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,24,24,128,1,float16,float16,3,0.010115200281143188
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,24,24,128,1,float16,fp8,3,0.009982399642467499
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,24,24,128,1,float16,fp8,4095,0.48717918395996096
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,24,24,128,1,float16,float16,7,0.010127999633550645
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,24,24,128,1,float16,float16,15,0.010116799920797347
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,24,24,128,1,float16,float16,4095,0.7297647953033447
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,24,24,128,1,float16,fp8,7,0.010187199711799622
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,24,24,128,1,float16,fp8,15,0.009976000338792802
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,24,24,128,1,float16,fp8,31,0.04090720117092132
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,24,24,128,1,float16,float16,63,0.010380800068378448
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,24,24,128,1,float16,float16,31,0.01040479987859726
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,24,24,128,1,float16,fp8,63,0.010132800042629241
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,24,24,128,1,float16,float16,127,0.010491199791431427
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,24,24,128,1,float16,float16,255,0.012145599722862244
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,32,128,1,float16,float16,127,0.01945440024137497
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,24,24,128,1,float16,float16,511,0.01379680037498474
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,24,24,128,1,float16,fp8,127,0.011023999750614166
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,24,24,128,1,float16,fp8,511,0.013566400110721587
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,24,24,128,1,float16,float16,1023,0.014299200475215912
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,24,24,128,1,float16,fp8,1023,0.013915200531482697
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,24,24,128,1,float16,float16,2047,0.016451199352741242
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,24,24,128,1,float16,float16,4095,0.02295680046081543
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,24,24,128,1,float16,fp8,4095,0.01942239999771118
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,24,24,128,1,float16,float16,1,0.010371199995279311
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,24,24,128,1,float16,fp8,1,0.010182400047779084
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,24,24,128,1,float16,float16,3,0.010384000092744827
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,24,24,128,1,float16,float16,7,0.010380800068378448
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,24,24,128,1,float16,fp8,7,0.010424000024795533
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,24,24,128,1,float16,float16,15,0.010449600219726563
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,24,24,128,1,float16,float16,31,0.010470400005578995
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,24,24,128,1,float16,fp8,31,0.010425599664449692
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,24,24,128,1,float16,fp8,15,0.010441599786281586
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,24,24,128,1,float16,float16,63,0.010419200360774993
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,24,24,128,1,float16,fp8,63,0.010228800028562546
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,24,24,128,1,float16,float16,127,0.010572800040245056
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,24,24,128,1,float16,fp8,127,0.01111999973654747
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,24,24,128,1,float16,float16,255,0.011988800019025803
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,24,24,128,1,float16,fp8,255,0.012110400199890136
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,24,24,128,1,float16,float16,511,0.014097599685192109
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,24,24,128,1,float16,fp8,511,0.01401280015707016
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,24,24,128,1,float16,float16,1023,0.01652639955282211
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,24,24,128,1,float16,fp8,1023,0.01923519968986511
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,24,24,128,1,float16,float16,2047,0.027556800842285158
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,24,24,128,1,float16,fp8,2047,0.025172799825668335
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,24,24,128,1,float16,float16,4095,0.046000000834465024
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,24,24,128,1,float16,float16,1,0.02425439953804016
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,24,24,128,1,float16,fp8,1,0.023856000602245332
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,24,24,128,1,float16,float16,3,0.024260799586772918
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,24,24,128,1,float16,fp8,3,0.024003200232982635
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,24,24,128,1,float16,float16,7,0.02450399994850159
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,24,24,128,1,float16,fp8,7,0.024111999571323393
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,24,24,128,1,float16,float16,15,0.02455199956893921
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,24,24,128,1,float16,fp8,31,0.010091199725866317
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,24,24,128,1,float16,fp8,15,0.02401279956102371
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,24,24,128,1,float16,float16,31,0.02468319982290268
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,24,24,128,1,float16,fp8,255,0.011870399862527848
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,24,24,128,1,float16,fp8,31,0.02414399981498718
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,32,32,128,1,float16,fp8,63,1.383182430267334
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,24,24,128,1,float16,float16,63,0.02452480047941208
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,24,24,128,1,float16,fp8,127,0.029425600171089174
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,24,24,128,1,float16,float16,127,0.03075839877128601
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,24,24,128,1,float16,fp8,2047,0.01573919951915741
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,24,24,128,1,float16,float16,255,0.04318079948425293
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,24,24,128,1,float16,fp8,3,0.010134399682283402
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,24,24,128,1,float16,float16,511,0.06402080059051514
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,24,24,128,1,float16,fp8,511,0.053865599632263186
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,24,24,128,1,float16,fp8,255,0.03675679862499237
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,24,24,128,1,float16,float16,1023,0.10863840579986572
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,24,24,128,1,float16,fp8,1023,0.08059679865837097
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,24,24,128,1,float16,float16,2047,0.195523202419281
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,24,24,128,1,float16,fp8,2047,0.13888479471206666
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,24,24,128,1,float16,float16,1,0.010555200278759003
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,24,24,128,1,float16,float16,3,0.010548800230026245
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,24,24,128,1,float16,fp8,4095,0.2492687940597534
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,24,24,128,1,float16,float16,4095,0.3683840036392212
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,24,24,128,1,float16,float16,7,0.01032319962978363
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,24,24,128,1,float16,fp8,7,0.010475199669599533
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,24,24,128,1,float16,fp8,15,0.010361599922180175
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,24,24,128,1,float16,fp8,4095,0.0391728013753891
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,24,24,128,1,float16,float16,31,0.010300800204277039
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,24,24,128,1,float16,float16,63,0.01043199971318245
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,24,24,128,1,float16,fp8,31,0.010731200128793717
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,24,24,128,1,float16,fp8,1,0.010009600222110749
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,24,24,128,1,float16,fp8,63,0.010623999685049058
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,24,24,128,1,float16,float16,127,0.010784000158309937
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,24,24,128,1,float16,fp8,127,0.011641599982976914
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,24,24,128,1,float16,float16,255,0.012223999947309494
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,24,24,128,1,float16,float16,511,0.014419199526309967
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,24,24,128,1,float16,fp8,511,0.018423999845981597
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,24,24,128,1,float16,float16,1023,0.026332798600196838
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,24,24,128,1,float16,float16,2047,0.03969280123710632
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,24,24,128,1,float16,fp8,2047,0.03364480137825012
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,24,24,128,1,float16,float16,4095,0.06312320232391358
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,24,24,128,1,float16,fp8,4095,0.05260480046272278
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,24,24,128,1,float16,float16,1,0.07264000177383423
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,24,24,128,1,float16,fp8,63,0.02414879947900772
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,24,24,128,1,float16,fp8,1,0.07245759963989258
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,24,24,128,1,float16,float16,3,0.07227200269699097
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,24,24,128,1,float16,fp8,3,0.07221919894218445
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,24,24,128,1,float16,float16,7,0.07240639925003052
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,24,24,128,1,float16,fp8,7,0.07217919826507568
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,24,24,128,1,float16,float16,15,0.07240639925003052
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,24,24,128,1,float16,fp8,15,0.07222239971160889
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,24,24,128,1,float16,float16,31,0.07382559776306152
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,24,24,128,1,float16,fp8,31,0.07221119999885559
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,24,24,128,1,float16,float16,63,0.08075519800186157
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,24,24,128,1,float16,fp8,63,0.0787872016429901
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,24,24,128,1,float16,float16,127,0.08843039870262145
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,24,24,128,1,float16,fp8,127,0.09757120013237
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,24,24,128,1,float16,float16,255,0.1276975989341736
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,24,24,128,1,float16,float16,15,0.010292799770832061
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,24,24,128,1,float16,fp8,255,0.12034399509429931
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,24,24,128,1,float16,float16,511,0.21327519416809082
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,24,24,128,1,float16,fp8,255,0.012379200011491776
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,24,24,128,1,float16,float16,1023,0.3857408046722412
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,24,24,128,1,float16,fp8,1023,0.022819200158119203
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,24,24,128,1,float16,float16,1,0.13610080480575562
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,24,24,128,1,float16,fp8,1,0.13443360328674317
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,24,24,128,1,float16,fp8,2047,0.5322559833526611
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,24,24,128,1,float16,float16,2047,0.7262271881103516
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,24,24,128,1,float16,fp8,3,0.13445600271224975
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,24,24,128,1,float16,float16,7,0.1363952040672302
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,24,24,128,1,float16,fp8,7,0.13432320356369018
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,24,24,128,1,float16,float16,15,0.13703360557556152
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,24,24,128,1,float16,fp8,15,0.13453919887542726
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,24,24,128,1,float16,float16,31,0.14481120109558104
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,24,24,128,1,float16,fp8,1,0.010545600205659866
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,24,24,128,1,float16,fp8,31,0.1373296022415161
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,24,24,128,1,float16,float16,63,0.1513535976409912
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,24,24,128,1,float16,fp8,3,0.010555200278759003
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,24,24,128,1,float16,fp8,63,0.14625279903411864
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,24,24,128,1,float16,float16,127,0.16276320219039916
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,24,24,128,1,float16,fp8,127,0.18472640514373778
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,24,24,128,1,float16,float16,255,0.23600320816040038
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,24,24,128,1,float16,fp8,255,0.22758080959320068
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,24,24,128,1,float16,fp8,511,0.18567039966583251
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,24,24,128,1,float16,float16,511,0.4134575843811035
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,24,24,128,1,float16,float16,1,0.26087679862976076
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,24,24,128,1,float16,fp8,1,0.25770881175994875
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,24,24,128,1,float16,float16,3,0.2614239931106567
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,24,24,128,1,float16,float16,7,0.2616175889968872
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,24,24,128,1,float16,fp8,7,0.25692000389099123
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,24,24,128,1,float16,fp8,1023,0.2909503936767578
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,24,24,128,1,float16,float16,15,0.2703200101852417
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,24,24,128,1,float16,fp8,15,0.25913760662078855
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,24,24,128,1,float16,float16,31,0.27809920310974123
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,24,24,128,1,float16,fp8,31,0.27415521144866944
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,24,24,128,1,float16,float16,63,0.29161601066589354
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,24,24,128,1,float16,fp8,63,0.28216960430145266
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,24,24,128,1,float16,float16,3,0.13579039573669432
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,24,24,128,1,float16,float16,127,0.3125040054321289
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,24,24,128,1,float16,float16,1,0.012812800705432892
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,24,24,128,1,float16,fp8,1,0.012745599448680877
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,24,24,128,1,float16,float16,3,0.012995199859142303
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,24,24,128,1,float16,fp8,3,0.012732799351215362
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,24,24,128,1,float16,fp8,255,0.4427807807922363
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,24,24,128,1,float16,float16,7,0.013020800054073333
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,24,24,128,1,float16,float16,15,0.013115200400352477
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,24,24,128,1,float16,fp8,7,0.012798400223255157
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,24,24,128,1,float16,fp8,15,0.012883199751377106
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,24,24,128,1,float16,float16,31,0.012964800000190735
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,24,24,128,1,float16,fp8,31,0.012814399600028992
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,24,24,128,1,float16,float16,63,0.013126400113105775
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,24,24,128,1,float16,fp8,63,0.012899200618267059
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,24,24,128,1,float16,float16,127,0.013368000090122224
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,24,24,128,1,float16,fp8,127,0.014764800667762756
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,24,24,128,1,float16,float16,255,0.01602399945259094
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,24,24,128,1,float16,fp8,255,0.01602399945259094
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,24,24,128,1,float16,float16,511,0.02324160039424896
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,24,24,128,1,float16,fp8,511,0.02237280011177063
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,24,24,128,1,float16,float16,1023,0.038489601016044615
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,24,24,128,1,float16,fp8,1023,0.03266560137271881
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,24,24,128,1,float16,float16,2047,0.06251680254936218
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,24,24,128,1,float16,fp8,2047,0.052830398082733154
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,24,24,128,1,float16,float16,4095,0.11071360111236572
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,24,24,128,1,float16,fp8,4095,0.09059680104255677
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,24,24,128,1,float16,fp8,511,0.35435359477996825
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,24,24,128,1,float16,float16,1,0.5098559856414795
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,24,24,128,1,float16,fp8,1,0.5005343914031982
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,24,24,128,1,float16,float16,3,0.5126031875610352
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,24,24,128,1,float16,fp8,3,0.2587791919708252
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,24,24,128,1,float16,fp8,3,0.5013904094696044
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,24,24,128,1,float16,float16,7,0.5273007869720459
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,24,24,128,1,float16,fp8,7,0.5023920059204101
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,24,24,128,1,float16,float16,15,0.5316112041473389
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,24,24,128,1,float16,fp8,15,0.5249872207641602
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,24,24,128,1,float16,float16,31,0.5427711963653564
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,24,24,128,1,float16,fp8,31,0.5392720222473144
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,24,24,128,1,float16,float16,63,0.5670415878295898
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,24,24,128,1,float16,fp8,127,0.3544352054595947
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,24,24,128,1,float16,fp8,63,0.5487887859344482
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,24,24,128,1,float16,float16,255,0.44635682106018065
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,24,24,128,1,float16,float16,127,0.6087679862976074
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,24,24,128,1,float16,fp8,127,0.6938528060913086
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,24,24,128,1,float16,float16,1,1.0388447761535644
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,24,24,128,1,float16,fp8,1,1.0041664123535157
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,24,24,128,1,float16,fp8,3,1.0097056388854981
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,24,24,128,1,float16,float16,7,1.0345343589782714
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,24,24,128,1,float16,fp8,7,1.0286687850952148
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,24,24,128,1,float16,float16,15,1.0460191726684571
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,24,24,128,1,float16,fp8,15,1.04345760345459
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,24,24,128,1,float16,float16,1,0.015800000727176668
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,24,24,128,1,float16,fp8,1,0.015868799388408662
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,24,24,128,1,float16,float16,3,0.015987199544906617
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,24,24,128,1,float16,float16,31,1.069179153442383
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,24,24,128,1,float16,fp8,3,0.01563359946012497
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,24,24,128,1,float16,float16,7,0.015910400450229643
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,24,24,128,1,float16,fp8,31,1.0535728454589843
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,24,24,128,1,float16,float16,15,0.015881599485874177
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,24,24,128,1,float16,fp8,15,0.01564960032701492
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,24,24,128,1,float16,float16,31,0.015907199680805208
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,24,24,128,1,float16,float16,63,1.1179743766784669
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,24,24,128,1,float16,fp8,31,0.015563200414180755
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,24,24,128,1,float16,float16,63,0.016038399934768677
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,24,24,128,1,float16,fp8,63,0.015740799903869628
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,24,24,128,1,float16,fp8,127,0.018345600366592406
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,24,24,128,1,float16,fp8,63,1.0846544265747071
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,24,24,128,1,float16,float16,127,0.016451199352741242
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,24,24,128,1,float16,float16,511,0.03729760050773621
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,24,24,128,1,float16,fp8,511,0.031998398900032046
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,24,24,128,1,float16,float16,1023,0.060731202363967896
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,24,24,128,1,float16,fp8,1023,0.045256000757217404
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,24,24,128,1,float16,float16,2047,0.10481120347976684
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,24,24,128,1,float16,fp8,2047,0.07496320009231568
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,24,24,128,1,float16,float16,4095,0.19138239622116088
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,16,128,1,float16,float16,1,0.03131999969482422
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,24,24,128,1,float16,fp8,4095,0.13031840324401855
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,16,128,1,float16,fp8,1,0.030899199843406677
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,16,128,1,float16,float16,3,0.03129119873046875
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,16,128,1,float16,float16,7,0.03174720108509064
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,16,128,1,float16,fp8,7,0.030769601464271545
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,16,128,1,float16,fp8,3,0.030649599432945252
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,16,128,1,float16,float16,31,0.03128640055656433
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,16,128,1,float16,fp8,31,0.030851200222969055
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,16,128,1,float16,fp8,15,0.030828800797462464
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,16,128,1,float16,float16,63,0.03194240033626557
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,16,128,1,float16,fp8,63,0.030774399638175964
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,16,128,1,float16,float16,127,0.03842880129814148
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,16,128,1,float16,fp8,127,0.03776000142097473
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,16,128,1,float16,float16,255,0.05457119941711426
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,16,128,1,float16,fp8,255,0.04722239971160889
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,16,128,1,float16,float16,511,0.0824400007724762
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,16,128,1,float16,fp8,511,0.07002080082893372
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,16,128,1,float16,float16,1023,0.14237439632415771
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,24,24,128,1,float16,float16,3,1.0292207717895507
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,16,128,1,float16,float16,2047,0.25767040252685547
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,16,128,1,float16,fp8,2047,0.18321919441223145
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,16,128,1,float16,float16,4095,0.48803200721740725
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,16,128,1,float16,fp8,4095,0.3323391914367676
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,16,128,1,float16,float16,1,0.009782399982213974
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,16,128,1,float16,fp8,1,0.009393599629402161
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,16,128,1,float16,float16,3,0.009696000069379807
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,24,24,128,1,float16,fp8,7,0.015652799606323244
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,16,128,1,float16,float16,8191,0.9527647972106934
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,24,24,128,1,float16,fp8,255,0.020979200303554536
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,16,128,1,float16,fp8,8191,0.625163221359253
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,16,128,1,float16,float16,7,0.010145600140094756
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,16,128,1,float16,fp8,7,0.009617599844932555
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,16,128,1,float16,fp8,15,0.009459199756383896
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,16,128,1,float16,float16,31,0.010046400129795074
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,16,128,1,float16,float16,63,0.010001599788665771
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,16,128,1,float16,fp8,63,0.009675200283527374
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,16,128,1,float16,float16,127,0.010153599828481675
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,16,128,1,float16,float16,15,0.03119199872016907
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,16,128,1,float16,float16,255,0.011099199950695037
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,16,128,1,float16,fp8,255,0.01111999973654747
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,16,128,1,float16,float16,511,0.01332319974899292
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,16,128,1,float16,fp8,511,0.012265600264072418
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,16,128,1,float16,float16,1023,0.013988800346851349
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,16,128,1,float16,fp8,1023,0.10498239994049072
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,16,128,1,float16,fp8,1023,0.01353919953107834
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,16,128,1,float16,fp8,2047,0.014852799475193024
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,16,128,1,float16,float16,4095,0.017609600722789765
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,16,128,1,float16,fp8,4095,0.01682240068912506
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,16,128,1,float16,float16,8191,0.028547200560569762
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,16,128,1,float16,fp8,8191,0.02104319930076599
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,16,128,1,float16,float16,1,0.009830400347709656
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,16,128,1,float16,fp8,1,0.009620799869298934
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,16,128,1,float16,float16,3,0.009849599748849868
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,16,128,1,float16,fp8,3,0.00979039967060089
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,16,128,1,float16,float16,7,0.010022400319576264
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,16,128,1,float16,fp8,7,0.009824000298976898
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,16,128,1,float16,float16,15,0.01003040000796318
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,16,128,1,float16,fp8,15,0.00981760025024414
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,16,128,1,float16,fp8,3,0.009876800328493118
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,16,128,1,float16,float16,31,0.009947200119495393
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,16,128,1,float16,fp8,31,0.009617599844932555
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,16,128,1,float16,float16,63,0.010198400169610978
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,16,128,1,float16,fp8,63,0.009612800180912017
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,16,128,1,float16,float16,127,0.010232000052928925
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,16,128,1,float16,fp8,127,0.01064639985561371
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,16,128,1,float16,float16,15,0.009932799637317658
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,16,128,1,float16,fp8,255,0.011523199826478958
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,16,128,1,float16,fp8,31,0.00947519987821579
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,16,128,1,float16,float16,511,0.013851200044155122
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,16,128,1,float16,fp8,511,0.013502399623394012
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,16,128,1,float16,float16,1023,0.015086400508880615
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,16,128,1,float16,fp8,1023,0.01404000073671341
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,16,128,1,float16,fp8,127,0.01019200012087822
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,16,128,1,float16,fp8,2047,0.016649599373340606
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,16,128,1,float16,float16,4095,0.028040000796318056
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,16,128,1,float16,float16,8191,0.043966400623321536
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,16,128,1,float16,fp8,4095,0.020908799767494202
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,16,128,1,float16,float16,1,0.019123199582099914
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,16,128,1,float16,fp8,1,0.018904000520706177
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,16,128,1,float16,fp8,3,0.018855999410152435
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,16,128,1,float16,float16,7,0.019262400269508363
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,16,128,1,float16,fp8,7,0.018774400651454925
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,16,128,1,float16,float16,15,0.019383999705314636
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,16,128,1,float16,fp8,15,0.018863999843597413
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,16,128,1,float16,float16,31,0.019483199715614317
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,16,128,1,float16,fp8,31,0.01882079988718033
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,16,128,1,float16,float16,2047,0.014996799826622009
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,16,128,1,float16,fp8,63,0.018934400379657747
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,16,128,1,float16,float16,127,0.019534400105476378
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,16,128,1,float16,fp8,127,0.022308799624443054
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,16,128,1,float16,float16,255,0.03253760039806366
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,16,128,1,float16,fp8,255,0.02592639923095703
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,16,128,1,float16,float16,511,0.04708319902420044
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,16,128,1,float16,fp8,511,0.03922559916973114
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,16,128,1,float16,float16,1023,0.07800800204277039
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,16,128,1,float16,fp8,1023,0.056625598669052125
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,16,128,1,float16,float16,2047,0.13679519891738892
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,16,128,1,float16,float16,4095,0.252239990234375
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,24,24,128,1,float16,float16,255,0.024297599494457246
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,16,128,1,float16,fp8,4095,0.16976959705352784
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,16,128,1,float16,float16,255,0.011828800290822982
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,16,128,1,float16,float16,1,0.010019200295209885
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,16,128,1,float16,fp8,1,0.00968799963593483
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,16,128,1,float16,float16,8191,0.4850287914276123
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,16,128,1,float16,float16,3,0.009988799691200256
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,16,128,1,float16,float16,2047,0.01650879979133606
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,16,128,1,float16,fp8,8191,0.32039680480957033
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,16,128,1,float16,fp8,7,0.009988799691200256
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,16,128,1,float16,fp8,3,0.01003360003232956
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,16,128,1,float16,fp8,15,0.009881599992513656
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,16,128,1,float16,fp8,8191,0.03221920132637024
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,16,128,1,float16,float16,15,0.010175999999046326
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,16,128,1,float16,float16,3,0.019113600254058838
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,16,128,1,float16,float16,63,0.010283199697732925
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,16,128,1,float16,fp8,63,0.00984319970011711
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,16,128,1,float16,float16,127,0.010486400127410889
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,16,128,1,float16,float16,255,0.01170239970088005
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,16,128,1,float16,fp8,255,0.01157120019197464
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,16,128,1,float16,float16,511,0.01393119990825653
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,16,128,1,float16,float16,63,0.019449600577354433
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,16,128,1,float16,float16,1023,0.016463999450206757
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,16,128,1,float16,fp8,1023,0.016542400419712066
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,16,128,1,float16,float16,2047,0.028465598821640015
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,16,128,1,float16,fp8,2047,0.02099519968032837
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,16,128,1,float16,float16,4095,0.04329920113086701
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,16,128,1,float16,fp8,4095,0.031711998581886294
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,16,128,1,float16,float16,8191,0.07390239834785461
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,16,128,1,float16,fp8,8191,0.05044159889221191
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,16,128,1,float16,fp8,2047,0.09602239727973938
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,16,128,1,float16,float16,1,0.054527997970581055
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,16,128,1,float16,fp8,1,0.054574400186538696
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,16,128,1,float16,fp8,3,0.05456159710884094
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,16,128,1,float16,float16,7,0.054771202802658084
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,16,128,1,float16,fp8,7,0.05450080037117004
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,16,128,1,float16,float16,15,0.05459200143814087
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,16,128,1,float16,fp8,15,0.05426880121231079
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,16,128,1,float16,float16,31,0.054980802536010745
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,16,128,1,float16,fp8,31,0.05464959740638733
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,16,128,1,float16,float16,7,0.010000000149011612
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,16,128,1,float16,float16,63,0.0597760021686554
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,16,128,1,float16,fp8,63,0.05464800000190735
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,16,128,1,float16,float16,127,0.06518560051918029
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,16,128,1,float16,fp8,127,0.07062079906463622
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,16,128,1,float16,float16,255,0.0941424012184143
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,16,128,1,float16,fp8,255,0.08560960292816162
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,16,128,1,float16,fp8,127,0.01069760024547577
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,16,128,1,float16,float16,511,0.15349600315093995
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,16,128,1,float16,fp8,511,0.013884800672531127
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,16,128,1,float16,float16,1023,0.2690335988998413
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,16,128,1,float16,fp8,1023,0.20004000663757324
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,16,128,1,float16,float16,2047,0.49744482040405275
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,16,128,1,float16,fp8,2047,0.3580751895904541
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,16,128,1,float16,float16,1,0.09962239861488342
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,16,128,1,float16,float16,3,0.0546783983707428
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,16,128,1,float16,fp8,1,0.0986624002456665
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,16,128,1,float16,float16,3,0.09957119822502136
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,16,128,1,float16,float16,7,0.09935680031776428
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,16,128,1,float16,fp8,4095,0.6470416069030762
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,16,128,1,float16,fp8,7,0.09839839935302734
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,16,128,1,float16,fp8,15,0.0988207995891571
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,16,128,1,float16,float16,15,0.09918879866600036
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,16,128,1,float16,float16,31,0.10183680057525635
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,16,128,1,float16,fp8,31,0.09836639761924744
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,16,128,1,float16,float16,63,0.10915039777755738
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,16,128,1,float16,fp8,63,0.10707679986953736
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,16,128,1,float16,fp8,127,0.13062560558319092
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,16,128,1,float16,float16,31,0.010214400291442872
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,16,128,1,float16,float16,255,0.17437280416488649
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,16,128,1,float16,fp8,255,0.16030399799346923
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,16,128,1,float16,fp8,511,0.12944480180740356
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,16,128,1,float16,float16,511,0.2879487991333008
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,16,128,1,float16,fp8,511,0.24775679111480714
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,16,128,1,float16,float16,1,0.1902351975440979
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,16,128,1,float16,fp8,1,0.1870800018310547
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,16,128,1,float16,fp8,1023,0.3871776103973389
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,16,128,1,float16,float16,3,0.1900272011756897
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,16,128,1,float16,fp8,3,0.18709280490875244
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,16,128,1,float16,float16,7,0.18840479850769043
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,16,128,1,float16,float16,15,0.1892143964767456
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,16,128,1,float16,fp8,3,0.0986624002456665
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,16,128,1,float16,fp8,15,0.18740320205688477
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,16,128,1,float16,float16,31,0.1987936019897461
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,16,128,1,float16,fp8,31,0.1970144033432007
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,16,128,1,float16,float16,63,0.2082911968231201
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,16,128,1,float16,fp8,63,0.20596001148223878
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,16,128,1,float16,float16,127,0.22583680152893065
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,16,128,1,float16,fp8,127,0.25236001014709475
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,16,128,1,float16,float16,255,0.3302191972732544
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,16,128,1,float16,float16,127,0.11775200366973877
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,16,128,1,float16,fp8,255,0.3130352020263672
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,16,128,1,float16,fp8,31,0.010100799798965453
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,16,128,1,float16,float16,3,0.010351999849081039
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,16,128,1,float16,float16,511,0.5532559871673584
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,16,128,1,float16,fp8,3,0.010100799798965453
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,16,128,1,float16,fp8,7,0.010043200105428696
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,16,128,1,float16,float16,7,0.010689599812030793
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,16,128,1,float16,fp8,511,0.47819838523864744
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,16,128,1,float16,float16,15,0.010377600044012069
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,16,128,1,float16,fp8,15,0.01035040020942688
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,16,128,1,float16,fp8,31,0.010500799864530563
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,16,128,1,float16,float16,31,0.010526400059461594
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,16,128,1,float16,fp8,63,0.010177599638700486
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,16,128,1,float16,float16,127,0.011030399799346923
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,16,128,1,float16,fp8,127,0.011348800361156463
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,16,128,1,float16,float16,255,0.012336000055074691
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,16,128,1,float16,float16,511,0.014399999380111694
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,16,128,1,float16,float16,1023,0.5167759895324707
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,16,128,1,float16,fp8,511,0.015377600491046906
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,16,128,1,float16,fp8,1023,0.019790400564670563
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,16,128,1,float16,float16,1023,0.029478400945663452
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,16,128,1,float16,float16,2047,0.04291200041770935
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,16,128,1,float16,fp8,2047,0.030934399366378783
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,16,128,1,float16,float16,4095,0.07260000109672546
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,16,128,1,float16,fp8,7,0.1882207989692688
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,16,128,1,float16,fp8,4095,0.049423998594284056
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,16,128,1,float16,float16,4095,0.960201644897461
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,16,128,1,float16,float16,8191,0.13144320249557495
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,16,128,1,float16,fp8,8191,0.08767840266227722
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,16,128,1,float16,float16,1,0.36279199123382566
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,16,128,1,float16,fp8,3,0.3618848085403442
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,16,128,1,float16,fp8,1,0.362774395942688
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,16,128,1,float16,float16,7,0.3622544050216675
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,16,128,1,float16,fp8,7,0.3617072105407715
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,16,128,1,float16,float16,15,0.37374238967895507
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,16,128,1,float16,fp8,15,0.36290080547332765
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,16,128,1,float16,float16,1,0.010440000146627427
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,16,128,1,float16,float16,31,0.3849600076675415
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,16,128,1,float16,fp8,31,0.3886847972869873
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,16,128,1,float16,float16,63,0.4008800029754639
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,16,128,1,float16,float16,63,0.010804799944162368
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,16,128,1,float16,fp8,255,0.011828800290822982
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,16,128,1,float16,fp8,127,0.4891808032989502
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,16,128,1,float16,float16,255,0.6388591766357422
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,16,128,1,float16,fp8,255,0.6037087917327881
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,16,16,128,1,float16,float16,1,0.7078815937042237
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,16,16,128,1,float16,fp8,1,0.7023280143737793
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,16,128,1,float16,float16,3,0.36333599090576174
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,16,16,128,1,float16,fp8,3,0.7049200057983398
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,16,16,128,1,float16,float16,7,0.7133535861968994
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,16,128,1,float16,fp8,1,0.010080000013113022
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,16,16,128,1,float16,float16,15,0.7267055988311768
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,16,16,128,1,float16,fp8,15,0.7389935970306396
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,16,128,1,float16,fp8,63,0.40075039863586426
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,16,16,128,1,float16,float16,31,0.7551695823669433
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,16,128,1,float16,float16,127,0.43512959480285646
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,16,16,128,1,float16,fp8,31,0.7580607891082763
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,16,128,1,float16,float16,1,0.012932799756526947
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,16,128,1,float16,fp8,1,0.012681600451469422
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,16,128,1,float16,float16,3,0.012958399951457977
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,16,128,1,float16,fp8,3,0.012644800543785095
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,16,16,128,1,float16,float16,63,0.7890031814575196
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,16,128,1,float16,float16,7,0.012955200672149659
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,16,16,128,1,float16,float16,3,0.7097343921661377
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,16,128,1,float16,float16,15,0.012999999523162841
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,16,128,1,float16,fp8,7,0.012720000743865967
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,16,16,128,1,float16,fp8,63,0.7844064235687256
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,16,128,1,float16,fp8,15,0.01276639997959137
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,16,16,128,1,float16,float16,127,0.8547663688659668
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,16,128,1,float16,fp8,31,0.013016000390052795
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,16,16,128,1,float16,fp8,127,0.9659487724304199
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,16,128,1,float16,fp8,63,0.012756800651550293
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,16,128,1,float16,float16,127,0.013806399703025819
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,16,128,1,float16,float16,63,0.013123199343681335
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,16,128,1,float16,fp8,511,0.02255679965019226
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,16,128,1,float16,fp8,127,0.014979200065135955
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,16,16,128,1,float16,fp8,7,0.7099631786346435
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,16,128,1,float16,fp8,255,0.01621920019388199
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,16,128,1,float16,float16,511,0.02812480032444
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,16,128,1,float16,float16,1023,0.044407999515533446
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,16,128,1,float16,fp8,1023,0.03315039873123169
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,16,128,1,float16,float16,2047,0.07480480074882508
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,16,128,1,float16,float16,4095,0.13322399854660033
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,12,12,128,1,float16,float16,1,0.025899198651313782
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,16,128,1,float16,fp8,4095,0.09007200002670288
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,16,128,1,float16,fp8,8191,0.16403520107269287
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,16,128,1,float16,float16,8191,0.25115199089050294
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,12,12,128,1,float16,float16,3,0.026096001267433167
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,12,12,128,1,float16,fp8,1,0.02545120120048523
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,12,12,128,1,float16,float16,7,0.025839999318122864
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,12,12,128,1,float16,fp8,15,0.02547520101070404
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,12,12,128,1,float16,fp8,3,0.025603199005126955
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,12,12,128,1,float16,float16,31,0.026056000590324403
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,12,12,128,1,float16,fp8,31,0.025516799092292784
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,12,12,128,1,float16,float16,63,0.025934401154518127
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,12,12,128,1,float16,fp8,63,0.025572800636291505
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,12,12,128,1,float16,float16,255,0.04572800099849701
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,12,12,128,1,float16,fp8,127,0.030614399909973146
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,12,12,128,1,float16,fp8,255,0.03820480108261108
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,12,12,128,1,float16,float16,511,0.06755200028419495
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,12,12,128,1,float16,fp8,511,0.056446397304534913
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,12,12,128,1,float16,float16,1023,0.11203839778900146
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,12,12,128,1,float16,fp8,1023,0.08286240100860595
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,16,128,1,float16,float16,31,0.013096000254154205
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,12,12,128,1,float16,float16,2047,0.2002351999282837
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,12,12,128,1,float16,fp8,2047,0.1418303966522217
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,16,128,1,float16,float16,255,0.017084799706935883
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,12,12,128,1,float16,fp8,4095,0.25160799026489256
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,12,12,128,1,float16,float16,1,0.009859199821949004
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,12,12,128,1,float16,fp8,1,0.009598399698734283
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,12,12,128,1,float16,float16,3,0.009886399656534196
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,16,128,1,float16,fp8,2047,0.052753597497940063
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,12,12,128,1,float16,fp8,3,0.009721600264310837
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,12,12,128,1,float16,float16,8191,0.7206416130065918
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,12,12,128,1,float16,fp8,7,0.009993600100278855
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,12,12,128,1,float16,fp8,15,0.009942399710416794
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,12,12,128,1,float16,float16,31,0.010225600004196167
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,12,12,128,1,float16,float16,15,0.010179200023412705
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,12,12,128,1,float16,fp8,31,0.010017599910497665
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,12,12,128,1,float16,fp8,8191,0.4791247844696045
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,12,12,128,1,float16,fp8,63,0.010035199671983719
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,12,12,128,1,float16,float16,63,0.01027199998497963
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,12,12,128,1,float16,fp8,127,0.010635200142860412
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,12,12,128,1,float16,float16,127,0.010391999781131745
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,12,12,128,1,float16,float16,15,0.026179200410842894
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,12,12,128,1,float16,float16,511,0.01361120045185089
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,12,12,128,1,float16,fp8,7,0.025729599595069885
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,12,12,128,1,float16,float16,1023,0.014838400483131408
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,12,12,128,1,float16,fp8,1023,0.014486399292945863
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,12,12,128,1,float16,float16,2047,0.01523360013961792
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,12,12,128,1,float16,float16,127,0.031697601079940796
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,12,12,128,1,float16,fp8,2047,0.015158399939537048
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,12,12,128,1,float16,float16,4095,0.017580799758434296
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,12,12,128,1,float16,fp8,4095,0.017105600237846373
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,12,12,128,1,float16,float16,8191,0.023609599471092223
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,12,12,128,1,float16,fp8,8191,0.020817600190639496
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,12,12,128,1,float16,fp8,1,0.010259199887514114
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,12,12,128,1,float16,float16,3,0.010203199833631516
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,12,12,128,1,float16,fp8,3,0.010103999823331832
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,12,12,128,1,float16,float16,7,0.010113599896430969
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,12,12,128,1,float16,fp8,7,0.010014399886131287
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,12,12,128,1,float16,float16,15,0.010153599828481675
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,12,12,128,1,float16,fp8,15,0.010036800056695938
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,12,12,128,1,float16,float16,31,0.010288000106811523
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,12,12,128,1,float16,fp8,31,0.010196799784898758
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,12,12,128,1,float16,float16,63,0.010406400263309478
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,12,12,128,1,float16,fp8,63,0.010440000146627427
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,12,12,128,1,float16,float16,127,0.010547199845314026
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,12,12,128,1,float16,fp8,127,0.01103999987244606
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,12,12,128,1,float16,float16,4095,0.3701119899749756
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,12,12,128,1,float16,float16,255,0.01180799975991249
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,12,12,128,1,float16,float16,511,0.013998399674892425
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,12,12,128,1,float16,fp8,511,0.014790399372577668
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,12,12,128,1,float16,fp8,255,0.011873599886894227
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,12,12,128,1,float16,float16,1023,0.015311999619007111
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,12,12,128,1,float16,float16,2047,0.01767680048942566
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,12,12,128,1,float16,fp8,1023,0.015068799257278442
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,12,12,128,1,float16,fp8,2047,0.016913600265979767
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,12,12,128,1,float16,float16,4095,0.023887999355793
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,12,12,128,1,float16,fp8,4095,0.02041279971599579
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,12,12,128,1,float16,float16,8191,0.037747201323509214
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,12,12,128,1,float16,fp8,8191,0.029547199606895447
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,12,12,128,1,float16,fp8,1,0.01647839993238449
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,12,12,128,1,float16,float16,3,0.01642400026321411
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,12,12,128,1,float16,float16,255,0.011462400108575821
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,12,12,128,1,float16,fp8,3,0.016433599591255187
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,12,12,128,1,float16,float16,7,0.016894400119781494
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,12,12,128,1,float16,float16,15,0.016846400499343873
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,12,12,128,1,float16,fp8,15,0.016543999314308167
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,12,12,128,1,float16,fp8,511,0.01366879940032959
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,12,12,128,1,float16,float16,31,0.016766400635242464
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,12,12,128,1,float16,fp8,31,0.01669279932975769
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,12,12,128,1,float16,float16,63,0.01680160015821457
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,12,12,128,1,float16,float16,1,0.010096000134944915
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,12,12,128,1,float16,fp8,63,0.01661120057106018
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,12,12,128,1,float16,float16,127,0.017004799842834473
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,12,12,128,1,float16,fp8,127,0.01902880072593689
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,12,12,128,1,float16,fp8,255,0.02183839976787567
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,12,12,128,1,float16,float16,511,0.038332799077034
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,12,12,128,1,float16,fp8,511,0.03205440044403076
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,12,12,128,1,float16,float16,1023,0.062212800979614256
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,12,12,128,1,float16,fp8,1023,0.04637919962406158
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,12,12,128,1,float16,float16,2047,0.106278395652771
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,12,12,128,1,float16,fp8,2047,0.07591999769210815
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,12,12,128,1,float16,fp8,4095,0.1309856057167053
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,12,12,128,1,float16,float16,7,0.010204800218343735
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,12,12,128,1,float16,float16,8191,0.3692239999771118
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,12,12,128,1,float16,float16,1,0.016675199568271636
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,12,12,128,1,float16,float16,1,0.01053439974784851
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,12,12,128,1,float16,fp8,1,0.010251200199127198
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,12,12,128,1,float16,fp8,8191,0.2441391944885254
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,12,12,128,1,float16,float16,3,0.010878399759531022
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,12,12,128,1,float16,fp8,3,0.010353600233793258
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,12,12,128,1,float16,fp8,255,0.011401599645614624
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,12,12,128,1,float16,float16,15,0.010441599786281586
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,12,12,128,1,float16,float16,7,0.010422399640083313
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,12,12,128,1,float16,float16,31,0.010465600341558457
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,12,12,128,1,float16,fp8,15,0.010153599828481675
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,12,12,128,1,float16,fp8,31,0.01040000021457672
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,12,12,128,1,float16,float16,63,0.010375999659299851
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,12,12,128,1,float16,fp8,63,0.010236799716949463
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,12,12,128,1,float16,fp8,127,0.011259199678897857
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,12,12,128,1,float16,float16,255,0.011919999867677689
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,12,12,128,1,float16,fp8,255,0.011939200013875962
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,12,12,128,1,float16,float16,511,0.014168000221252442
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,12,12,128,1,float16,fp8,511,0.015304000675678253
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,12,12,128,1,float16,float16,1023,0.0176256000995636
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,12,12,128,1,float16,fp8,1023,0.02017119973897934
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,12,12,128,1,float16,float16,2047,0.02894560098648071
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,12,12,128,1,float16,fp8,2047,0.025990399718284606
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,12,12,128,1,float16,float16,255,0.02561439871788025
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,12,12,128,1,float16,float16,4095,0.047366398572921756
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,12,12,128,1,float16,fp8,4095,0.04031200110912323
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,12,12,128,1,float16,fp8,8191,0.06707680225372314
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,12,12,128,1,float16,float16,1,0.043844801187515256
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,12,12,128,1,float16,float16,8191,0.07852479815483093
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,12,12,128,1,float16,fp8,1,0.04347839951515198
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,12,12,128,1,float16,fp8,3,0.044012799859046936
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,12,12,128,1,float16,float16,7,0.04442879855632782
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,12,12,128,1,float16,fp8,7,0.04389919936656952
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,12,12,128,1,float16,float16,15,0.04444479942321777
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,12,12,128,1,float16,fp8,15,0.04417439997196197
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,12,12,128,1,float16,float16,31,0.044556799530982974
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,12,12,128,1,float16,fp8,31,0.04430719912052154
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,12,12,128,1,float16,float16,63,0.04843519926071167
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,12,12,128,1,float16,fp8,63,0.04452320039272308
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,12,12,128,1,float16,fp8,127,0.05736960172653198
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,12,12,128,1,float16,float16,255,0.0780463993549347
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,12,12,128,1,float16,fp8,255,0.06946240067481994
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,12,12,128,1,float16,float16,511,0.11862560510635375
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,12,12,128,1,float16,fp8,511,0.10296000242233276
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,12,12,128,1,float16,float16,1023,0.20773279666900635
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,12,12,128,1,float16,fp8,1023,0.15629760026931763
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,12,12,128,1,float16,fp8,2047,0.2741983890533447
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,12,12,128,1,float16,float16,2047,0.3808543920516968
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,12,12,128,1,float16,fp8,7,0.010375999659299851
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,12,12,128,1,float16,fp8,7,0.016515199840068818
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,12,12,128,1,float16,float16,1,0.07843199968338013
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,12,12,128,1,float16,float16,127,0.0105103999376297
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,12,12,128,1,float16,fp8,1,0.07799040079116822
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,12,12,128,1,float16,float16,3,0.07778239846229554
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,12,12,128,1,float16,fp8,3,0.07813760042190551
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,12,12,128,1,float16,float16,7,0.07785919904708863
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,12,12,128,1,float16,fp8,4095,0.4945663928985596
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,12,12,128,1,float16,float16,15,0.07824159860610962
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,12,12,128,1,float16,float16,4095,0.7229584217071533
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,12,12,128,1,float16,fp8,15,0.07784000039100647
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,12,12,128,1,float16,float16,63,0.08676639795303345
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,12,12,128,1,float16,float16,3,0.04390400052070618
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,12,12,128,1,float16,fp8,31,0.07848479747772216
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,12,12,128,1,float16,fp8,63,0.08445119857788086
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,12,12,128,1,float16,float16,127,0.09433919787406922
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,12,12,128,1,float16,fp8,127,0.10471199750900269
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,12,12,128,1,float16,float16,255,0.14242559671401978
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,12,12,128,1,float16,fp8,255,0.12558239698410034
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,12,12,128,1,float16,float16,4095,0.19308799505233765
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,12,12,128,1,float16,float16,511,0.2205728054046631
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,12,12,128,1,float16,fp8,511,0.19536800384521485
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,12,12,128,1,float16,float16,1,0.14752800464630128
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,12,12,128,1,float16,fp8,1,0.14601919651031495
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,12,12,128,1,float16,float16,1023,0.3953007936477661
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,12,12,128,1,float16,float16,3,0.14720319509506224
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,12,12,128,1,float16,fp8,1023,0.2991744041442871
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,12,12,128,1,float16,float16,7,0.1467344045639038
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,12,12,128,1,float16,fp8,7,0.1461151957511902
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,12,12,128,1,float16,float16,15,0.14723360538482666
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,12,12,128,1,float16,fp8,15,0.14525599479675294
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,12,12,128,1,float16,float16,31,0.15434720516204833
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,12,12,128,1,float16,fp8,31,0.14836159944534302
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,12,12,128,1,float16,float16,63,0.16365920305252074
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,12,12,128,1,float16,fp8,63,0.16075040102005006
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,12,12,128,1,float16,float16,127,0.18008480072021485
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,12,12,128,1,float16,fp8,127,0.19569599628448486
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,12,12,128,1,float16,float16,255,0.2649791955947876
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,12,12,128,1,float16,fp8,255,0.2420896053314209
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,12,12,128,1,float16,float16,1,0.01053759977221489
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,12,12,128,1,float16,fp8,1,0.01064319983124733
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,12,12,128,1,float16,float16,511,0.424513578414917
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,12,12,128,1,float16,fp8,7,0.07797440290451049
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,12,12,128,1,float16,fp8,511,0.3737551927566528
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,12,12,128,1,float16,float16,7,0.010664000362157821
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,12,12,128,1,float16,fp8,3,0.010387200117111205
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,12,12,128,1,float16,float16,31,0.07928640246391297
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,12,12,128,1,float16,fp8,15,0.010742399841547012
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,12,12,128,1,float16,float16,31,0.010731200128793717
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,12,12,128,1,float16,fp8,31,0.010782399773597717
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,12,12,128,1,float16,float16,63,0.010756800323724747
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,12,12,128,1,float16,fp8,63,0.010777600109577179
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,12,12,128,1,float16,float16,127,0.010907199978828431
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,12,12,128,1,float16,fp8,127,0.01186399981379509
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,12,12,128,1,float16,fp8,255,0.012508800625801087
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,12,12,128,1,float16,float16,511,0.014440000057220459
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,12,12,128,1,float16,fp8,511,0.020054399967193604
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,12,12,128,1,float16,float16,127,0.05434880256652832
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,12,12,128,1,float16,float16,1023,0.02704800069332123
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,12,12,128,1,float16,fp8,1023,0.02409760057926178
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,12,12,128,1,float16,float16,2047,0.040454399585723874
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,12,12,128,1,float16,fp8,2047,0.03522239923477173
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,12,12,128,1,float16,float16,4095,0.06427519917488098
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,12,12,128,1,float16,fp8,4095,0.053819197416305545
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,12,12,128,1,float16,fp8,3,0.14565919637680053
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,12,12,128,1,float16,float16,8191,0.11323039531707764
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,12,12,128,1,float16,fp8,8191,0.09238560199737549
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,12,12,128,1,float16,float16,1,0.27803199291229247
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,12,12,128,1,float16,fp8,1,0.27565441131591795
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,12,12,128,1,float16,fp8,3,0.2750943899154663
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,12,12,128,1,float16,fp8,7,0.2756207942962646
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,12,12,128,1,float16,float16,15,0.28371200561523435
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,12,12,128,1,float16,float16,3,0.010655999928712846
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,12,12,128,1,float16,fp8,15,0.27694239616394045
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,12,12,128,1,float16,fp8,7,0.010841599851846694
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,12,12,128,1,float16,float16,31,0.2957792043685913
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,12,12,128,1,float16,fp8,31,0.29580159187316896
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,12,12,128,1,float16,float16,255,0.012163200229406358
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,12,12,128,1,float16,float16,63,0.3135983943939209
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,12,12,128,1,float16,fp8,63,0.3060319900512695
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,12,12,128,1,float16,float16,127,0.33834400177001955
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,12,12,128,1,float16,fp8,127,0.375545597076416
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,12,12,128,1,float16,float16,255,0.4981872081756592
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,12,12,128,1,float16,fp8,255,0.46578240394592285
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,12,12,128,1,float16,float16,1,0.5382063865661622
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,12,12,128,1,float16,fp8,1,0.5344960212707519
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,12,12,128,1,float16,float16,3,0.5414303779602051
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,12,12,128,1,float16,float16,3,0.27808480262756347
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,12,12,128,1,float16,float16,7,0.5422095775604248
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,12,12,128,1,float16,fp8,7,0.5353375911712647
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,12,12,128,1,float16,float16,7,0.27740960121154784
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,12,12,128,1,float16,float16,15,0.5591392040252685
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,12,12,128,1,float16,fp8,15,0.5552048206329345
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,12,12,128,1,float16,float16,31,0.5737552165985107
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,12,12,128,1,float16,fp8,31,0.5759007930755615
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,12,12,128,1,float16,float16,63,0.6047920227050781
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,12,12,128,1,float16,float16,1,0.013142399489879608
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,12,12,128,1,float16,fp8,63,0.5970943927764892
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,12,12,128,1,float16,float16,3,0.013046400249004364
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,12,12,128,1,float16,fp8,1,0.012980799376964568
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,12,12,128,1,float16,float16,15,0.010569600015878677
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,12,12,128,1,float16,fp8,7,0.012852799892425538
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,12,12,128,1,float16,fp8,3,0.013084800541400909
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,12,12,128,1,float16,float16,15,0.012996800243854523
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,12,12,128,1,float16,float16,127,0.6572751998901367
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,12,12,128,1,float16,float16,31,0.013153600692749023
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,12,12,128,1,float16,fp8,31,0.01310880035161972
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,12,12,128,1,float16,fp8,63,0.012956799566745758
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,12,12,128,1,float16,float16,127,0.013686400651931763
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,12,12,128,1,float16,float16,63,0.013462400436401368
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,12,12,128,1,float16,fp8,127,0.015187199413776397
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,12,12,128,1,float16,float16,255,0.016438399255275727
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,12,12,128,1,float16,fp8,255,0.01645440012216568
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,12,12,128,1,float16,float16,511,0.02298399955034256
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,12,12,128,1,float16,float16,1023,0.038852798938751223
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,12,12,128,1,float16,fp8,1023,0.03293119966983795
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,12,12,128,1,float16,float16,2047,0.0626479983329773
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,12,12,128,1,float16,fp8,2047,0.05339679718017578
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,12,12,128,1,float16,float16,4095,0.11070560216903687
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,12,12,128,1,float16,fp8,4095,0.09132159948348999
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,12,12,128,1,float16,float16,8191,0.20507359504699707
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,8,128,1,float16,float16,1,0.019996799528598785
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,12,12,128,1,float16,fp8,3,0.5375328063964844
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,8,128,1,float16,fp8,1,0.019547200202941893
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,8,128,1,float16,float16,3,0.019732800126075745
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,8,128,1,float16,float16,7,0.019916799664497376
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,8,128,1,float16,fp8,7,0.01971520036458969
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,8,128,1,float16,float16,15,0.019969600439071655
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,8,128,1,float16,fp8,15,0.01968639940023422
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,8,128,1,float16,float16,31,0.020046399533748628
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,8,128,1,float16,fp8,31,0.019684800505638124
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,8,128,1,float16,float16,63,0.019977599382400513
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,8,128,1,float16,fp8,63,0.019622400403022766
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,8,128,1,float16,float16,127,0.020238399505615234
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,12,12,128,1,float16,float16,7,0.013089600205421447
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,12,12,128,1,float16,fp8,15,0.012851199507713318
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,8,128,1,float16,fp8,127,0.02295999974012375
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,8,128,1,float16,float16,255,0.03355039954185486
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,8,128,1,float16,fp8,255,0.02678079903125763
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,8,128,1,float16,float16,511,0.04794079959392548
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,8,128,1,float16,fp8,511,0.04037919938564301
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,12,12,128,1,float16,fp8,511,0.022993600368499754
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,8,128,1,float16,float16,2047,0.13735040426254272
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,8,128,1,float16,fp8,2047,0.09719359874725342
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,8,128,1,float16,float16,4095,0.2532480001449585
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,8,128,1,float16,fp8,4095,0.17077120542526245
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,12,12,128,1,float16,fp8,8191,0.16912959814071654
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,8,128,1,float16,float16,8191,0.4866000175476074
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,8,128,1,float16,fp8,8191,0.32100958824157716
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,8,128,1,float16,fp8,3,0.019524799287319185
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,8,128,1,float16,fp8,1,0.009408000111579894
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,8,128,1,float16,float16,1,0.009379199892282485
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,8,128,1,float16,float16,3,0.00950080007314682
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,8,128,1,float16,fp8,3,0.009409599751234055
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,8,128,1,float16,float16,7,0.009691199660301209
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,8,128,1,float16,fp8,7,0.009494400024414063
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,8,128,1,float16,float16,15,0.009571199864149093
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,8,128,1,float16,fp8,15,0.009440000355243682
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,8,128,1,float16,float16,31,0.009547200053930283
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,8,128,1,float16,fp8,31,0.009561599791049957
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,8,128,1,float16,float16,63,0.009510400146245957
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,8,128,1,float16,fp8,63,0.009601599723100662
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,8,128,1,float16,float16,127,0.009560000151395798
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,8,128,1,float16,fp8,127,0.010339199751615524
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,8,128,1,float16,float16,16383,0.958681583404541
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,8,128,1,float16,float16,255,0.010831999778747558
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,8,128,1,float16,fp8,255,0.011265599727630615
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,8,128,1,float16,fp8,511,0.01220960021018982
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,8,128,1,float16,float16,1023,0.013619199395179749
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,8,128,1,float16,float16,511,0.013977600634098053
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,12,12,128,1,float16,fp8,127,0.733132791519165
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,8,128,1,float16,float16,4095,0.015523199737071992
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,8,128,1,float16,fp8,4095,0.014822399616241455
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,8,128,1,float16,float16,8191,0.01781439930200577
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,8,128,1,float16,float16,16383,0.028748801350593566
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,8,128,1,float16,fp8,16383,0.6322480201721191
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,8,128,1,float16,fp8,16383,0.02263679951429367
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,8,128,1,float16,fp8,8191,0.017371200025081635
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,8,128,1,float16,fp8,1,0.009699200093746186
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,8,128,1,float16,float16,3,0.009801600128412247
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,8,128,1,float16,fp8,3,0.009812799841165542
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,8,128,1,float16,fp8,1023,0.013203200697898865
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,8,128,1,float16,float16,1,0.00950080007314682
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,8,128,1,float16,float16,7,0.009787199646234512
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,8,128,1,float16,fp8,7,0.009481599926948548
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,8,128,1,float16,float16,15,0.009755200147628785
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,8,128,1,float16,float16,31,0.009825599938631057
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,8,128,1,float16,fp8,15,0.009598399698734283
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,8,128,1,float16,fp8,31,0.009350399672985076
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,8,128,1,float16,float16,63,0.009761600196361542
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,8,128,1,float16,fp8,127,0.01027040034532547
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,8,128,1,float16,float16,127,0.010022400319576264
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,8,128,1,float16,float16,255,0.011196800321340562
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,8,128,1,float16,float16,511,0.013187199831008911
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,8,128,1,float16,float16,1023,0.07859839797019959
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,8,128,1,float16,fp8,2047,0.01451839953660965
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,8,128,1,float16,float16,1023,0.013758400082588195
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,8,128,1,float16,fp8,1023,0.057817602157592775
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,8,128,1,float16,float16,2047,0.014870400726795196
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,8,128,1,float16,fp8,1023,0.013513599336147309
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,8,128,1,float16,float16,4095,0.0174687996506691
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,8,128,1,float16,fp8,4095,0.016966399550437928
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,8,128,1,float16,float16,8191,0.02837440073490143
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,8,128,1,float16,fp8,8191,0.02141280025243759
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,8,128,1,float16,float16,16383,0.04328159987926483
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,8,128,1,float16,float16,1,0.013281600177288055
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,8,128,1,float16,fp8,1,0.01295360028743744
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,8,128,1,float16,float16,3,0.013051199913024902
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,8,128,1,float16,fp8,3,0.012947200238704682
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,8,128,1,float16,float16,7,0.013079999387264252
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,8,128,1,float16,fp8,7,0.013209599256515502
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,8,128,1,float16,float16,15,0.013303999602794648
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,8,128,1,float16,fp8,15,0.01313599944114685
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,8,128,1,float16,float16,31,0.01316480040550232
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,8,128,1,float16,fp8,31,0.013070400059223174
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,8,128,1,float16,float16,63,0.013387200236320496
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,8,128,1,float16,fp8,63,0.013284799456596375
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,8,128,1,float16,float16,127,0.013654400408267976
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,8,128,1,float16,fp8,127,0.015169599652290344
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,8,128,1,float16,float16,255,0.016828800737857818
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,8,128,1,float16,fp8,255,0.016603200137615202
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,8,128,1,float16,float16,511,0.027822399139404298
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,8,128,1,float16,fp8,511,0.02296479940414429
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,8,128,1,float16,float16,1023,0.044736000895500186
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,8,128,1,float16,fp8,1023,0.03333759903907776
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,8,128,1,float16,float16,2047,0.07475839853286743
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,8,128,1,float16,float16,2047,0.014260800182819366
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,8,128,1,float16,fp8,2047,0.053179198503494264
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,8,128,1,float16,fp8,63,0.009609600156545639
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,8,128,1,float16,float16,4095,0.13287999629974365
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,8,128,1,float16,fp8,255,0.011283200234174728
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,8,128,1,float16,fp8,511,0.012452799826860428
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,8,128,1,float16,fp8,4095,0.0904528021812439
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,8,128,1,float16,float16,1,0.009691199660301209
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,8,128,1,float16,fp8,8191,0.16549439430236818
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,8,128,1,float16,float16,8191,0.2509200096130371
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,8,128,1,float16,fp8,1,0.00966079980134964
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,8,128,1,float16,float16,7,0.009587199985980987
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,8,128,1,float16,fp8,3,0.009895999729633332
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,8,128,1,float16,float16,3,0.009667199850082398
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,8,128,1,float16,fp8,16383,0.03283840119838714
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,8,128,1,float16,float16,15,0.009894400089979171
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,8,128,1,float16,fp8,16383,0.31369121074676515
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,8,128,1,float16,float16,31,0.009915199875831605
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,8,128,1,float16,fp8,63,0.00979359969496727
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,8,128,1,float16,float16,63,0.009654399752616883
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,8,128,1,float16,fp8,127,0.01058880016207695
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,8,128,1,float16,float16,255,0.011513599753379821
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,8,128,1,float16,fp8,255,0.01159999966621399
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,8,128,1,float16,float16,511,0.013441599905490875
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,8,128,1,float16,fp8,511,0.013318400084972381
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,8,128,1,float16,float16,1023,0.014899200201034546
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,8,128,1,float16,fp8,1023,0.013950400054454803
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,8,128,1,float16,float16,2047,0.01611199975013733
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,8,128,1,float16,fp8,2047,0.016390399634838106
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,8,128,1,float16,float16,4095,0.027563199400901794
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,8,128,1,float16,fp8,4095,0.020776000618934632
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,8,128,1,float16,float16,8191,0.04329920113086701
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,8,128,1,float16,fp8,8191,0.03198879957199097
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,8,128,1,float16,float16,16383,0.07268800139427185
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,8,128,1,float16,fp8,16383,0.05078880190849304
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,8,128,1,float16,float16,1,0.032199999690055846
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,8,128,1,float16,fp8,1,0.031707200407981875
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,8,128,1,float16,fp8,2047,0.014297600090503692
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,8,128,1,float16,float16,3,0.03219360113143921
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,8,128,1,float16,fp8,3,0.03186239898204803
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,8,128,1,float16,float16,7,0.03215680122375488
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,8,128,1,float16,fp8,7,0.009505599737167358
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,8,128,1,float16,float16,15,0.03219040036201477
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,8,128,1,float16,float16,16383,0.48592801094055177
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,8,128,1,float16,fp8,15,0.032183998823165895
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,8,128,1,float16,float16,31,0.032579201459884646
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,8,128,1,float16,fp8,31,0.03235200047492981
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,8,128,1,float16,fp8,63,0.03226560056209564
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,8,128,1,float16,float16,127,0.04017280042171478
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,8,128,1,float16,fp8,127,0.03923999965190887
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,8,128,1,float16,float16,127,0.010016000270843506
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,8,128,1,float16,fp8,255,0.048614400625228885
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,8,128,1,float16,float16,255,0.055088001489639285
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,8,128,1,float16,float16,511,0.08368480205535889
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,8,128,1,float16,fp8,511,0.07126399874687195
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,8,128,1,float16,fp8,1023,0.10630719661712647
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,8,128,1,float16,float16,1023,0.14362879991531372
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,8,128,1,float16,float16,2047,0.26077919006347655
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,8,128,1,float16,fp8,2047,0.18586399555206298
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,8,128,1,float16,fp8,4095,0.33161919116973876
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,8,128,1,float16,float16,4095,0.48888158798217773
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,8,128,1,float16,float16,1,0.056086397171020506
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,8,128,1,float16,fp8,1,0.05550720095634461
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,8,128,1,float16,float16,3,0.056195199489593506
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,8,128,1,float16,float16,7,0.05618720054626465
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,8,128,1,float16,fp8,7,0.05604000091552734
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,8,128,1,float16,fp8,8191,0.6337711811065674
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,8,128,1,float16,float16,15,0.056536000967025754
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,8,128,1,float16,float16,31,0.056169599294662476
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,8,128,1,float16,fp8,15,0.05633119940757751
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,8,128,1,float16,fp8,31,0.05610560178756714
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,8,128,1,float16,float16,63,0.06219999790191651
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,8,128,1,float16,fp8,63,0.05620800256729126
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,8,128,1,float16,float16,127,0.06820480227470398
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,8,128,1,float16,fp8,127,0.07276160120964051
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,8,128,1,float16,fp8,7,0.031969600915908815
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,8,128,1,float16,float16,255,0.09646559953689575
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,8,128,1,float16,fp8,15,0.009640000015497207
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,8,128,1,float16,fp8,255,0.08817920088768005
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,8,128,1,float16,float16,511,0.1520751953125
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,8,128,1,float16,float16,63,0.032918399572372435
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,8,128,1,float16,fp8,511,0.13207679986953735
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,8,128,1,float16,float16,1023,0.2708031892776489
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,8,128,1,float16,fp8,1023,0.20333600044250488
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,8,128,1,float16,float16,1,0.10247999429702759
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,8,128,1,float16,fp8,1,0.10263359546661377
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,8,128,1,float16,float16,3,0.10267200469970703
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,8,128,1,float16,fp8,3,0.1033951997756958
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,8,128,1,float16,float16,2047,0.4983935832977295
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,8,128,1,float16,float16,7,0.1026144027709961
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,8,128,1,float16,fp8,7,0.10360959768295289
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,8,128,1,float16,fp8,15,0.1030959963798523
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,8,128,1,float16,float16,15,0.10375039577484131
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,8,128,1,float16,fp8,31,0.10296479463577271
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,8,128,1,float16,float16,31,0.10518079996109009
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,8,128,1,float16,float16,63,0.11384320259094238
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,8,128,1,float16,fp8,63,0.11099040508270264
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,8,128,1,float16,fp8,3,0.05592160224914551
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,8,128,1,float16,float16,127,0.12413439750671387
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,8,128,1,float16,float16,255,0.1763856053352356
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,8,128,1,float16,float16,8191,0.9542976379394531
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,8,128,1,float16,fp8,255,0.16588959693908692
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,8,128,1,float16,float16,511,0.2926271915435791
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,8,128,1,float16,fp8,511,0.2517008066177368
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,8,128,1,float16,float16,1,0.010281600058078766
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,8,128,1,float16,fp8,1,0.01011039987206459
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,8,128,1,float16,float16,3,0.010215999931097031
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,8,128,1,float16,fp8,3,0.010196799784898758
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,8,128,1,float16,float16,7,0.010310400277376175
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,8,128,1,float16,fp8,7,0.010092800110578537
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,8,128,1,float16,fp8,1023,0.39589600563049315
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,8,128,1,float16,fp8,15,0.010143999755382539
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,8,128,1,float16,fp8,31,0.010284800082445145
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,8,128,1,float16,float16,31,0.010183999687433243
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,8,128,1,float16,float16,63,0.010395199805498124
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,8,128,1,float16,fp8,31,0.009864000231027603
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,8,128,1,float16,fp8,63,0.01005920022726059
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,8,128,1,float16,float16,127,0.010531199723482132
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,8,128,1,float16,fp8,127,0.011108800023794174
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,8,128,1,float16,float16,255,0.011638399958610535
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,8,128,1,float16,float16,511,0.014108799397945404
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,8,128,1,float16,fp8,511,0.01401119977235794
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,8,128,1,float16,float16,1023,0.0166143998503685
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,8,128,1,float16,fp8,1023,0.016271999478340148
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,8,128,1,float16,float16,2047,0.027961599826812743
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,8,128,1,float16,fp8,2047,0.021030400693416596
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,8,128,1,float16,float16,4095,0.04325760006904602
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,8,128,1,float16,fp8,4095,0.03173440098762512
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,8,128,1,float16,float16,8191,0.07219200134277344
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,8,128,1,float16,fp8,8191,0.05008000135421753
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,8,128,1,float16,float16,16383,0.13205920457839965
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,8,128,1,float16,fp8,2047,0.36163361072540284
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,8,8,128,1,float16,float16,1,0.1928671956062317
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,8,8,128,1,float16,fp8,1,0.19052000045776368
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,8,8,128,1,float16,float16,3,0.19301120042800904
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,8,128,1,float16,fp8,127,0.135862398147583
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,8,8,128,1,float16,float16,7,0.19329919815063476
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,8,8,128,1,float16,fp8,7,0.19024319648742677
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,8,8,128,1,float16,fp8,15,0.19048960208892823
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,8,8,128,1,float16,float16,31,0.20193440914154054
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,8,128,1,float16,float16,1023,0.5527455806732178
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,8,8,128,1,float16,fp8,31,0.19824320077896118
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,8,8,128,1,float16,fp8,63,0.2095168113708496
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,8,128,1,float16,fp8,255,0.011699199676513672
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,8,8,128,1,float16,float16,127,0.23002080917358397
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,8,8,128,1,float16,fp8,127,0.2576767921447754
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,8,8,128,1,float16,float16,255,0.3337199926376343
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,8,8,128,1,float16,fp8,255,0.3143071889877319
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,8,128,1,float16,fp8,16383,0.08814240097999573
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,8,8,128,1,float16,float16,1,0.36567039489746095
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,8,8,128,1,float16,float16,511,0.5758639812469483
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,8,8,128,1,float16,fp8,511,0.48131837844848635
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,8,8,128,1,float16,fp8,1,0.36259520053863525
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,8,8,128,1,float16,float16,3,0.36499040126800536
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,8,8,128,1,float16,fp8,3,0.36608479022979734
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,8,8,128,1,float16,fp8,3,0.19135839939117433
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,8,8,128,1,float16,fp8,7,0.36313600540161134
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,8,8,128,1,float16,float16,7,0.36617441177368165
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,8,8,128,1,float16,float16,15,0.37460319995880126
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,8,8,128,1,float16,float16,15,0.1937551975250244
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,8,8,128,1,float16,fp8,15,0.36934719085693357
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,8,8,128,1,float16,float16,31,0.38487679958343507
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,8,8,128,1,float16,fp8,31,0.3901792049407959
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,8,8,128,1,float16,float16,63,0.407532787322998
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,8,8,128,1,float16,fp8,63,0.3999936103820801
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,8,128,1,float16,float16,15,0.010190399736166
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,8,128,1,float16,float16,1,0.010187199711799622
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,8,8,128,1,float16,fp8,127,0.49901599884033204
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,8,128,1,float16,float16,3,0.010206399857997895
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,8,128,1,float16,fp8,1,0.010230399668216705
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,8,128,1,float16,fp8,7,0.010076799988746643
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,8,128,1,float16,float16,7,0.0103472001850605
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,8,128,1,float16,float16,15,0.010524799674749374
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,8,128,1,float16,fp8,15,0.010156799852848054
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,8,128,1,float16,float16,31,0.010550399869680404
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,8,128,1,float16,fp8,31,0.010419200360774993
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,8,128,1,float16,float16,63,0.010355199873447418
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,8,8,128,1,float16,float16,63,0.21263680458068848
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,8,8,128,1,float16,float16,255,0.6523695945739746
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,8,128,1,float16,float16,127,0.010657600313425063
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,8,8,128,1,float16,fp8,255,0.6075920104980469
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,8,128,1,float16,fp8,255,0.011750400066375732
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,8,128,1,float16,fp8,127,0.011220800131559372
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,8,128,1,float16,float16,255,0.011923199892044068
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,8,128,1,float16,fp8,511,0.015356799960136414
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,8,128,1,float16,float16,1023,0.02725279927253723
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,8,128,1,float16,float16,511,0.014238399267196656
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,8,128,1,float16,fp8,1023,0.019734400510787963
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,8,128,1,float16,fp8,2047,0.031121599674224853
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,8,128,1,float16,float16,2047,0.042217600345611575
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,8,128,1,float16,fp8,4095,0.049774399399757384
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,8,128,1,float16,float16,4095,0.07297279834747314
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,8,128,1,float16,float16,8191,0.13290239572525026
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,8,128,1,float16,fp8,8191,0.08718720078468323
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,4,128,1,float16,float16,1,0.013900800049304963
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,4,128,1,float16,fp8,1,0.014084799587726593
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,4,128,1,float16,float16,3,0.013840000331401824
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,8,128,1,float16,fp8,16383,0.16163840293884277
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,4,128,1,float16,fp8,3,0.014214399456977844
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,4,128,1,float16,fp8,7,0.014108799397945404
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,4,128,1,float16,float16,15,0.014115199446678162
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,4,128,1,float16,fp8,15,0.014071999490261078
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,4,128,1,float16,float16,31,0.014256000518798828
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,4,128,1,float16,float16,63,0.014019200205802917
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,4,128,1,float16,fp8,63,0.014054399728775025
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,4,128,1,float16,float16,127,0.014404800534248353
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,4,128,1,float16,fp8,127,0.01589920073747635
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,4,128,1,float16,float16,255,0.01736160069704056
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,4,128,1,float16,fp8,255,0.0176704004406929
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,8,8,128,1,float16,float16,127,0.44312319755554197
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,4,128,1,float16,float16,511,0.029300799965858458
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,8,128,1,float16,fp8,3,0.010131199657917023
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,4,128,1,float16,float16,1023,0.04575200080871582
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,4,128,1,float16,fp8,511,0.02425439953804016
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,4,128,1,float16,float16,2047,0.07543519735336304
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,8,128,1,float16,fp8,63,0.010620799660682679
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,4,128,1,float16,fp8,1023,0.03481279909610748
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,4,128,1,float16,fp8,2047,0.05421760082244873
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,4,128,1,float16,float16,4095,0.13432480096817018
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,4,128,1,float16,fp8,4095,0.09090399742126465
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,4,128,1,float16,fp8,8191,0.16755839586257934
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,4,128,1,float16,float16,8191,0.2515903949737549
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,4,128,1,float16,float16,16383,0.4864783763885498
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,4,128,1,float16,fp8,16383,0.3158960103988647
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,4,128,1,float16,float16,7,0.013835200667381286
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,4,128,1,float16,float16,1,0.009446399658918381
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,4,128,1,float16,fp8,1,0.009294400364160538
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,4,128,1,float16,fp8,3,0.00934239998459816
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,4,128,1,float16,float16,3,0.009369599819183349
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,4,128,1,float16,fp8,31,0.01401440054178238
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,4,128,1,float16,fp8,32767,0.6233727931976318
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,4,128,1,float16,float16,7,0.009384000301361084
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,4,128,1,float16,float16,15,0.009331200271844864
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,4,128,1,float16,fp8,15,0.009419199824333192
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,4,128,1,float16,fp8,31,0.009297599643468856
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,4,128,1,float16,float16,63,0.009435199946165086
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,4,128,1,float16,float16,32767,0.964350414276123
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,4,128,1,float16,fp8,63,0.009355200082063675
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,4,128,1,float16,float16,31,0.009364800155162811
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,4,128,1,float16,fp8,255,0.010806400328874588
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,4,128,1,float16,fp8,127,0.01008479967713356
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,4,128,1,float16,float16,255,0.011097600311040878
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,4,128,1,float16,fp8,511,0.013027200102806091
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,4,128,1,float16,float16,1023,0.014369599521160126
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,4,128,1,float16,float16,2047,0.014368000626564025
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,4,128,1,float16,fp8,1023,0.014286400377750396
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,4,128,1,float16,fp8,2047,0.013704000413417817
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,4,128,1,float16,float16,4095,0.014839999377727509
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,4,128,1,float16,fp8,8191,0.01576319932937622
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,4,128,1,float16,fp8,4095,0.014710399508476257
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,4,128,1,float16,float16,16383,0.018691200017929076
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,4,128,1,float16,fp8,16383,0.01881919950246811
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,4,128,1,float16,float16,32767,0.03027999997138977
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,4,128,1,float16,float16,1,0.009545599669218063
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,4,128,1,float16,fp8,1,0.009356799721717834
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,4,128,1,float16,float16,3,0.009544000029563904
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,4,128,1,float16,fp8,3,0.009523200243711472
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,4,128,1,float16,float16,7,0.009710399806499482
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,4,128,1,float16,fp8,7,0.009595199674367904
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,4,128,1,float16,float16,15,0.009758400171995163
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,4,128,1,float16,fp8,15,0.009748800098896027
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,4,128,1,float16,float16,31,0.009804800152778625
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,4,128,1,float16,fp8,31,0.009705600142478944
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,4,128,1,float16,float16,63,0.009830400347709656
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,4,128,1,float16,fp8,63,0.009571199864149093
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,4,128,1,float16,float16,127,0.009699200093746186
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,4,128,1,float16,fp8,127,0.010289599746465683
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,4,128,1,float16,float16,255,0.010779199749231338
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,8,128,1,float16,float16,16383,0.24970879554748535
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,4,128,1,float16,fp8,255,0.010920000076293946
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,4,128,1,float16,fp8,511,0.013315199315547943
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,4,128,1,float16,float16,511,0.012723200023174286
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,4,128,1,float16,float16,1023,0.014443199336528777
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,4,128,1,float16,float16,2047,0.015022400021553039
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,4,128,1,float16,fp8,1023,0.013672000169754029
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,4,128,1,float16,fp8,2047,0.014153599739074707
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,4,128,1,float16,float16,4095,0.01592320054769516
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,4,128,1,float16,float16,127,0.009491200000047684
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,4,128,1,float16,fp8,4095,0.01507679969072342
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,4,128,1,float16,float16,8191,0.018139199912548067
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,4,128,1,float16,float16,511,0.012923200428485871
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,4,128,1,float16,float16,16383,0.02935200035572052
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,4,128,1,float16,fp8,32767,0.034292799234390256
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,4,128,1,float16,float16,32767,0.04591200053691864
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,4,128,1,float16,fp8,16383,0.023052799701690673
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,4,128,1,float16,fp8,1,0.010785599797964096
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,4,128,1,float16,float16,8191,0.015932799875736238
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,4,128,1,float16,fp8,3,0.010793600231409073
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,4,128,1,float16,float16,7,0.01069760024547577
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,4,128,1,float16,fp8,7,0.010756800323724747
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,4,128,1,float16,float16,15,0.010679999738931656
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,4,128,1,float16,float16,31,0.010700800269842149
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,4,128,1,float16,fp8,32767,0.024052800238132478
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,4,128,1,float16,fp8,31,0.010686399787664414
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,4,128,1,float16,fp8,63,0.010732799768447876
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,4,128,1,float16,float16,63,0.010811199992895126
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,4,128,1,float16,float16,127,0.010952000319957734
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,4,128,1,float16,fp8,127,0.011761599779129028
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,4,128,1,float16,float16,255,0.012193600088357926
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,4,128,1,float16,fp8,255,0.012359999865293504
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,4,128,1,float16,float16,511,0.014595200121402741
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,4,128,1,float16,fp8,511,0.016118399798870087
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,4,128,1,float16,float16,1023,0.02747200131416321
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,4,128,1,float16,fp8,1023,0.020852799713611602
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,4,128,1,float16,float16,2047,0.042814400792121884
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,4,128,1,float16,fp8,2047,0.0318336009979248
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,4,128,1,float16,float16,4095,0.07271199822425842
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,4,128,1,float16,float16,8191,0.13203359842300416
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,4,128,1,float16,fp8,8191,0.08816319704055786
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,4,128,1,float16,fp8,7,0.009363199770450591
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,4,128,1,float16,float16,16383,0.253110408782959
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,4,128,1,float16,fp8,16383,0.1632192015647888
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,4,128,1,float16,float16,1,0.010929600149393082
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,4,128,1,float16,float16,3,0.010788799822330475
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,4,128,1,float16,float16,1,0.009940800070762635
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,4,128,1,float16,fp8,1,0.009544000029563904
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,4,128,1,float16,fp8,32767,0.3114943981170654
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,4,128,1,float16,fp8,3,0.00947519987821579
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,4,128,1,float16,fp8,15,0.010692799836397171
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,4,128,1,float16,fp8,7,0.009446399658918381
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,4,128,1,float16,float16,7,0.009702400118112565
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,4,128,1,float16,float16,15,0.009625600278377533
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,4,128,1,float16,fp8,15,0.009537599980831146
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,4,128,1,float16,float16,31,0.009745600074529648
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,4,128,1,float16,fp8,31,0.009625600278377533
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,4,128,1,float16,float16,63,0.009862399846315383
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,4,128,1,float16,fp8,63,0.009380800276994705
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,4,128,1,float16,float16,127,0.010260800272226334
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,4,128,1,float16,fp8,127,0.010328000038862228
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,4,128,1,float16,float16,255,0.011044800281524658
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,4,128,1,float16,fp8,255,0.01127680018544197
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,4,128,1,float16,float16,511,0.013220800459384919
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,4,128,1,float16,fp8,511,0.013439999520778656
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,4,128,1,float16,float16,1023,0.014819200336933135
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,4,128,1,float16,fp8,2047,0.015406399965286255
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,4,128,1,float16,fp8,4095,0.017841599881649017
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,4,128,1,float16,float16,8191,0.028859201073646545
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,4,128,1,float16,fp8,8191,0.022009600698947907
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,4,128,1,float16,fp8,4095,0.05055519938468933
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,4,128,1,float16,float16,16383,0.04410879909992218
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,4,128,1,float16,fp8,16383,0.03367519974708557
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,4,128,1,float16,float16,32767,0.07408319711685181
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,4,128,1,float16,fp8,32767,0.052249598503112796
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,4,128,1,float16,float16,1,0.02104640007019043
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,4,128,1,float16,fp8,1,0.02110240012407303
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,4,128,1,float16,float16,3,0.021035200357437132
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,4,128,1,float16,fp8,3,0.021155199408531188
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,4,128,1,float16,float16,7,0.02102559953927994
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,4,128,1,float16,fp8,7,0.021158400177955627
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,4,128,1,float16,float16,15,0.021268799901008606
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,4,128,1,float16,fp8,15,0.02126079946756363
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,4,128,1,float16,float16,31,0.021305599808692934
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,4,128,1,float16,fp8,31,0.02107200026512146
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,4,128,1,float16,float16,63,0.021491199731826782
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,4,128,1,float16,fp8,63,0.021031999588012697
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,4,128,1,float16,float16,127,0.02165440022945404
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,4,128,1,float16,fp8,127,0.024465599656105043
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,4,128,1,float16,float16,255,0.034641599655151366
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,4,128,1,float16,fp8,8191,0.017847999930381775
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,4,128,1,float16,float16,511,0.04881280064582825
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,4,128,1,float16,fp8,511,0.041945600509643556
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,4,128,1,float16,float16,1023,0.08071039915084839
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,4,128,1,float16,fp8,1023,0.05987840294837952
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,4,128,1,float16,float16,32767,0.49358081817626953
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,4,128,1,float16,float16,2047,0.13924640417099
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,4,128,1,float16,fp8,2047,0.09828640222549438
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,4,128,1,float16,float16,3,0.0097680002450943
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,4,128,1,float16,fp8,4095,0.17191040515899658
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,4,128,1,float16,float16,4095,0.25436160564422605
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,4,128,1,float16,float16,8191,0.4873744010925293
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,4,128,1,float16,fp8,1023,0.014297600090503692
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,4,128,1,float16,float16,4095,0.01842239946126938
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,4,4,128,1,float16,float16,1,0.03487679958343506
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,4,4,128,1,float16,float16,3,0.03484320044517517
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,4,4,128,1,float16,fp8,3,0.034385600686073305
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,4,4,128,1,float16,float16,7,0.03489440083503723
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,4,4,128,1,float16,fp8,7,0.03482399880886078
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,4,4,128,1,float16,float16,15,0.03458240032196045
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,4,128,1,float16,float16,16383,0.9583120346069336
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,4,4,128,1,float16,fp8,15,0.03452000021934509
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,4,128,1,float16,fp8,16383,0.6365024089813233
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,4,4,128,1,float16,fp8,31,0.03455840051174164
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,4,4,128,1,float16,float16,31,0.034720000624656674
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,4,4,128,1,float16,fp8,63,0.03444800078868866
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,4,4,128,1,float16,float16,127,0.04384160041809082
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,4,4,128,1,float16,fp8,127,0.04175040125846863
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,4,128,1,float16,fp8,255,0.02837440073490143
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,4,4,128,1,float16,fp8,255,0.05082079768180847
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,4,4,128,1,float16,float16,511,0.08651040196418762
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,4,4,128,1,float16,fp8,511,0.0746511995792389
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,4,4,128,1,float16,float16,1023,0.14434880018234253
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,4,4,128,1,float16,fp8,1023,0.10965440273284913
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,4,4,128,1,float16,float16,2047,0.26218400001525877
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,4,4,128,1,float16,fp8,2047,0.18689119815826416
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,4,128,1,float16,fp8,8191,0.3234384059906006
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,4,128,1,float16,float16,2047,0.016152000427246092
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,4,4,128,1,float16,fp8,1,0.034985598921775815
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,4,4,128,1,float16,fp8,1,0.06191840171813965
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,4,4,128,1,float16,float16,3,0.062015998363494876
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,4,4,128,1,float16,float16,1,0.0612496018409729
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,4,4,128,1,float16,float16,4095,0.49106559753417967
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,4,4,128,1,float16,fp8,3,0.06234239935874939
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,4,4,128,1,float16,float16,7,0.06144800186157227
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,4,4,128,1,float16,fp8,7,0.06232320070266724
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,4,4,128,1,float16,fp8,15,0.06278560161590577
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,4,4,128,1,float16,float16,31,0.061737602949142455
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,4,4,128,1,float16,fp8,31,0.06232960224151611
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,4,4,128,1,float16,float16,63,0.06741920113563538
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,4,4,128,1,float16,fp8,63,0.06256800293922424
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,4,4,128,1,float16,float16,127,0.07176160216331481
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,4,4,128,1,float16,fp8,127,0.08008480072021484
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,4,4,128,1,float16,float16,255,0.1009600043296814
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,4,4,128,1,float16,float16,511,0.1578223943710327
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,4,4,128,1,float16,float16,63,0.03536159992218017
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,4,4,128,1,float16,fp8,511,0.13928799629211425
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,4,4,128,1,float16,float16,1023,0.27367520332336426
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,4,4,128,1,float16,fp8,1023,0.2099008083343506
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,4,4,128,1,float16,float16,255,0.05753120183944702
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,4,128,1,float16,float16,1,0.010043200105428696
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,4,128,1,float16,fp8,1,0.00990239977836609
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,4,128,1,float16,float16,3,0.009921599924564362
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,4,4,128,1,float16,float16,2047,0.5009712219238281
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,4,128,1,float16,fp8,3,0.009947200119495393
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,4,128,1,float16,float16,7,0.009889599680900574
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,4,128,1,float16,float16,15,0.010009600222110749
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,4,4,128,1,float16,fp8,2047,0.36814720630645753
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,4,128,1,float16,fp8,7,0.009985599666833878
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,4,128,1,float16,float16,31,0.009958399832248688
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,4,128,1,float16,fp8,31,0.009860800206661224
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,4,128,1,float16,float16,63,0.010239999741315842
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,4,128,1,float16,float16,127,0.010089600086212158
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,4,128,1,float16,fp8,127,0.010795199871063232
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,4,128,1,float16,float16,255,0.011596799641847611
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,4,128,1,float16,fp8,255,0.01133119985461235
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,4,128,1,float16,float16,511,0.013635200262069703
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,4,128,1,float16,fp8,511,0.014524799585342408
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,4,128,1,float16,float16,1023,0.015982399880886077
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,4,128,1,float16,fp8,1023,0.014958399534225463
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,4,128,1,float16,fp8,2047,0.017505599558353423
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,4,128,1,float16,float16,4095,0.028937599062919615
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,4,128,1,float16,fp8,4095,0.022064000368118286
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,4,128,1,float16,float16,8191,0.04465759992599487
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,4,128,1,float16,fp8,8191,0.03371039927005768
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,4,4,128,1,float16,fp8,4095,0.3341439962387085
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,4,128,1,float16,float16,16383,0.07463200092315674
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,4,4,128,1,float16,float16,15,0.06254720091819763
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,4,128,1,float16,fp8,16383,0.05552639961242676
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,4,128,1,float16,fp8,32767,0.09053599834442139
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,4,4,128,1,float16,float16,1,0.10892640352249146
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,4,4,128,1,float16,fp8,255,0.09528160095214844
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,4,4,128,1,float16,fp8,1,0.11005599498748779
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,4,4,128,1,float16,float16,3,0.10898879766464234
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,4,4,128,1,float16,fp8,3,0.1096384048461914
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,4,4,128,1,float16,float16,7,0.10997439622879028
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,4,4,128,1,float16,fp8,7,0.1090399980545044
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,4,4,128,1,float16,float16,15,0.1094048023223877
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,4,4,128,1,float16,fp8,15,0.11001280546188355
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,4,4,128,1,float16,fp8,31,0.10951679944992065
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,4,4,128,1,float16,float16,63,0.12078239917755126
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,4,4,128,1,float16,fp8,63,0.11838560104370117
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,4,4,128,1,float16,float16,127,0.12757760286331177
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,4,4,128,1,float16,fp8,127,0.14315680265426636
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,4,128,1,float16,fp8,15,0.009867200255393982
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,4,4,128,1,float16,float16,255,0.1811840057373047
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,4,4,128,1,float16,fp8,255,0.17396479845046997
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,4,4,128,1,float16,fp8,511,0.26208159923553465
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,4,128,1,float16,float16,2047,0.018724800646305086
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,4,4,128,1,float16,float16,1,0.20349280834197997
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,4,4,128,1,float16,float16,1023,0.5265151977539062
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,4,4,128,1,float16,fp8,1023,0.40055041313171386
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,4,4,128,1,float16,fp8,1,0.20269598960876464
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,4,4,128,1,float16,float16,3,0.20368640422821044
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,4,4,128,1,float16,fp8,3,0.20189120769500732
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,4,4,128,1,float16,fp8,7,0.20313599109649658
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,4,4,128,1,float16,fp8,15,0.2020143985748291
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,4,4,128,1,float16,float16,31,0.21515200138092042
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,4,128,1,float16,float16,32767,0.1330656051635742
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,4,4,128,1,float16,fp8,31,0.21149280071258544
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,4,4,128,1,float16,float16,63,0.2219615936279297
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,4,4,128,1,float16,fp8,63,0.22228798866271973
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,4,4,128,1,float16,float16,127,0.23457760810852052
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,4,4,128,1,float16,float16,31,0.11217759847640991
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,4,4,128,1,float16,fp8,127,0.2706768035888672
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,4,4,128,1,float16,float16,255,0.3410095930099487
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,4,4,128,1,float16,fp8,255,0.32930879592895507
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,4,128,1,float16,float16,1,0.010131199657917023
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,4,128,1,float16,fp8,63,0.009745600074529648
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,4,128,1,float16,fp8,1,0.01011200025677681
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,4,128,1,float16,float16,3,0.010251200199127198
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,4,128,1,float16,fp8,3,0.010100799798965453
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,4,4,128,1,float16,float16,511,0.571776008605957
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,4,4,128,1,float16,float16,511,0.29668478965759276
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,4,128,1,float16,float16,7,0.010199999809265137
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,4,128,1,float16,fp8,7,0.010086400061845779
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,4,128,1,float16,float16,15,0.009924799948930741
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,4,128,1,float16,fp8,15,0.01022080034017563
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,4,128,1,float16,fp8,31,0.010153599828481675
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,4,4,128,1,float16,fp8,511,0.49953598976135255
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,4,128,1,float16,float16,31,0.010121600329875946
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,4,128,1,float16,fp8,63,0.010390400141477584
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,4,128,1,float16,float16,127,0.010635200142860412
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,4,128,1,float16,float16,63,0.010265599936246872
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,4,128,1,float16,fp8,127,0.010903999954462052
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,4,128,1,float16,fp8,255,0.011822400242090225
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,4,128,1,float16,float16,255,0.011796800047159195
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,4,128,1,float16,fp8,511,0.01733600050210953
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,4,128,1,float16,float16,1023,0.017500799894332886
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,4,128,1,float16,fp8,1023,0.01764640063047409
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,4,128,1,float16,float16,2047,0.029436799883842468
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,4,128,1,float16,float16,4095,0.04482559859752655
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,4,128,1,float16,fp8,4095,0.03282879889011383
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,4,128,1,float16,float16,8191,0.07457280158996582
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,4,128,1,float16,fp8,8191,0.052367997169494626
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,4,4,128,1,float16,float16,7,0.2020656108856201
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,4,4,128,1,float16,float16,15,0.20436320304870606
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,4,128,1,float16,float16,16383,0.13361120223999023
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,4,128,1,float16,fp8,16383,0.08944000005722046
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,1,0.01125440001487732
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,3,0.011247999966144562
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,7,0.011230400204658509
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,4,128,1,float16,fp8,32767,0.1642799973487854
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,2,2,128,1,float16,fp8,1,0.011371199786663056
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,2,2,128,1,float16,fp8,15,0.01117279976606369
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,2,2,128,1,float16,fp8,7,0.01120000034570694
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,15,0.011259199678897857
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,2,2,128,1,float16,fp8,31,0.011313600093126297
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,63,0.011220800131559372
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,2,2,128,1,float16,fp8,63,0.011351999640464783
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,127,0.011355199664831162
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,2,2,128,1,float16,fp8,127,0.01241919994354248
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,255,0.012860800325870513
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,2,2,128,1,float16,fp8,255,0.013055999577045441
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,511,0.015003199875354766
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,2,2,128,1,float16,fp8,511,0.016711999475955964
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,1023,0.028352001309394838
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,2,2,128,1,float16,fp8,1023,0.02118239998817444
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,2047,0.04332000017166138
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,4095,0.07389280200004578
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,2,2,128,1,float16,fp8,4095,0.05108000040054321
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,8191,0.13276000022888185
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,2,2,128,1,float16,fp8,8191,0.08951680064201355
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,4,128,1,float16,float16,511,0.014148800075054169
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,4,128,1,float16,fp8,2047,0.024486400187015533
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,16383,0.25071680545806885
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,2,2,128,1,float16,fp8,16383,0.16550079584121705
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,4,128,1,float16,float16,32767,0.25183520317077634
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,32767,0.48786239624023436
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,2,2,128,1,float16,fp8,32767,0.3152656078338623
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,31,0.011190400272607804
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,2,2,128,1,float16,fp8,1,0.00928800031542778
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,2,2,128,1,float16,fp8,3,0.009302400052547455
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,3,0.009726399928331375
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,7,0.00950239971280098
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,2,2,128,1,float16,fp8,7,0.009356799721717834
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,15,0.00957759991288185
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,31,0.00957759991288185
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,2,2,128,1,float16,fp8,31,0.009270399808883667
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,63,0.009726399928331375
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,2,2,128,1,float16,fp8,2047,0.032462400197982785
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,2,2,128,1,float16,fp8,63,0.009406399726867676
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,65535,0.9577648162841796
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,127,0.009772799909114838
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,2,2,128,1,float16,fp8,255,0.010915199667215348
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,255,0.01091040000319481
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,511,0.0130048006772995
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,1023,0.014323200285434722
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,2,2,128,1,float16,fp8,511,0.013067199289798737
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,2,2,128,1,float16,fp8,1023,0.013814400136470794
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,2047,0.014460800588130951
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,4095,0.014684799313545226
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,2,2,128,1,float16,fp8,4095,0.015945599973201753
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,8191,0.01674399971961975
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,2,2,128,1,float16,fp8,8191,0.015884800255298613
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,16383,0.01809920072555542
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,2,2,128,1,float16,fp8,16383,0.0175135999917984
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,32767,0.020399999618530274
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,2,2,128,1,float16,fp8,32767,0.020233599841594695
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,65535,0.032201600074768064
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,2,2,128,1,float16,fp8,65535,0.025510400533676147
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,1,0.009665600210428237
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,1,0.009491200000047684
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,3,0.009460800141096116
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,2,2,128,1,float16,fp8,3,0.009281600266695023
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,2,2,128,1,float16,fp8,1,0.00928959995508194
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,7,0.009347199648618697
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,2,2,128,1,float16,fp8,7,0.009187199920415879
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,2,2,128,1,float16,fp8,15,0.00939520001411438
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,15,0.009374400228261947
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,2,2,128,1,float16,fp8,15,0.009177599847316743
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,31,0.009643200039863586
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,63,0.00945120006799698
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,2,2,128,1,float16,fp8,63,0.009379199892282485
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,127,0.009384000301361084
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,2,2,128,1,float16,fp8,127,0.009966400265693665
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,255,0.010831999778747558
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,2,2,128,1,float16,fp8,65535,0.6280511856079102
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,511,0.012990400195121765
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,1023,0.014681600034236908
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,2,2,128,1,float16,fp8,127,0.010126399993896484
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,2,2,128,1,float16,fp8,1023,0.014110399782657624
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,2047,0.014771200716495514
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,4095,0.014923200011253357
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,2,2,128,1,float16,fp8,4095,0.01574240028858185
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,8191,0.017798399925231932
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,2,2,128,1,float16,fp8,8191,0.01679359972476959
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,16383,0.019875200092792512
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,2,2,128,1,float16,fp8,2047,0.01408800035715103
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,2,2,128,1,float16,fp8,16383,0.019403199851512908
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,32767,0.03127999901771546
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,2,2,128,1,float16,fp8,32767,0.02513439953327179
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,65535,0.04786880016326904
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,1,0.010728000104427338
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,2,2,128,1,float16,fp8,65535,0.035820800065994265
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,2,2,128,1,float16,fp8,1,0.010582400113344192
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,2,2,128,1,float16,fp8,3,0.010608000308275222
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,3,0.010632000118494033
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,7,0.010758399963378906
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,15,0.01072319969534874
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,2,2,128,1,float16,fp8,15,0.010574399679899215
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,31,0.010657600313425063
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,2,2,128,1,float16,fp8,3,0.011296000331640244
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,2,2,128,1,float16,fp8,31,0.01061599999666214
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,63,0.0106175996363163
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,2,2,128,1,float16,fp8,63,0.010681600123643876
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,127,0.010967999696731567
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,2,2,128,1,float16,fp8,127,0.011385600268840789
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,255,0.01225920021533966
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,2,2,128,1,float16,fp8,255,0.01207360029220581
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,511,0.014425599575042724
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,2,2,128,1,float16,fp8,511,0.02027200013399124
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,1023,0.018423999845981597
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,2,2,128,1,float16,fp8,1023,0.01823520064353943
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,2047,0.030260801315307617
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,2,2,128,1,float16,fp8,2047,0.027484801411628724
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,2,2,128,1,float16,fp8,31,0.009415999799966813
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,4095,0.045316800475120544
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,2,2,128,1,float16,fp8,4095,0.035739201307296756
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,2,2,128,1,float16,fp8,8191,0.055396801233291625
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,8191,0.07686399817466735
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,16383,0.13577120304107665
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,2,2,128,1,float16,fp8,511,0.013406400382518769
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,2,2,128,1,float16,fp8,2047,0.014095999300479889
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,32767,0.25652639865875243
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,2,2,128,1,float16,fp8,32767,0.16966400146484376
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,1,0.009723199903964997
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,3,0.00952640026807785
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,2,2,128,1,float16,fp8,1,0.0095551997423172
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,2,2,128,1,float16,fp8,3,0.009390400350093841
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,65535,0.49572000503540037
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,7,0.009667199850082398
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,2,2,128,1,float16,fp8,7,0.00939679965376854
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,2,2,128,1,float16,fp8,65535,0.3197839975357056
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,2,2,128,1,float16,fp8,15,0.00947680026292801
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,31,0.009609600156545639
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,15,0.00968960002064705
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,63,0.009732799977064133
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,2,2,128,1,float16,fp8,63,0.009545599669218063
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,2,2,128,1,float16,fp8,127,0.010230399668216705
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,127,0.009550400078296661
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,2,2,128,1,float16,fp8,31,0.009385599941015243
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,2,2,128,1,float16,fp8,255,0.01117279976606369
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,511,0.012865599989891053
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,2,2,128,1,float16,fp8,511,0.013393600285053254
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,1023,0.01478559970855713
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,2,2,128,1,float16,fp8,1023,0.014025600254535675
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,2047,0.015263999998569488
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,2,2,128,1,float16,fp8,2047,0.014553600549697876
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,4095,0.016335999965667723
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,2,2,128,1,float16,fp8,4095,0.016942399740219116
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,8191,0.019985599815845488
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,2,2,128,1,float16,fp8,8191,0.019462400674819948
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,16383,0.030630400776863097
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,2,2,128,1,float16,fp8,16383,0.024831999838352204
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,32767,0.04703679978847504
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,2,2,128,1,float16,fp8,32767,0.03571679890155792
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,65535,0.07789120078086853
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,2,2,128,1,float16,float16,1,0.015404799580574035
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,2,2,128,1,float16,fp8,65535,0.056358402967453
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,2,2,128,1,float16,fp8,1,0.01576319932937622
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,2,2,128,1,float16,float16,3,0.015401600301265717
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,2,2,128,1,float16,fp8,3,0.015467199683189391
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,2,2,128,1,float16,float16,7,0.015385599434375763
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,2,2,128,1,float16,fp8,7,0.015255999565124512
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,2,2,128,1,float16,float16,15,0.015699200332164764
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,2,2,128,1,float16,fp8,15,0.015430399775505066
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,2,2,128,1,float16,float16,31,0.01557759940624237
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,2,2,128,1,float16,fp8,31,0.015600000321865082
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,2,2,128,1,float16,float16,63,0.015667200088500977
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,2,2,128,1,float16,fp8,63,0.015328000485897064
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,2,2,128,1,float16,float16,127,0.015985600650310516
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,2,2,128,1,float16,fp8,127,0.017427200078964235
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,2,2,128,1,float16,float16,255,0.01802079975605011
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,2,2,128,1,float16,fp8,255,0.01897760033607483
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,2,2,128,1,float16,float16,511,0.030859199166297913
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,2,2,128,1,float16,fp8,511,0.025305598974227905
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,2,2,128,1,float16,float16,1023,0.04746879935264588
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,2,2,128,1,float16,fp8,1023,0.03634720146656036
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,2,2,128,1,float16,float16,2047,0.07763360142707824
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,2,2,128,1,float16,fp8,2047,0.05598719716072083
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,2,2,128,1,float16,float16,4095,0.1358464002609253
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,2,2,128,1,float16,fp8,255,0.01101439967751503
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,2,2,128,1,float16,fp8,4095,0.09277439713478089
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,2,2,128,1,float16,float16,8191,0.2532448053359985
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,2,2,128,1,float16,fp8,16383,0.09286879897117614
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,2,2,128,1,float16,fp8,8191,0.17086880207061766
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,2,2,128,1,float16,float16,1,0.023548799753189086
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,2,2,128,1,float16,fp8,1,0.023423999547958374
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,2,2,128,1,float16,fp8,16383,0.31982879638671874
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,2,2,128,1,float16,float16,16383,0.4875904083251953
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,2,2,128,1,float16,float16,3,0.02348800003528595
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,2,2,128,1,float16,fp8,7,0.023452800512313843
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,2,2,128,1,float16,fp8,3,0.02343039959669113
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,2,2,128,1,float16,float16,15,0.023790399730205535
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,2,2,128,1,float16,fp8,15,0.023472000658512116
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,2,2,128,1,float16,float16,31,0.023841600120067596
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,2,2,128,1,float16,fp8,31,0.023472000658512116
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,2,2,128,1,float16,float16,63,0.023907199501991272
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,2,2,128,1,float16,fp8,63,0.02354719936847687
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,2,2,128,1,float16,float16,127,0.02399359941482544
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,2,2,128,1,float16,fp8,127,0.02678079903125763
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,2,2,128,1,float16,float16,255,0.037238401174545285
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,2,2,128,1,float16,float16,32767,0.9642928123474122
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,2,2,128,1,float16,fp8,255,0.03091840147972107
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,2,2,128,1,float16,float16,511,0.051744002103805545
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,2,2,128,1,float16,fp8,511,0.046086400747299194
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,2,2,128,1,float16,float16,1023,0.08342559933662415
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,2,2,128,1,float16,fp8,1023,0.06352639794349671
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,2,2,128,1,float16,fp8,2047,0.10178719758987427
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,2,2,128,1,float16,float16,2047,0.15624159574508667
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,2,2,128,1,float16,float16,4095,0.25695199966430665
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,2,2,128,1,float16,fp8,4095,0.17599200010299682
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,2,2,128,1,float16,float16,1,0.040217599272727965
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,2,2,128,1,float16,float16,8191,0.4886447906494141
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,2,2,128,1,float16,fp8,1,0.04023680090904236
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,2,2,128,1,float16,fp8,8191,0.33054399490356445
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,2,2,128,1,float16,fp8,3,0.04019519984722138
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,2,2,128,1,float16,float16,3,0.04063200056552887
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,2,2,128,1,float16,fp8,7,0.040212801098823546
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,2,2,128,1,float16,float16,15,0.040731200575828554
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,2,2,128,1,float16,fp8,15,0.03991200029850006
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,2,2,128,1,float16,float16,31,0.04030719995498657
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,2,2,128,1,float16,fp8,31,0.04019039869308472
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,2,2,128,1,float16,float16,63,0.0407696008682251
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,2,2,128,1,float16,fp8,63,0.04010879993438721
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,2,2,128,1,float16,float16,127,0.047393599152565004
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,2,2,128,1,float16,fp8,127,0.04717600047588348
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,2,2,128,1,float16,float16,255,0.06405439972877502
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,2,2,128,1,float16,fp8,255,0.05729759931564331
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,2,2,128,1,float16,float16,511,0.09185119867324829
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,2,2,128,1,float16,fp8,511,0.08122400045394898
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,2,2,128,1,float16,float16,7,0.023367999494075774
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,2,2,128,1,float16,float16,1023,0.15204319953918458
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,2,2,128,1,float16,fp8,1023,0.11688640117645263
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,2,2,128,1,float16,fp8,32767,0.6213840007781982
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,255,0.010971199721097946
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,2,2,128,1,float16,float16,2047,0.2685136079788208
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,1,0.010012800246477127
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,2,2,128,1,float16,fp8,2047,0.19530880451202393
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,3,0.010014399886131287
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,2,2,128,1,float16,fp8,1,0.009672000259160995
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,7,0.00992320030927658
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,2,2,128,1,float16,fp8,7,0.00955199971795082
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,2,2,128,1,float16,fp8,7,0.010678400099277497
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,15,0.009808000177145004
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,2,2,128,1,float16,fp8,15,0.00955199971795082
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,2,2,128,1,float16,float16,4095,0.49555201530456544
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,2,2,128,1,float16,fp8,31,0.00968480035662651
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,31,0.009911999851465226
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,127,0.009646400064229965
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,2,2,128,1,float16,fp8,63,0.00944959968328476
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,255,0.011505600064992905
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,2,2,128,1,float16,fp8,255,0.011393599957227708
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,511,0.013289600610733032
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,2,2,128,1,float16,fp8,511,0.013875199854373932
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,1023,0.015503999590873719
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,2,2,128,1,float16,fp8,1023,0.014556799829006196
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,2047,0.01653600037097931
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,2,2,128,1,float16,fp8,2047,0.015300799906253815
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,4095,0.01863519996404648
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,2,2,128,1,float16,float16,7,0.04023680090904236
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,8191,0.03138079941272735
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,2,2,128,1,float16,fp8,8191,0.024134400486946105
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,16383,0.046481600403785704
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,2,2,128,1,float16,fp8,16383,0.03543519973754883
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,32767,0.07621600031852722
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,2,2,128,1,float16,fp8,32767,0.05464640259742737
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,2,2,128,1,float16,fp8,65535,0.09266719818115235
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,65535,0.13728480339050292
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,2,2,128,1,float16,float16,1,0.0683791995048523
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,2,2,128,1,float16,fp8,1,0.0684495985507965
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,2,2,128,1,float16,float16,3,0.06792640089988708
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,2,2,128,1,float16,fp8,3,0.06843680143356323
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,2,2,128,1,float16,float16,7,0.06761760115623475
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,2,2,128,1,float16,fp8,7,0.06854720115661621
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,2,2,128,1,float16,float16,15,0.06799359917640686
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,2,2,128,1,float16,fp8,15,0.06860479712486267
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,2,2,128,1,float16,float16,31,0.06807839870452881
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,2,2,128,1,float16,fp8,31,0.06838399767875672
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,2,2,128,1,float16,fp8,3,0.0095551997423172
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,2,2,128,1,float16,float16,63,0.07456160187721253
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,2,2,128,1,float16,fp8,63,0.06878880262374878
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,63,0.009729599952697754
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,2,2,128,1,float16,float16,127,0.08027679920196533
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,2,2,128,1,float16,fp8,4095,0.34325759410858153
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,2,2,128,1,float16,float16,255,0.10918079614639283
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,2,2,128,1,float16,fp8,255,0.102401602268219
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,2,2,128,1,float16,float16,511,0.1641983985900879
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,2,2,128,1,float16,fp8,511,0.14778399467468262
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,2,2,128,1,float16,float16,1023,0.2830591917037964
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,2,2,128,1,float16,fp8,1023,0.21864640712738037
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,2,2,128,1,float16,fp8,4095,0.01958400011062622
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,2,2,128,1,float16,float16,1,0.11961120367050171
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,2,2,128,1,float16,fp8,2047,0.3755264043807983
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,2,2,128,1,float16,float16,2047,0.5106319904327392
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,2,2,128,1,float16,float16,3,0.11891839504241944
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,2,2,128,1,float16,float16,7,0.11939359903335571
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,2,2,128,1,float16,float16,15,0.11927200555801391
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,2,2,128,1,float16,fp8,7,0.121288001537323
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,2,2,128,1,float16,fp8,15,0.12064640522003174
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,2,2,128,1,float16,float16,31,0.12153760194778443
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,2,2,128,1,float16,fp8,31,0.12149920463562011
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,2,2,128,1,float16,float16,63,0.13214399814605712
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,2,2,128,1,float16,fp8,63,0.13076159954071045
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,2,2,128,1,float16,float16,127,0.1411072015762329
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,2,2,128,1,float16,fp8,127,0.1571280002593994
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,2,2,128,1,float16,float16,255,0.19232159852981567
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,2,2,128,1,float16,fp8,255,0.18963520526885985
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,2,2,128,1,float16,float16,511,0.3012943983078003
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,2,2,128,1,float16,fp8,511,0.2758336067199707
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,1,0.009913600236177444
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,2,2,128,1,float16,fp8,1,0.009862399846315383
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,3,0.009889599680900574
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,2,2,128,1,float16,float16,1023,0.5435023784637452
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,2,2,128,1,float16,fp8,127,0.08702560067176819
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,2,2,128,1,float16,fp8,3,0.009828799962997436
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,2,2,128,1,float16,fp8,1023,0.4176447868347168
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,2,2,128,1,float16,fp8,15,0.00986879989504814
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,2,2,128,1,float16,fp8,7,0.009942399710416794
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,7,0.010232000052928925
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,2,2,128,1,float16,fp8,31,0.009769599884748459
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,63,0.010203199833631516
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,31,0.00979200005531311
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,2,2,128,1,float16,fp8,63,0.009785600006580353
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,2,2,128,1,float16,fp8,127,0.011161600053310395
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,255,0.011473599821329117
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,2,2,128,1,float16,fp8,255,0.01165279969573021
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,511,0.013595199584960938
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,2,2,128,1,float16,fp8,511,0.014803199470043183
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,1023,0.016356800496578217
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,2,2,128,1,float16,fp8,1023,0.015112000703811645
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,2047,0.01759999990463257
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,2,2,128,1,float16,fp8,2047,0.017983999848365784
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,4095,0.029504001140594482
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,2,2,128,1,float16,fp8,4095,0.023899200558662414
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,8191,0.046623998880386354
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,2,2,128,1,float16,fp8,8191,0.035124799609184264
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,16383,0.07633600234985352
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,2,2,128,1,float16,fp8,1,0.12032320499420165
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,2,2,128,1,float16,fp8,16383,0.054203200340271
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,32767,0.13689759969711304
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,2,2,128,1,float16,fp8,32767,0.09228799939155578
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,2,2,128,1,float16,fp8,3,0.12079360485076904
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,1,0.01156959980726242
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,3,0.010913600027561188
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,2,2,128,1,float16,fp8,65535,0.16911040544509887
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,65535,0.2557615995407104
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,1,1,128,1,float16,fp8,1,0.011273600161075592
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,1,1,128,1,float16,fp8,15,0.011339200288057327
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,15,0.011472000181674958
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,31,0.011270400136709213
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,63,0.011286400258541107
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,1,1,128,1,float16,fp8,31,0.011667200177907944
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,1,1,128,1,float16,fp8,63,0.011360000073909759
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,127,0.011643200367689132
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,1,1,128,1,float16,fp8,127,0.012217599898576736
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,255,0.013044799864292144
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,1,1,128,1,float16,fp8,255,0.012932799756526947
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,511,0.014932799339294433
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,1,1,128,1,float16,fp8,511,0.021641600131988525
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,1023,0.018913599848747253
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,1,1,128,1,float16,fp8,1023,0.020099200308322906
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,2047,0.030695998668670656
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,1,1,128,1,float16,fp8,2047,0.028832000494003297
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,4095,0.04584319889545441
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,2,2,128,1,float16,fp8,127,0.01024319976568222
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,1,1,128,1,float16,fp8,4095,0.0369264006614685
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,15,0.010284800082445145
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,1,1,128,1,float16,fp8,8191,0.05937280058860779
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,127,0.00997920036315918
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,16383,0.13986079692840575
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,1,1,128,1,float16,fp8,16383,0.09757120013237
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,32767,0.2581968069076538
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,1,1,128,1,float16,fp8,32767,0.17336159944534302
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,65535,0.5000271797180176
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,1,1,128,1,float16,fp8,65535,0.32387681007385255
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,1,1,128,1,float16,fp8,3,0.011598400026559829
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,1,1,128,1,float16,fp8,1,0.00952960029244423
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,1,0.009332799911499023
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,3,0.009547200053930283
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,1,1,128,1,float16,fp8,7,0.010980799794197083
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,1,1,128,1,float16,fp8,3,0.0094480000436306
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,7,0.009585600346326828
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,1,1,128,1,float16,fp8,7,0.009415999799966813
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,15,0.009593600034713745
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,31,0.0095551997423172
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,1,1,128,1,float16,fp8,31,0.009393599629402161
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,63,0.009699200093746186
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,131071,0.9784399986267089
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,127,0.00968799963593483
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,1,1,128,1,float16,fp8,131071,0.6241439819335938
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,255,0.010679999738931656
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,511,0.012668800354003907
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,1,1,128,1,float16,fp8,511,0.012804800271987915
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,1,1,128,1,float16,fp8,255,0.010843200236558914
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,1023,0.014051200449466705
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,1,1,128,1,float16,fp8,1023,0.01340319961309433
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,1,1,128,1,float16,fp8,2047,0.01377280056476593
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,4095,0.014572800695896148
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,1,1,128,1,float16,fp8,4095,0.015441599488258361
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,8191,0.07773119807243348
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,1,1,128,1,float16,fp8,8191,0.01924320012331009
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,16383,0.02003840059041977
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,1,1,128,1,float16,fp8,16383,0.019519999623298645
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,32767,0.021011200547218323
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,1,1,128,1,float16,fp8,32767,0.020854400098323823
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,65535,0.02407840043306351
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,1,1,128,1,float16,fp8,65535,0.02364159971475601
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,131071,0.034720000624656674
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,1,1,128,1,float16,fp8,131071,0.029782399535179138
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,1,0.00950239971280098
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,1,1,128,1,float16,fp8,1,0.009244800359010697
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,1,1,128,1,float16,fp8,15,0.009495999664068222
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,3,0.009340800344944
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,1,1,128,1,float16,fp8,3,0.009379199892282485
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,7,0.00963200032711029
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,1,1,128,1,float16,fp8,7,0.009332799911499023
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,1,1,128,1,float16,fp8,63,0.009414400160312652
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,15,0.009574399888515472
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,31,0.009723199903964997
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,1,1,128,1,float16,fp8,31,0.009324800223112106
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,63,0.00963200032711029
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,127,0.00963039994239807
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,1,1,128,1,float16,fp8,127,0.010052800178527832
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,255,0.010889600217342376
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,1,1,128,1,float16,fp8,255,0.010836800187826156
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,1,1,128,1,float16,fp8,127,0.010214400291442872
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,511,0.013084800541400909
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,1,1,128,1,float16,fp8,511,0.012908799946308136
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,1023,0.014382399618625641
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,2047,0.01435839980840683
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,1,1,128,1,float16,fp8,2047,0.014000000059604644
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,4095,0.014771200716495514
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,1,1,128,1,float16,fp8,4095,0.015809600055217744
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,8191,0.01687680035829544
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,1,1,128,1,float16,fp8,8191,0.019054399430751802
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,16383,0.021244800090789794
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,1,1,128,1,float16,fp8,16383,0.020043200254440306
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,32767,0.023452800512313843
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,1,1,128,1,float16,fp8,32767,0.02260479927062988
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,8191,0.01637279987335205
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,65535,0.03430559933185577
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,1,1,128,1,float16,fp8,65535,0.02792479991912842
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,1,0.010735999792814255
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,1,1,128,1,float16,fp8,131071,0.038710400462150574
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,1,1,128,1,float16,fp8,1,0.010609599947929382
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,3,0.010491199791431427
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,1,1,128,1,float16,fp8,3,0.01048479974269867
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,1,1,128,1,float16,fp8,7,0.010463999956846238
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,15,0.010476800054311753
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,1,1,128,1,float16,fp8,15,0.010254400223493576
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,31,0.010446400195360184
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,1,1,128,1,float16,fp8,31,0.01040000021457672
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,63,0.010728000104427338
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,1,1,128,1,float16,fp8,63,0.010651200264692306
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,7,0.011060799658298492
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,127,0.010547199845314026
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,1,1,128,1,float16,fp8,127,0.011401599645614624
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,255,0.01226240023970604
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,1,1,128,1,float16,fp8,255,0.012256000190973282
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,511,0.014470399916172027
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,1,1,128,1,float16,fp8,15,0.009596800059080124
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,1,1,128,1,float16,fp8,511,0.015455999970436096
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,1,1,128,1,float16,fp8,63,0.00950080007314682
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,1023,0.016891199350357055
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,2047,0.018374399840831758
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,4095,0.030435198545455934
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,1,1,128,1,float16,fp8,1023,0.013916799426078796
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,1,1,128,1,float16,fp8,4095,0.024878400564193725
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,2047,0.014268800616264343
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,8191,0.046670401096343996
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,1,1,128,1,float16,fp8,8191,0.03887200057506561
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,1,1,128,1,float16,fp8,16383,0.05780320167541504
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,32767,0.14103360176086427
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,1,1,128,1,float16,fp8,32767,0.09602720141410828
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,131071,0.05145440101623535
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,65535,0.2608911991119385
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,1,1,128,1,float16,fp8,65535,0.171670401096344
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,7,0.01040000021457672
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,1,0.009550400078296661
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,3,0.009486400336027146
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,1,1,128,1,float16,fp8,3,0.00942559987306595
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,131071,0.49434719085693357
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,1,1,128,1,float16,fp8,131071,0.3243632078170776
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,1,1,128,1,float16,fp8,7,0.009510400146245957
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,15,0.009662400186061858
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,1,1,128,1,float16,fp8,15,0.009483200311660767
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,1,1,128,1,float16,fp8,31,0.009438399970531464
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,63,0.009510400146245957
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,1,1,128,1,float16,fp8,63,0.009294400364160538
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,127,0.009284800291061402
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,1,1,128,1,float16,fp8,127,0.009905599802732468
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,255,0.01061599999666214
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,1,1,128,1,float16,fp8,255,0.010969600081443787
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,511,0.012955200672149659
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,1,1,128,1,float16,fp8,2047,0.018639999628067016
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,1,1,128,1,float16,fp8,511,0.013417600095272065
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,1023,0.014604799449443817
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,1,1,128,1,float16,fp8,1023,0.013900800049304963
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,2047,0.014584000408649444
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,4095,0.014931200444698334
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,1,1,128,1,float16,fp8,4095,0.015659199655056
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,16383,0.08055040240287781
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,8191,0.017632000148296356
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,1,1,128,1,float16,fp8,8191,0.020139199495315552
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,16383,0.02343679964542389
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,1,1,128,1,float16,fp8,16383,0.02279680073261261
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,32767,0.03469119966030121
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,1,1,128,1,float16,fp8,32767,0.02815839946269989
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,65535,0.05130559802055359
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,131071,0.08153280019760131
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,1,1,128,1,float16,fp8,1,0.009302400052547455
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,1,0.012329600006341934
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,1,1,128,1,float16,fp8,1,0.012329600006341934
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,3,0.012401600182056428
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,7,0.009566400200128555
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,1,1,128,1,float16,fp8,3,0.012329600006341934
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,7,0.012585599720478059
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,15,0.01266079992055893
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,1,1,128,1,float16,fp8,15,0.012574400007724761
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,31,0.012787200510501862
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,31,0.009628800302743911
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,1,1,128,1,float16,fp8,31,0.01255040019750595
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,63,0.013265599310398103
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,1,1,128,1,float16,fp8,63,0.012585599720478059
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,127,0.013097600638866424
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,1,1,128,1,float16,fp8,127,0.01356479972600937
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,255,0.014931200444698334
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,1,1,128,1,float16,fp8,255,0.014435200393199921
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,511,0.016891199350357055
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,1,1,128,1,float16,fp8,511,0.01838400065898895
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,1023,0.03057279884815216
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,1,1,128,1,float16,fp8,1023,0.022856000065803527
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,2047,0.04524799883365631
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,1,1,128,1,float16,fp8,2047,0.03394080102443695
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,4095,0.07537279725074768
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,1,1,128,1,float16,fp8,2047,0.013942399621009826
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,8191,0.13381439447402954
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,1,1,128,1,float16,fp8,8191,0.09127680063247681
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,1,1,128,1,float16,fp8,16383,0.16702879667282106
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,16383,0.2552031993865967
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,1,1,128,1,float16,fp8,65535,0.03912000060081482
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,1,1,128,1,float16,fp8,131071,0.05952479839324951
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,1,1,128,1,float16,fp8,32767,0.3189888000488281
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,1,1,128,1,float16,float16,1,0.017873600125312805
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,32767,0.4884672164916992
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,1,1,128,1,float16,float16,3,0.017790399491786957
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,1,1,128,1,float16,fp8,1,0.01765599995851517
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,1,1,128,1,float16,float16,7,0.01764959990978241
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,1,1,128,1,float16,fp8,7,0.01769919991493225
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,1,1,128,1,float16,fp8,7,0.012323199957609176
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,1,1,128,1,float16,fp8,15,0.017851200699806214
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,1,1,128,1,float16,float16,31,0.018116800487041472
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,1,1,128,1,float16,fp8,31,0.017820799350738527
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,1,1,128,1,float16,float16,63,0.01785759925842285
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,1,1,128,1,float16,fp8,63,0.017983999848365784
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,1,1,128,1,float16,fp8,65535,0.6214335918426513
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,1,1,128,1,float16,float16,127,0.01803999990224838
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,65535,0.9607952117919922
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,1,1,128,1,float16,float16,255,0.020734399557113647
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,1,1,128,1,float16,fp8,255,0.021294400095939636
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,1,1,128,1,float16,fp8,1023,0.01704320013523102
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,1,1,128,1,float16,fp8,511,0.028169599175453187
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,1,1,128,1,float16,float16,1023,0.050251197814941403
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,1,1,128,1,float16,fp8,1023,0.039094400405883786
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,1,1,128,1,float16,fp8,4095,0.05280640125274658
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,1,1,128,1,float16,float16,2047,0.08009759783744812
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,1,1,128,1,float16,fp8,2047,0.05907679796218872
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,1,1,128,1,float16,fp8,4095,0.09589920043945313
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,1,1,128,1,float16,float16,8191,0.2557519912719727
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,1,1,128,1,float16,fp8,8191,0.17238880395889283
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,1,1,128,1,float16,fp8,16383,0.321616005897522
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,1,1,128,1,float16,float16,16383,0.4932447910308838
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,1,1,128,1,float16,float16,1,0.02937760055065155
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,1,1,128,1,float16,fp8,3,0.01775359958410263
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,1,1,128,1,float16,float16,15,0.01775840073823929
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,1,1,128,1,float16,float16,3,0.028964799642562867
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,1,1,128,1,float16,fp8,3,0.02938399910926819
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,1,1,128,1,float16,float16,15,0.02937600016593933
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,1,1,128,1,float16,fp8,15,0.029380801320075988
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,1,1,128,1,float16,float16,31,0.029393601417541503
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,1,1,128,1,float16,fp8,31,0.029315200448036195
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,1,1,128,1,float16,float16,63,0.029598399996757507
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,1,1,128,1,float16,float16,511,0.03294560015201568
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,1,1,128,1,float16,fp8,63,0.029531198740005492
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,1,1,128,1,float16,float16,127,0.029417601227760316
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,1,1,128,1,float16,fp8,127,0.03283199965953827
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,1,1,128,1,float16,fp8,255,0.03692159950733185
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,1,1,128,1,float16,float16,511,0.05633599758148193
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,1,1,128,1,float16,fp8,511,0.05124959945678711
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,1,1,128,1,float16,float16,1023,0.08902559876441955
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,1,1,128,1,float16,float16,4095,0.1381168007850647
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,1,1,128,1,float16,fp8,1023,0.0698207974433899
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,1,1,128,1,float16,float16,2047,0.14768799543380737
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,1,1,128,1,float16,fp8,2047,0.10866400003433227
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,1,1,128,1,float16,fp8,4095,0.18327200412750244
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,1,1,128,1,float16,float16,4095,0.26325440406799316
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,1,1,128,1,float16,fp8,8191,0.3336800098419189
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,1,1,128,1,float16,fp8,1,0.029495999217033386
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,1,1,128,1,float16,float16,8191,0.495911979675293
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,1,1,128,1,float16,float16,7,0.028998398780822755
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,1,0.010175999999046326
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,1,1,128,1,float16,fp8,1,0.00992320030927658
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,1,1,128,1,float16,fp8,3,0.009721600264310837
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,3,0.010025600343942643
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,1,1,128,1,float16,fp8,7,0.009827200323343277
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,15,0.009811200201511383
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,1,1,128,1,float16,fp8,31,0.009833600372076035
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,31,0.009950400143861771
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,63,0.009801600128412247
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,1,1,128,1,float16,fp8,63,0.009737599641084671
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,127,0.009929600358009338
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,255,0.011068800091743469
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,1,1,128,1,float16,fp8,255,0.011308799684047698
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,511,0.01342719942331314
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,1,1,128,1,float16,fp8,511,0.013784000277519226
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,1023,0.015035200119018554
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,1,1,128,1,float16,fp8,1023,0.014283199608325959
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,1,1,128,1,float16,float16,255,0.0418368011713028
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,1,1,128,1,float16,fp8,2047,0.014723199605941772
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,4095,0.016008000075817107
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,8191,0.020132799446582795
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,1,1,128,1,float16,fp8,8191,0.02282080054283142
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,16383,0.03375999927520752
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,1,1,128,1,float16,fp8,16383,0.028038400411605834
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,32767,0.05063520073890686
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,1,1,128,1,float16,fp8,32767,0.039124798774719236
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,65535,0.08051199913024902
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,1,1,128,1,float16,fp8,65535,0.060063999891281125
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,131071,0.14556000232696534
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,1,1,128,1,float16,fp8,131071,0.10057280063629151
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,1,1,128,1,float16,float16,1,0.046408000588417056
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,1,1,128,1,float16,fp8,1,0.04641439914703369
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,1,1,128,1,float16,float16,3,0.04632959961891174
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,1,1,128,1,float16,fp8,3,0.046428799629211426
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,7,0.010187199711799622
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,1,1,128,1,float16,float16,7,0.04657439887523651
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,1,1,128,1,float16,fp8,15,0.009814400225877762
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,1,1,128,1,float16,float16,15,0.04657599925994873
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,1,1,128,1,float16,fp8,7,0.046649599075317384
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,1,1,128,1,float16,fp8,15,0.04666079878807068
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,1,1,128,1,float16,float16,31,0.04697119891643524
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,1,1,128,1,float16,fp8,31,0.04643999934196472
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,1,1,128,1,float16,float16,63,0.047193598747253415
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,1,1,128,1,float16,fp8,63,0.04643999934196472
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,1,1,128,1,float16,float16,127,0.053014397621154785
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,1,1,128,1,float16,fp8,127,0.05355520248413086
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,2047,0.015364800393581391
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,1,1,128,1,float16,float16,511,0.09607359766960144
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,1,1,128,1,float16,fp8,511,0.0883679986000061
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,1,1,128,1,float16,float16,1023,0.1586959958076477
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,1,1,128,1,float16,fp8,1023,0.12367839813232422
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,1,1,128,1,float16,fp8,2047,0.2014240026473999
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,1,1,128,1,float16,float16,4095,0.5023583889007568
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,1,1,128,1,float16,fp8,7,0.02890399992465973
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,1,1,128,1,float16,fp8,4095,0.35100479125976564
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,1,1,128,1,float16,float16,1,0.07867519855499268
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,1,1,128,1,float16,fp8,1,0.07900000214576722
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,1,1,128,1,float16,fp8,127,0.019392000138759614
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,1,1,128,1,float16,float16,3,0.0780623972415924
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,1,1,128,1,float16,float16,7,0.0780784010887146
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,1,1,128,1,float16,fp8,3,0.07930399775505066
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,1,1,128,1,float16,fp8,7,0.07849439978599548
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,1,1,128,1,float16,float16,15,0.07898560166358948
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,1,1,128,1,float16,float16,255,0.06862239837646485
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,1,1,128,1,float16,fp8,15,0.07869120240211487
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,1,1,128,1,float16,fp8,4095,0.01706400066614151
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,1,1,128,1,float16,fp8,31,0.07888000011444092
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,1,1,128,1,float16,float16,63,0.08569759726524354
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,1,1,128,1,float16,fp8,63,0.0793615996837616
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,1,1,128,1,float16,float16,127,0.09147199988365173
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,1,1,128,1,float16,fp8,127,0.0978767991065979
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,1,1,128,1,float16,float16,255,0.11543200016021729
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,1,1,128,1,float16,float16,511,0.17145919799804688
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,1,1,128,1,float16,fp8,511,0.16149920225143433
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,1,1,128,1,float16,float16,2047,0.2737760066986084
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,1,1,128,1,float16,float16,1023,0.2977184057235718
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,1,1,128,1,float16,fp8,1023,0.2326672077178955
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,1,0.009601599723100662
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,1,1,128,1,float16,fp8,1,0.009539200365543366
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,3,0.009569600224494934
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,1,1,128,1,float16,float16,2047,0.5221615791320801
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,1,1,128,1,float16,fp8,3,0.009520000219345093
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,1,1,128,1,float16,fp8,7,0.00952799990773201
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,1,1,128,1,float16,fp8,2047,0.38818399906158446
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,15,0.009692800045013428
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,31,0.009769599884748459
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,1,1,128,1,float16,fp8,15,0.010060799866914749
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,63,0.009849599748849868
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,1,1,128,1,float16,fp8,63,0.009894400089979171
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,127,0.009728000313043595
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,1,1,128,1,float16,fp8,127,0.010292799770832061
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,1,1,128,1,float16,fp8,127,0.01024480015039444
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,255,0.01138719990849495
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,1,1,128,1,float16,fp8,255,0.011748799681663513
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,511,0.013291199505329133
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,1023,0.015199999511241912
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,1,1,128,1,float16,fp8,255,0.06427519917488098
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,1,1,128,1,float16,float16,31,0.07955679893493653
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,1,1,128,1,float16,fp8,2047,0.015467199683189391
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,4095,0.018692800402641298
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,1,1,128,1,float16,fp8,4095,0.01972319930791855
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,1,1,128,1,float16,fp8,255,0.11495200395584107
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,8191,0.03154079914093018
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,1,1,128,1,float16,fp8,8191,0.027272000908851624
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,16383,0.04918240010738373
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,1,1,128,1,float16,fp8,16383,0.038108798861503604
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,32767,0.079312002658844
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,1,1,128,1,float16,fp8,32767,0.05735679864883423
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,1,1,128,1,float16,fp8,65535,0.09578400254249572
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,65535,0.1383695960044861
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,1,1,128,1,float16,fp8,131071,0.17293280363082886
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,131071,0.2599600076675415
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,7,0.010134399682283402
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,1,128,1,float16,fp8,1,0.013784000277519226
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,1,128,1,float16,float16,3,0.01311199963092804
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,1,128,1,float16,fp8,3,0.013519999384880067
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,1,128,1,float16,float16,7,0.013105599582195282
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,1,128,1,float16,fp8,7,0.013524800539016724
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,1,128,1,float16,float16,15,0.013203200697898865
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,1,128,1,float16,fp8,15,0.013603200018405915
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,1,128,1,float16,float16,31,0.013316799700260163
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,1,128,1,float16,float16,63,0.013120000064373017
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,1,128,1,float16,fp8,31,0.013935999572277069
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,1,128,1,float16,float16,127,0.013550400733947754
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,1,128,1,float16,fp8,127,0.013916799426078796
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,1,128,1,float16,float16,255,0.01509920060634613
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,1,128,1,float16,fp8,255,0.015457600355148315
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,1,1,128,1,float16,fp8,511,0.014056000113487243
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,1,128,1,float16,fp8,511,0.018680000305175783
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,1,1,128,1,float16,fp8,1023,0.01465280055999756
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,1,128,1,float16,float16,1023,0.025934401154518127
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,2047,0.01666239947080612
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,1,128,1,float16,float16,2047,0.03754720091819763
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,2,128,1,float16,float16,1,0.011406400054693223
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,2,128,1,float16,float16,3,0.011819200217723846
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,2,128,1,float16,fp8,1,0.01122559979557991
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,2,128,1,float16,fp8,3,0.011363200098276138
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,2,128,1,float16,float16,7,0.011407999694347382
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,2,128,1,float16,fp8,7,0.01149120032787323
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,2,128,1,float16,float16,15,0.011287999898195266
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,2,128,1,float16,float16,31,0.011395200341939925
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,2,128,1,float16,fp8,31,0.01125119999051094
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,2,128,1,float16,float16,63,0.01170400008559227
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,2,128,1,float16,fp8,63,0.011337599903345107
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,2,128,1,float16,float16,127,0.01199679970741272
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,2,128,1,float16,fp8,127,0.012494400143623352
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,2,128,1,float16,float16,255,0.013273599743843078
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,2,128,1,float16,fp8,255,0.012887999415397644
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,2,128,1,float16,float16,511,0.01541759967803955
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,2,128,1,float16,fp8,511,0.016971200704574585
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,2,128,1,float16,float16,1023,0.02999520003795624
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,1,128,1,float16,float16,1,0.013782399892807006
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,2,128,1,float16,fp8,1023,0.02189279943704605
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,1,1,128,1,float16,fp8,31,0.009710399806499482
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,2,128,1,float16,float16,2047,0.04557600021362305
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,2,128,1,float16,fp8,2047,0.03338879942893982
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,4,128,1,float16,float16,1,0.014115199446678162
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,4,128,1,float16,fp8,1,0.014131200313568116
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,4,128,1,float16,float16,3,0.014284799993038177
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,4,128,1,float16,fp8,3,0.014188799262046813
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,4,128,1,float16,float16,7,0.01414559930562973
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,4,128,1,float16,fp8,7,0.01393280029296875
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,1,128,1,float16,fp8,63,0.013675199449062347
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,4,128,1,float16,fp8,15,0.014081600308418273
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,4,128,1,float16,float16,31,0.01449120044708252
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,4,128,1,float16,fp8,31,0.014044800400733947
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,1,128,1,float16,float16,511,0.018275199830532073
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,4,128,1,float16,float16,63,0.014553600549697876
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,4,128,1,float16,float16,127,0.014694400131702423
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,1,128,1,float16,fp8,1023,0.024401600658893585
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,4,128,1,float16,fp8,127,0.015727999806404113
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,1,128,1,float16,fp8,2047,0.030451199412345885
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,4,128,1,float16,fp8,255,0.01757120043039322
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,4,128,1,float16,float16,511,0.030713599920272828
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,4,128,1,float16,fp8,511,0.024391999840736388
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,4,128,1,float16,float16,1023,0.04747520089149475
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,4,128,1,float16,fp8,1023,0.03564479947090149
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,4,128,1,float16,fp8,2047,0.05541759729385376
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,4,128,1,float16,float16,2047,0.07769119739532471
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,8,128,1,float16,fp8,1,0.01977760046720505
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,8,128,1,float16,float16,3,0.019972799718379973
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,8,128,1,float16,fp8,3,0.019787199795246124
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,8,128,1,float16,float16,7,0.020227199792861937
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,8,128,1,float16,fp8,7,0.01976799964904785
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,8,128,1,float16,float16,15,0.02022079974412918
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,8,128,1,float16,fp8,15,0.01977120041847229
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,8,128,1,float16,float16,31,0.020105600357055664
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,8,128,1,float16,fp8,31,0.019556799530982973
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,8,128,1,float16,float16,63,0.020151999592781068
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,8,128,1,float16,fp8,63,0.019336000084877014
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,8,128,1,float16,float16,127,0.020948800444602966
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,8,128,1,float16,fp8,127,0.02282239943742752
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,8,128,1,float16,float16,255,0.03491039872169495
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,8,128,1,float16,fp8,255,0.02635360062122345
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,8,128,1,float16,float16,511,0.04901120066642761
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,8,128,1,float16,fp8,511,0.041448000073432925
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,8,128,1,float16,float16,1023,0.08166400194168091
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,8,128,1,float16,fp8,1023,0.05929440259933472
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,8,128,1,float16,fp8,2047,0.09890559911727906
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,1,128,1,float16,float16,1,0.011315199732780456
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,1,128,1,float16,fp8,1,0.01167680025100708
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,1,128,1,float16,float16,3,0.011224000155925751
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,1,128,1,float16,fp8,3,0.011606399714946748
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,1,128,1,float16,float16,7,0.011025600135326385
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,4,128,1,float16,float16,15,0.014102399349212646
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,1,128,1,float16,float16,15,0.011259199678897857
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,1,128,1,float16,fp8,15,0.011563199758529662
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,1,128,1,float16,float16,31,0.011240000277757645
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,4,128,1,float16,fp8,63,0.014248000085353851
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,1,128,1,float16,fp8,31,0.011596799641847611
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,1,128,1,float16,float16,63,0.01146399974822998
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,1,128,1,float16,float16,127,0.011534400284290314
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,1,128,1,float16,fp8,127,0.011655999720096588
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,1,128,1,float16,float16,255,0.01287200003862381
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,1,128,1,float16,fp8,255,0.013380800187587739
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,1,128,1,float16,float16,511,0.015772800147533416
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,8,128,1,float16,float16,1,0.019657599925994872
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,1,128,1,float16,fp8,511,0.016862399876117706
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,2,128,1,float16,fp8,15,0.011367999762296677
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,1,128,1,float16,fp8,1023,0.01644960045814514
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,1,128,1,float16,float16,2047,0.016260799765586854
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,2,128,1,float16,float16,1,0.009848000109195709
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,2,128,1,float16,fp8,1,0.009591999650001525
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,2,128,1,float16,float16,3,0.009708800166845322
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,2,128,1,float16,fp8,3,0.00942239984869957
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,2,128,1,float16,float16,7,0.009694399684667588
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,2,128,1,float16,fp8,7,0.009465599805116654
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,2,128,1,float16,float16,15,0.00974079966545105
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,2,128,1,float16,fp8,15,0.009521599858999252
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,2,128,1,float16,float16,31,0.009864000231027603
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,2,128,1,float16,fp8,31,0.009567999839782714
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,2,128,1,float16,float16,63,0.009857600182294845
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,2,128,1,float16,fp8,63,0.00960479974746704
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,8,128,1,float16,float16,2047,0.1407039999961853
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,2,128,1,float16,fp8,127,0.010369600355625152
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,2,128,1,float16,fp8,255,0.01096320003271103
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,2,128,1,float16,float16,255,0.011049599945545196
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,2,128,1,float16,fp8,511,0.012700800597667695
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,2,128,1,float16,float16,1023,0.01393440067768097
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,2,128,1,float16,fp8,1023,0.013065600395202636
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,1,128,1,float16,fp8,7,0.011507199704647064
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,2,128,1,float16,float16,2047,0.014020800590515137
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,2,128,1,float16,fp8,2047,0.01345600038766861
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,4,128,1,float16,float16,1,0.009532800316810608
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,4,128,1,float16,fp8,1,0.009403199702501298
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,4,128,1,float16,float16,3,0.009558399766683578
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,4,128,1,float16,fp8,3,0.00936639979481697
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,4,128,1,float16,float16,7,0.009651199728250504
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,4,128,1,float16,fp8,7,0.00926240012049675
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,4,128,1,float16,float16,15,0.009731200337409974
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,4,128,1,float16,fp8,15,0.009278400242328644
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,1,128,1,float16,fp8,63,0.011724799871444702
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,4,128,1,float16,float16,31,0.00968639999628067
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,4,128,1,float16,fp8,31,0.00926079973578453
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,128,4,128,1,float16,float16,255,0.01733600050210953
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,4,128,1,float16,float16,63,0.00960640013217926
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,4,128,1,float16,fp8,63,0.009457600116729737
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,4,128,1,float16,fp8,127,0.010073599964380264
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,4,128,1,float16,float16,255,0.010769599676132202
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,4,128,1,float16,fp8,255,0.010835199803113937
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,4,128,1,float16,float16,511,0.012768000364303589
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,4,128,1,float16,fp8,511,0.012542399764060973
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,4,128,1,float16,float16,1023,0.013396799564361572
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,4,128,1,float16,fp8,1023,0.013052800297737121
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,4,128,1,float16,float16,2047,0.013897599279880523
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,4,128,1,float16,fp8,2047,0.01356000006198883
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,8,128,1,float16,float16,1,0.009683199971914292
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,8,128,1,float16,fp8,1,0.009803199768066406
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,8,128,1,float16,float16,3,0.009699200093746186
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,8,128,1,float16,fp8,3,0.009691199660301209
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,8,128,1,float16,float16,7,0.009667199850082398
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,8,128,1,float16,fp8,7,0.009656000137329101
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,8,128,1,float16,float16,15,0.009670399874448777
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,8,128,1,float16,fp8,15,0.009662400186061858
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,8,128,1,float16,float16,31,0.009607999771833419
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,8,128,1,float16,fp8,31,0.009692800045013428
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,8,128,1,float16,float16,63,0.009518399834632874
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,8,128,1,float16,float16,127,0.009540800005197525
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,8,128,1,float16,fp8,127,0.010014399886131287
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,8,128,1,float16,float16,255,0.010756800323724747
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,8,128,1,float16,fp8,255,0.010942400246858597
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,8,128,1,float16,float16,511,0.012990400195121765
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,8,128,1,float16,fp8,511,0.012647999823093415
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,8,128,1,float16,float16,1023,0.013643200695514678
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,8,128,1,float16,fp8,1023,0.013206399977207184
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,2,128,1,float16,float16,127,0.00968639999628067
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,8,128,1,float16,float16,2047,0.014193600416183472
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,1,128,1,float16,float16,1,0.011297599971294403
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,2,128,1,float16,float16,511,0.013012799620628356
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,1,128,1,float16,fp8,1,0.0117296002805233
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,1,128,1,float16,fp8,3,0.011817599833011627
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,1,128,1,float16,float16,7,0.01143999993801117
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,1,128,1,float16,fp8,7,0.011615999788045884
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,1,128,1,float16,float16,15,0.011347199976444244
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,1,128,1,float16,fp8,15,0.011591999977827071
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,1,128,1,float16,float16,31,0.011289600282907486
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,1,128,1,float16,fp8,31,0.011627200245857238
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,1,128,1,float16,float16,63,0.011428800225257874
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,1,128,1,float16,fp8,63,0.01178399994969368
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,1,128,1,float16,float16,127,0.011521600186824799
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,1,128,1,float16,fp8,127,0.011750400066375732
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,1,128,1,float16,float16,255,0.013020800054073333
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,1,128,1,float16,fp8,255,0.01342879980802536
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,1,128,1,float16,float16,511,0.015886400640010834
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,1,128,1,float16,fp8,511,0.01671839952468872
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,1,128,1,float16,float16,1023,0.01595200002193451
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,1,128,1,float16,float16,1023,0.015644800662994385
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,1,128,1,float16,fp8,2047,0.016734400391578676
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,1,128,1,float16,float16,2047,0.016068799793720244
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,2,128,1,float16,float16,1,0.009550400078296661
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,1,128,1,float16,fp8,2047,0.016739200055599212
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,2,128,1,float16,fp8,1,0.009411200135946273
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,2,128,1,float16,float16,3,0.009635200351476669
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,8,128,1,float16,fp8,63,0.009697599709033966
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,2,128,1,float16,float16,7,0.009596800059080124
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,2,128,1,float16,fp8,7,0.009398400038480758
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,2,128,1,float16,float16,15,0.009576000273227692
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,2,128,1,float16,fp8,15,0.009348800033330917
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,2,128,1,float16,float16,31,0.009460800141096116
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,2,128,1,float16,fp8,31,0.009414400160312652
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,2,128,1,float16,float16,63,0.009507200121879578
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,2,128,1,float16,fp8,63,0.009486400336027146
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,2,128,1,float16,float16,127,0.009638399630784989
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,2,128,1,float16,fp8,127,0.01019359976053238
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,2,128,1,float16,float16,255,0.010977599769830704
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,2,128,1,float16,fp8,255,0.01103999987244606
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,2,128,1,float16,float16,511,0.01289760023355484
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,2,128,1,float16,float16,1023,0.013583999872207642
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,2,128,1,float16,fp8,1023,0.01308320015668869
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,8,128,1,float16,fp8,2047,0.014009599387645722
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,2,128,1,float16,float16,2047,0.014022399485111237
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,2,128,1,float16,fp8,2047,0.013795199990272521
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,1,128,1,float16,float16,3,0.011462400108575821
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,4,128,1,float16,float16,1,0.009544000029563904
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,4,128,1,float16,float16,3,0.009678400307893752
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,4,128,1,float16,fp8,3,0.009516800194978714
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,4,128,1,float16,fp8,7,0.009524799883365631
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,4,128,1,float16,float16,15,0.009745600074529648
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,4,128,1,float16,fp8,15,0.0095040000975132
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,4,128,1,float16,float16,31,0.009756799787282944
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,4,128,1,float16,fp8,31,0.009390400350093841
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,4,128,1,float16,float16,63,0.009617599844932555
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,4,128,1,float16,fp8,63,0.009356799721717834
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,4,128,1,float16,float16,127,0.009556800127029419
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,4,128,1,float16,fp8,127,0.00997759997844696
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,4,128,1,float16,float16,255,0.011019200086593628
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,4,128,1,float16,float16,511,0.013046400249004364
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,4,128,1,float16,fp8,511,0.012476799637079239
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,4,128,1,float16,float16,1023,0.013566400110721587
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,1,128,1,float16,fp8,1023,0.016673600673675536
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,4,128,1,float16,float16,2047,0.01443839967250824
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,2,128,1,float16,fp8,3,0.009446399658918381
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,4,128,1,float16,fp8,2047,0.014033600687980652
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,8,128,1,float16,float16,1,0.009911999851465226
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,8,128,1,float16,fp8,1,0.009507200121879578
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,8,128,1,float16,float16,3,0.009763199836015701
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,8,128,1,float16,fp8,3,0.009702400118112565
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,8,128,1,float16,float16,7,0.009544000029563904
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,8,128,1,float16,fp8,7,0.009515199810266495
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,8,128,1,float16,float16,15,0.00968480035662651
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,8,128,1,float16,fp8,15,0.009532800316810608
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,8,128,1,float16,float16,31,0.009691199660301209
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,8,128,1,float16,fp8,31,0.009611199796199798
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,8,128,1,float16,float16,63,0.009833600372076035
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,8,128,1,float16,fp8,63,0.009728000313043595
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,8,128,1,float16,float16,127,0.009932799637317658
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,4,128,1,float16,fp8,1,0.009595199674367904
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,8,128,1,float16,fp8,127,0.010742399841547012
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,8,128,1,float16,fp8,255,0.011395200341939925
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,8,128,1,float16,float16,511,0.013249599933624267
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,8,128,1,float16,fp8,511,0.012628799676895142
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,8,128,1,float16,float16,1023,0.014022399485111237
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,8,128,1,float16,fp8,1023,0.013600000739097595
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,8,128,1,float16,float16,2047,0.01573919951915741
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,128,4,128,1,float16,float16,127,0.009431999921798707
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,8,128,1,float16,fp8,2047,0.014726400375366211
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,1,128,1,float16,float16,1,0.012556800246238708
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,1,128,1,float16,fp8,1,0.01305440068244934
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,4,128,1,float16,fp8,255,0.01106560006737709
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,1,128,1,float16,fp8,3,0.012950399518013
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,1,128,1,float16,fp8,7,0.01279519945383072
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,1,128,1,float16,float16,7,0.012505599856376648
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,1,128,1,float16,float16,15,0.0127920001745224
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,1,128,1,float16,fp8,15,0.013208000361919403
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,1,128,1,float16,float16,31,0.012806400656700134
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,4,128,1,float16,fp8,1023,0.012980799376964568
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,1,128,1,float16,fp8,31,0.013316799700260163
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,1,128,1,float16,float16,63,0.012831999361515046
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,1,128,1,float16,fp8,63,0.012593600153923034
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,1,128,1,float16,fp8,127,0.013143999874591828
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,1,128,1,float16,float16,255,0.014148800075054169
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,1,128,1,float16,float16,127,0.012779200077056884
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,1,128,1,float16,fp8,255,0.014843200147151948
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,1,128,1,float16,float16,511,0.01730719953775406
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,1,128,1,float16,float16,1023,0.02021919935941696
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,1,128,1,float16,fp8,1023,0.019952000677585603
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,1,128,1,float16,float16,2047,0.02435680031776428
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,1,128,1,float16,fp8,2047,0.02325280010700226
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,2,128,1,float16,float16,1,0.01101600006222725
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,2,128,1,float16,fp8,1,0.010526400059461594
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,2,128,1,float16,float16,3,0.010860799998044967
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,2,128,1,float16,fp8,3,0.010648000240325927
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,2,128,1,float16,fp8,511,0.012825599312782288
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,2,128,1,float16,fp8,7,0.010385599732398988
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,2,128,1,float16,float16,15,0.010635200142860412
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,2,128,1,float16,fp8,15,0.01067039966583252
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,2,128,1,float16,float16,31,0.01048479974269867
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,2,128,1,float16,fp8,31,0.010406400263309478
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,2,128,1,float16,float16,63,0.01072480008006096
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,2,128,1,float16,fp8,63,0.010526400059461594
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,2,128,1,float16,float16,127,0.010760000348091126
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,2,128,1,float16,fp8,127,0.011401599645614624
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,2,128,1,float16,float16,255,0.01218239963054657
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,2,128,1,float16,fp8,255,0.012193600088357926
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,2,128,1,float16,float16,511,0.014534400403499603
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,8,128,1,float16,float16,255,0.0112527996301651
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,2,128,1,float16,float16,1023,0.01879200041294098
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,2,128,1,float16,fp8,1023,0.01846559941768646
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,2,128,1,float16,fp8,2047,0.02322079986333847
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,4,128,1,float16,float16,1,0.010755199939012527
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,4,128,1,float16,fp8,1,0.010652799904346467
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,4,128,1,float16,float16,3,0.010761599987745285
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,4,128,1,float16,fp8,3,0.010811199992895126
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,1,128,1,float16,float16,3,0.012831999361515046
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,4,128,1,float16,float16,7,0.010982400178909302
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,4,128,1,float16,fp8,7,0.010977599769830704
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,4,128,1,float16,float16,15,0.010969600081443787
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,4,128,1,float16,fp8,15,0.010929600149393082
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,4,128,1,float16,float16,31,0.010896000266075134
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,4,128,1,float16,fp8,31,0.010726399719715118
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,4,128,1,float16,float16,63,0.011161600053310395
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,4,128,1,float16,fp8,63,0.010784000158309937
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,4,128,1,float16,fp8,127,0.0118367999792099
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,4,128,1,float16,float16,255,0.012521600723266602
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,4,128,1,float16,fp8,255,0.012223999947309494
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,4,128,1,float16,float16,511,0.014679999649524688
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,4,128,1,float16,fp8,511,0.016076800227165223
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,1,128,1,float16,fp8,511,0.017926399409770966
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,4,128,1,float16,float16,1023,0.0284527987241745
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,4,128,1,float16,fp8,1023,0.020747199654579163
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,4,128,1,float16,float16,2047,0.044095999002456664
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,4,128,1,float16,fp8,2047,0.03188480138778686
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,2,128,1,float16,float16,7,0.01096320003271103
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,8,128,1,float16,fp8,1,0.013172799348831176
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,8,128,1,float16,float16,3,0.01356160044670105
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,8,128,1,float16,fp8,3,0.013332800567150116
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,8,128,1,float16,float16,7,0.013628800213336945
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,8,128,1,float16,fp8,7,0.01313440054655075
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,8,128,1,float16,fp8,15,0.013513599336147309
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,8,128,1,float16,float16,31,0.013275200128555298
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,8,128,1,float16,fp8,31,0.013527999818325042
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,8,128,1,float16,float16,63,0.013513599336147309
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,8,128,1,float16,fp8,63,0.013556799292564392
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,8,128,1,float16,float16,127,0.01385280042886734
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,8,128,1,float16,fp8,127,0.015094399452209473
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,8,128,1,float16,float16,255,0.01693439930677414
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,128,4,128,1,float16,float16,7,0.009753599762916565
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,8,128,1,float16,fp8,255,0.016761599481105803
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,8,128,1,float16,float16,511,0.028892800211906433
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,8,128,1,float16,fp8,511,0.02328319996595383
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,2,128,1,float16,float16,2047,0.03195360004901886
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,8,128,1,float16,fp8,1023,0.03364959955215454
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,8,128,1,float16,float16,2047,0.07590399980545044
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,8,128,1,float16,fp8,2047,0.05384640097618103
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,1,128,1,float16,float16,1,0.011334399878978729
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,1,128,1,float16,fp8,1,0.011915200203657151
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,1,128,1,float16,float16,3,0.011529599875211715
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,1,128,1,float16,fp8,3,0.01194240003824234
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,1,128,1,float16,float16,7,0.011427199840545655
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,1,128,1,float16,fp8,7,0.011907199770212174
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,1,128,1,float16,float16,15,0.011561600118875503
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,1,128,1,float16,fp8,15,0.011956799775362015
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,1,128,1,float16,float16,31,0.011539199948310852
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,1,128,1,float16,fp8,31,0.01186719983816147
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,1,128,1,float16,float16,63,0.011407999694347382
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,1,128,1,float16,fp8,63,0.011827199906110763
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,1,128,1,float16,float16,127,0.011404799669981003
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,1,128,1,float16,fp8,127,0.011697600036859513
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,1,128,1,float16,float16,255,0.012958399951457977
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,1,128,1,float16,fp8,255,0.01332319974899292
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,1,128,1,float16,float16,511,0.01592320054769516
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,1,128,1,float16,fp8,511,0.016598400473594666
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,1,128,1,float16,float16,1023,0.01589120030403137
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,1,128,1,float16,fp8,1023,0.01680160015821457
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,1,128,1,float16,fp8,2047,0.01703519970178604
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,2,128,1,float16,float16,1,0.009851200133562088
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,2,128,1,float16,fp8,1,0.009542399644851684
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,2,128,1,float16,float16,3,0.009862399846315383
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,2,128,1,float16,fp8,3,0.00960640013217926
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,2,128,1,float16,float16,7,0.009726399928331375
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,8,128,1,float16,float16,1,0.013556799292564392
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,2,128,1,float16,fp8,7,0.009393599629402161
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,2,128,1,float16,float16,15,0.009860800206661224
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,2,128,1,float16,fp8,15,0.00958240032196045
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,2,128,1,float16,float16,31,0.009758400171995163
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,8,128,1,float16,float16,15,0.01358720064163208
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,2,128,1,float16,fp8,31,0.009545599669218063
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,2,128,1,float16,fp8,63,0.0095040000975132
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,2,128,1,float16,float16,127,0.009756799787282944
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,2,128,1,float16,fp8,127,0.01027199998497963
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,2,128,1,float16,float16,255,0.011033599823713302
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,2,128,1,float16,fp8,255,0.011105599999427795
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,2,128,1,float16,float16,511,0.013265599310398103
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,2,128,1,float16,fp8,511,0.012734399735927581
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,2,128,1,float16,float16,1023,0.01422400027513504
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,2,128,1,float16,fp8,1023,0.0133775994181633
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,2,128,1,float16,fp8,511,0.01615840047597885
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,8,128,1,float16,float16,1023,0.045440000295639035
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,4,128,1,float16,float16,1,0.009860800206661224
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,4,128,1,float16,float16,3,0.009798400104045868
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,4,128,1,float16,fp8,3,0.009691199660301209
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,4,128,1,float16,float16,7,0.009712000191211701
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,4,128,1,float16,fp8,7,0.00955199971795082
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,4,128,1,float16,float16,15,0.009785600006580353
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,4,128,1,float16,fp8,15,0.00957920029759407
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,4,128,1,float16,float16,31,0.00979520007967949
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,4,128,1,float16,fp8,31,0.009696000069379807
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,4,128,1,float16,float16,63,0.009862399846315383
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,4,128,1,float16,fp8,63,0.009724800288677216
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,1,128,1,float16,float16,2047,0.01656640022993088
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,4,128,1,float16,fp8,127,0.010390400141477584
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,4,128,1,float16,float16,255,0.01114879995584488
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,4,128,1,float16,fp8,255,0.011099199950695037
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,4,128,1,float16,float16,511,0.013391999900341034
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,4,128,1,float16,fp8,511,0.012889599800109864
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,4,128,1,float16,float16,1023,0.013891200721263885
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,4,128,1,float16,fp8,1023,0.01409280002117157
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,4,128,1,float16,float16,2047,0.015727999806404113
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,2,128,1,float16,float16,63,0.009753599762916565
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,4,128,1,float16,fp8,2047,0.015825599431991577
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,8,128,1,float16,float16,1,0.009852799773216247
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,8,128,1,float16,fp8,1,0.009539200365543366
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,8,128,1,float16,fp8,3,0.009531199932098389
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,8,128,1,float16,float16,7,0.00960479974746704
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,8,128,1,float16,fp8,7,0.009536000341176987
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,8,128,1,float16,float16,15,0.009860800206661224
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,2,128,1,float16,float16,2047,0.014977599680423736
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,8,128,1,float16,fp8,15,0.00979200005531311
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,2,128,1,float16,fp8,2047,0.014814400672912597
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,8,128,1,float16,float16,31,0.00984319970011711
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,8,128,1,float16,fp8,31,0.009748800098896027
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,8,128,1,float16,fp8,63,0.010123199969530105
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,4,128,1,float16,fp8,1,0.009590400010347366
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,8,128,1,float16,float16,255,0.011417599767446518
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,8,128,1,float16,fp8,255,0.011311999708414077
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,8,128,1,float16,float16,511,0.013686400651931763
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,8,128,1,float16,fp8,511,0.013769599795341491
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,8,128,1,float16,float16,1023,0.015167999267578124
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,8,128,1,float16,fp8,1023,0.014870400726795196
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,8,128,1,float16,float16,2047,0.017771199345588684
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,8,128,1,float16,fp8,2047,0.016811199486255646
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,1,128,1,float16,fp8,1,0.015030400454998016
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,4,128,1,float16,float16,127,0.009836799651384353
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,1,128,1,float16,float16,3,0.014742399752140044
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,1,128,1,float16,float16,7,0.014591999351978302
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,8,128,1,float16,float16,3,0.009913600236177444
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,1,128,1,float16,fp8,7,0.015267199277877808
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,1,128,1,float16,float16,15,0.015411199629306793
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,1,128,1,float16,fp8,15,0.015568000078201295
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,1,128,1,float16,float16,31,0.015478399395942689
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,8,128,1,float16,float16,63,0.009707199782133103
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,1,128,1,float16,fp8,31,0.015323199331760406
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,1,128,1,float16,float16,63,0.01510079950094223
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,1,128,1,float16,fp8,63,0.015409600734710694
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,1,128,1,float16,float16,127,0.015483200550079346
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,1,128,1,float16,fp8,127,0.01525759994983673
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,1,128,1,float16,float16,255,0.017215999960899352
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,1,128,1,float16,fp8,255,0.017046399414539337
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,1,128,1,float16,float16,511,0.021457600593566894
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,1,128,1,float16,fp8,511,0.02022079974412918
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,1,128,1,float16,float16,1023,0.034539198875427245
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,1,128,1,float16,fp8,1023,0.02815679907798767
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,2,128,1,float16,float16,1,0.015824000537395477
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,2,128,1,float16,fp8,1,0.015753600001335143
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,2,128,1,float16,float16,3,0.015518400073051452
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,2,128,1,float16,fp8,3,0.015782399475574492
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,2,128,1,float16,float16,7,0.015571199357509613
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,2,128,1,float16,fp8,7,0.01579360067844391
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,2,128,1,float16,float16,15,0.015777599811553956
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,2,128,1,float16,fp8,15,0.015828800201416016
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,2,128,1,float16,float16,31,0.015958400070667268
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,1,128,1,float16,fp8,3,0.015211200714111328
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,2,128,1,float16,fp8,31,0.015724800527095795
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,2,128,1,float16,float16,63,0.01595360040664673
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,2,128,1,float16,fp8,63,0.015547199547290802
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,2,128,1,float16,fp8,127,0.01711679995059967
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,2,128,1,float16,float16,127,0.016913600265979767
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,2,128,1,float16,float16,255,0.019812799990177155
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,2,128,1,float16,fp8,255,0.018889600038528444
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,2,128,1,float16,float16,511,0.03331519961357117
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,2,128,1,float16,fp8,511,0.026412799954414368
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,2,128,1,float16,fp8,1023,0.03889600038528442
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,2,128,1,float16,float16,1023,0.05109279751777649
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,4,128,1,float16,float16,1,0.02128479927778244
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,4,128,1,float16,fp8,1,0.021406400203704833
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,4,128,1,float16,fp8,3,0.02102400064468384
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,4,128,1,float16,float16,7,0.021489599347114564
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,4,128,1,float16,fp8,7,0.02096800059080124
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,4,128,1,float16,fp8,15,0.021040000021457672
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,4,128,1,float16,float16,15,0.02144320011138916
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,4,128,1,float16,fp8,31,0.02126239985227585
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,4,128,1,float16,float16,31,0.02130240052938461
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,4,128,1,float16,float16,63,0.021615999937057494
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,4,128,1,float16,fp8,63,0.021035200357437132
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,4,128,1,float16,float16,127,0.02343360036611557
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,4,128,1,float16,float16,255,0.037006399035453795
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,4,128,1,float16,fp8,255,0.029155200719833373
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,8,128,1,float16,float16,127,0.010180799663066864
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,4,128,1,float16,float16,511,0.052534401416778564
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,4,128,1,float16,fp8,511,0.044440001249313354
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,8,128,1,float16,float16,1,0.03304319977760315
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,4,128,1,float16,float16,1023,0.08497440218925476
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,8,128,1,float16,fp8,1,0.032318401336669925
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,8,128,1,float16,float16,3,0.03280960023403168
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,128,4,128,1,float16,float16,127,0.011308799684047698
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,8,128,1,float16,float16,7,0.03281280100345611
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,1,128,1,float16,float16,1,0.015345600247383118
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,8,128,1,float16,float16,15,0.03298879861831665
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,8,128,1,float16,float16,31,0.03294720053672791
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,8,128,1,float16,fp8,31,0.03218719959259033
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,8,128,1,float16,float16,63,0.03342399895191193
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,8,128,1,float16,fp8,63,0.0322847992181778
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,8,128,1,float16,float16,127,0.04170719981193542
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,8,128,1,float16,fp8,127,0.039284801483154295
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,8,128,1,float16,float16,255,0.057206398248672484
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,8,128,1,float16,fp8,255,0.04975999891757965
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,4,128,1,float16,float16,3,0.021247999370098115
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,8,128,1,float16,float16,511,0.0881600022315979
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,8,128,1,float16,fp8,511,0.07356799840927124
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,8,128,1,float16,float16,1023,0.14928799867630005
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,8,128,1,float16,fp8,1023,0.10941920280456544
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,128,1,128,1,float16,float16,1,0.021473599970340727
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,128,1,128,1,float16,fp8,1,0.022115199267864226
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,128,1,128,1,float16,float16,3,0.020902399718761445
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,128,1,128,1,float16,fp8,3,0.022243200242519377
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,128,1,128,1,float16,float16,7,0.020891200006008147
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,128,8,128,1,float16,fp8,127,0.010793600231409073
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,128,1,128,1,float16,float16,15,0.020945599675178526
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,128,1,128,1,float16,fp8,15,0.02253440022468567
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,128,1,128,1,float16,float16,31,0.021134400367736818
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,128,1,128,1,float16,fp8,31,0.02253279983997345
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,128,1,128,1,float16,float16,63,0.02113119959831238
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,4,128,1,float16,fp8,1023,0.06309120059013366
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,128,1,128,1,float16,fp8,63,0.022511999309062957
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,128,1,128,1,float16,float16,127,0.022158400714397432
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,128,1,128,1,float16,float16,255,0.029931199550628663
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,128,1,128,1,float16,fp8,255,0.025646400451660157
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,128,2,128,1,float16,fp8,1,0.023830400407314302
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,128,2,128,1,float16,float16,3,0.0238864004611969
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,8,128,1,float16,fp8,7,0.03188160061836243
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,8,128,1,float16,fp8,15,0.03214240074157715
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,128,2,128,1,float16,fp8,3,0.023715199530124666
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,128,2,128,1,float16,float16,7,0.024137599766254424
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,128,2,128,1,float16,fp8,7,0.023721599578857423
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,128,2,128,1,float16,fp8,15,0.023947200179100035
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,128,2,128,1,float16,float16,31,0.024086399376392363
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,128,2,128,1,float16,fp8,31,0.02380319982767105
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,128,2,128,1,float16,float16,63,0.024843199551105498
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,128,2,128,1,float16,fp8,63,0.023931199312210084
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,128,2,128,1,float16,float16,127,0.03073599934577942
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,128,2,128,1,float16,fp8,127,0.027425599098205567
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,128,2,128,1,float16,float16,255,0.04117920100688934
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,4,128,1,float16,fp8,127,0.024244800209999084
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,128,2,128,1,float16,fp8,255,0.032332798838615416
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,128,1,128,1,float16,fp8,7,0.02224159985780716
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,128,4,128,1,float16,fp8,1,0.03491680026054382
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,128,4,128,1,float16,fp8,3,0.03484799861907959
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,128,4,128,1,float16,float16,3,0.03495840132236481
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,128,4,128,1,float16,float16,7,0.034959998726844785
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,128,4,128,1,float16,fp8,7,0.03487679958343506
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,128,4,128,1,float16,float16,15,0.035046398639678955
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,128,2,128,1,float16,float16,1,0.024014399945735933
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,128,4,128,1,float16,fp8,15,0.03479520082473755
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,128,4,128,1,float16,float16,31,0.03537600040435791
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,128,4,128,1,float16,fp8,31,0.03529919981956482
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,128,4,128,1,float16,float16,63,0.03952319920063019
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,128,4,128,1,float16,fp8,63,0.03537600040435791
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,128,4,128,1,float16,float16,127,0.04547359943389893
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,128,4,128,1,float16,float16,255,0.06139199733734131
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,128,4,128,1,float16,fp8,255,0.0529744029045105
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,128,8,128,1,float16,float16,1,0.05648959875106811
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,128,8,128,1,float16,float16,3,0.056657600402832034
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,128,8,128,1,float16,fp8,3,0.05622879862785339
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,128,8,128,1,float16,float16,7,0.05638239979743957
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,128,2,128,1,float16,float16,15,0.02398560047149658
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,128,8,128,1,float16,fp8,7,0.055955201387405396
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,128,8,128,1,float16,fp8,15,0.056086397171020506
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,128,8,128,1,float16,float16,15,0.05658400058746338
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,128,8,128,1,float16,float16,31,0.05688160061836243
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,128,8,128,1,float16,fp8,31,0.05624480247497558
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,128,8,128,1,float16,float16,63,0.0642192006111145
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,128,8,128,1,float16,float16,127,0.072297602891922
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,128,8,128,1,float16,fp8,127,0.07433120012283326
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,128,8,128,1,float16,float16,255,0.09971519708633422
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,128,4,128,1,float16,float16,1,0.035078400373458864
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,128,8,128,1,float16,fp8,255,0.08928800225257874
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,128,1,128,1,float16,float16,1,0.03522560000419617
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,128,1,128,1,float16,fp8,1,0.03712640106678009
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,128,1,128,1,float16,float16,3,0.03510079979896545
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,128,1,128,1,float16,fp8,3,0.03713119924068451
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,128,1,128,1,float16,float16,7,0.035366401076316833
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,128,1,128,1,float16,fp8,127,0.02295839935541153
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,128,1,128,1,float16,fp8,7,0.037544000148773196
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,128,1,128,1,float16,float16,15,0.0354559987783432
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,128,1,128,1,float16,fp8,15,0.03761439919471741
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,128,1,128,1,float16,float16,31,0.03625600039958954
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,128,1,128,1,float16,fp8,31,0.03724640011787415
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,128,1,128,1,float16,float16,63,0.04039359986782074
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,128,4,128,1,float16,fp8,127,0.043424001336097716
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,128,1,128,1,float16,float16,127,0.04363360106945038
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,128,1,128,1,float16,fp8,127,0.03918080031871796
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,128,2,128,1,float16,float16,1,0.04128159880638123
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,128,8,128,1,float16,fp8,1,0.05617280006408691
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,128,2,128,1,float16,fp8,3,0.040956801176071166
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,128,2,128,1,float16,float16,7,0.041526401042938234
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,128,2,128,1,float16,fp8,7,0.040934398770332336
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,128,2,128,1,float16,float16,15,0.04193759858608246
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,128,2,128,1,float16,fp8,15,0.041231998801231386
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,128,2,128,1,float16,fp8,31,0.041438400745391846
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,128,2,128,1,float16,float16,63,0.05008159875869751
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,128,2,128,1,float16,fp8,63,0.044268798828125
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,128,2,128,1,float16,float16,127,0.055225598812103274
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,128,2,128,1,float16,fp8,127,0.0525439977645874
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,128,4,128,1,float16,float16,1,0.06267679929733276
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,128,4,128,1,float16,fp8,1,0.06355680227279663
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,128,4,128,1,float16,float16,3,0.06252959966659546
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,128,4,128,1,float16,fp8,3,0.06348320245742797
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,128,4,128,1,float16,float16,7,0.06242560148239136
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,128,4,128,1,float16,fp8,7,0.06355199813842774
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,128,4,128,1,float16,fp8,15,0.06355680227279663
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,128,4,128,1,float16,float16,31,0.06867520213127136
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,128,4,128,1,float16,fp8,31,0.06395999789237976
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,128,4,128,1,float16,float16,63,0.07314879894256592
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,128,4,128,1,float16,fp8,63,0.06969119906425476
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,128,4,128,1,float16,float16,127,0.07934240102767945
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,128,4,128,1,float16,fp8,127,0.08176640272140503
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,128,8,128,1,float16,fp8,3,0.0322847992181778
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,128,8,128,1,float16,float16,1,0.10382879972457885
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,128,8,128,1,float16,fp8,1,0.10337599515914916
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,128,8,128,1,float16,fp8,3,0.10310399532318115
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,128,2,128,1,float16,fp8,1,0.040935999155044554
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,128,8,128,1,float16,float16,7,0.10351200103759765
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,128,8,128,1,float16,fp8,7,0.10333759784698486
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,128,2,128,1,float16,float16,31,0.04311360120773315
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,128,8,128,1,float16,float16,31,0.1112064003944397
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,128,8,128,1,float16,fp8,31,0.1091871976852417
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,128,8,128,1,float16,fp8,63,0.06062560081481934
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,128,4,128,1,float16,float16,15,0.06370400190353394
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,128,8,128,1,float16,fp8,63,0.11404639482498169
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,128,8,128,1,float16,float16,127,0.13125920295715332
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,128,8,128,1,float16,fp8,127,0.13712639808654786
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,1,128,1,float16,float16,1,0.011720000207424164
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,1,128,1,float16,float16,3,0.012025599926710128
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,1,128,1,float16,fp8,1,0.011928000301122666
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,1,128,1,float16,fp8,3,0.012200000137090683
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,1,128,1,float16,float16,7,0.011638399958610535
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,1,128,1,float16,fp8,7,0.011857599765062333
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,1,128,1,float16,float16,15,0.011606399714946748
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,1,128,1,float16,fp8,15,0.01202239990234375
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,1,128,1,float16,float16,31,0.011763200163841248
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,1,128,1,float16,fp8,31,0.012084800004959106
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,1,128,1,float16,float16,63,0.011774399876594543
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,1,128,1,float16,fp8,63,0.011998400092124939
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,1,128,1,float16,float16,127,0.01173119992017746
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,1,128,1,float16,fp8,127,0.01186240017414093
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,128,8,128,1,float16,float16,3,0.10335040092468262
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,1,128,1,float16,fp8,255,0.013628800213336945
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,1,128,1,float16,float16,511,0.016420799493789672
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,128,2,128,1,float16,float16,3,0.04174399971961975
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,1,128,1,float16,float16,1023,0.01661919951438904
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,128,8,128,1,float16,float16,15,0.10671679973602295
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,1,128,1,float16,fp8,1023,0.017446400225162507
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,1,128,1,float16,fp8,2047,0.017606399953365326
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,2,128,1,float16,float16,1,0.010096000134944915
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,2,128,1,float16,fp8,1,0.00979039967060089
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,2,128,1,float16,float16,3,0.010118400305509567
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,2,128,1,float16,fp8,3,0.009947200119495393
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,2,128,1,float16,float16,7,0.010041599720716476
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,2,128,1,float16,fp8,7,0.009707199782133103
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,2,128,1,float16,float16,15,0.009988799691200256
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,128,8,128,1,float16,float16,63,0.11892319917678833
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,2,128,1,float16,float16,31,0.0100832000374794
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,2,128,1,float16,fp8,31,0.009833600372076035
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,2,128,1,float16,float16,63,0.010036800056695938
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,2,128,1,float16,fp8,63,0.009731200337409974
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,2,128,1,float16,float16,127,0.010043200105428696
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,2,128,1,float16,fp8,127,0.010516799986362457
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,2,128,1,float16,float16,255,0.011323200166225433
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,2,128,1,float16,fp8,255,0.011055999994277954
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,2,128,1,float16,float16,511,0.013823999464511872
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,2,128,1,float16,fp8,511,0.014878399670124054
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,2,128,1,float16,float16,1023,0.01485760062932968
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,2,128,1,float16,fp8,1023,0.015688000619411467
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,2,128,1,float16,float16,2047,0.016657599806785585
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,2,128,1,float16,fp8,2047,0.01685599982738495
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,4,128,1,float16,float16,1,0.009859199821949004
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,1,128,1,float16,float16,255,0.01324319988489151
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,4,128,1,float16,fp8,1,0.009657599776983262
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,4,128,1,float16,float16,3,0.009947200119495393
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,4,128,1,float16,float16,7,0.010046400129795074
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,4,128,1,float16,fp8,7,0.009895999729633332
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,4,128,1,float16,float16,15,0.010235200077295304
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,4,128,1,float16,fp8,15,0.010017599910497665
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,4,128,1,float16,float16,31,0.01013600006699562
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,4,128,1,float16,fp8,31,0.009884800016880035
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,1,128,1,float16,float16,2047,0.017310400307178498
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,4,128,1,float16,float16,63,0.010091199725866317
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,4,128,1,float16,float16,127,0.010377600044012069
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,4,128,1,float16,fp8,127,0.010932800173759461
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,4,128,1,float16,float16,255,0.011771199852228164
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,4,128,1,float16,fp8,255,0.011385600268840789
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,4,128,1,float16,float16,511,0.013680000603199006
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,4,128,1,float16,fp8,511,0.014591999351978302
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,4,128,1,float16,float16,1023,0.015380799770355225
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,4,128,1,float16,fp8,1023,0.01539359986782074
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,4,128,1,float16,float16,2047,0.01796800047159195
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,4,128,1,float16,fp8,2047,0.0176816001534462
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,8,128,1,float16,float16,1,0.01013600006699562
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,8,128,1,float16,fp8,1,0.010041599720716476
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,8,128,1,float16,float16,3,0.010169599950313569
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,8,128,1,float16,fp8,3,0.010113599896430969
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,8,128,1,float16,float16,7,0.010100799798965453
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,8,128,1,float16,fp8,7,0.010068800300359726
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,8,128,1,float16,float16,15,0.010100799798965453
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,8,128,1,float16,fp8,15,0.009996800124645234
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,8,128,1,float16,fp8,31,0.010143999755382539
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,8,128,1,float16,float16,63,0.010208000242710114
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,8,128,1,float16,fp8,63,0.010080000013113022
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,8,128,1,float16,float16,127,0.010718400031328202
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,128,1,128,1,float16,fp8,63,0.037934398651123045
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,8,128,1,float16,fp8,127,0.010999999940395355
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,4,128,1,float16,fp8,3,0.009774400293827057
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,8,128,1,float16,fp8,255,0.011547199636697768
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,8,128,1,float16,float16,255,0.011737599968910217
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,8,128,1,float16,fp8,511,0.015092800557613372
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,8,128,1,float16,float16,1023,0.01748320013284683
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,8,128,1,float16,fp8,1023,0.017056000232696534
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,8,128,1,float16,float16,2047,0.02893120050430298
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,8,128,1,float16,fp8,2047,0.02192640006542206
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,128,1,128,1,float16,float16,1,0.0643455982208252
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,128,1,128,1,float16,fp8,1,0.06745759844779968
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,128,8,128,1,float16,fp8,15,0.10354080200195312
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,128,1,128,1,float16,float16,3,0.0646511971950531
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,4,128,1,float16,fp8,63,0.009918399900197983
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,128,1,128,1,float16,fp8,3,0.06753119826316833
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,128,1,128,1,float16,fp8,7,0.06736800074577332
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,128,1,128,1,float16,fp8,15,0.06793919801712037
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,128,1,128,1,float16,float16,15,0.06569120287895203
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,128,1,128,1,float16,float16,31,0.06588320136070251
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,128,1,128,1,float16,fp8,31,0.0683456003665924
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,128,1,128,1,float16,float16,63,0.06819999814033509
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,128,2,128,1,float16,float16,1,0.07885439991950989
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,2,128,1,float16,fp8,15,0.009753599762916565
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,128,2,128,1,float16,float16,3,0.07886559963226318
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,128,2,128,1,float16,fp8,3,0.07741119861602783
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,128,2,128,1,float16,float16,7,0.07901920080184936
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,128,2,128,1,float16,fp8,7,0.07699999809265137
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,128,2,128,1,float16,float16,15,0.07924320101737976
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,128,2,128,1,float16,fp8,15,0.07734879851341248
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,128,2,128,1,float16,float16,31,0.08051679730415344
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,128,2,128,1,float16,fp8,31,0.07764639854431152
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,8,128,1,float16,float16,31,0.010116799920797347
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,128,2,128,1,float16,float16,63,0.08441920280456543
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,128,2,128,1,float16,fp8,63,0.07951520085334778
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,128,4,128,1,float16,float16,1,0.12051520347595215
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,128,4,128,1,float16,fp8,1,0.12000800371170044
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,128,4,128,1,float16,float16,3,0.12067199945449829
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,1,128,1,float16,fp8,511,0.016991999745368958
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,128,4,128,1,float16,float16,7,0.12056959867477417
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,128,4,128,1,float16,fp8,7,0.12062400579452515
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,128,4,128,1,float16,float16,15,0.12150399684906006
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,128,4,128,1,float16,fp8,15,0.12127679586410522
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,128,4,128,1,float16,float16,31,0.12333279848098755
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,128,1,128,1,float16,float16,7,0.06490719914436341
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,128,4,128,1,float16,fp8,31,0.122871994972229
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,128,4,128,1,float16,float16,63,0.128656005859375
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,128,4,128,1,float16,fp8,63,0.1252671957015991
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,128,1,128,1,float16,fp8,63,0.06930239796638489
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,128,8,128,1,float16,fp8,1,0.20177600383758545
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,128,8,128,1,float16,float16,3,0.20024800300598145
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,128,2,128,1,float16,fp8,1,0.07691839933395386
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,128,8,128,1,float16,fp8,3,0.20147840976715087
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,128,8,128,1,float16,float16,7,0.20064001083374022
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,128,8,128,1,float16,fp8,7,0.20253920555114746
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,128,8,128,1,float16,float16,15,0.20334239006042482
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,128,8,128,1,float16,fp8,15,0.2045680046081543
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,128,8,128,1,float16,float16,31,0.20840320587158204
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,128,8,128,1,float16,fp8,31,0.20789599418640137
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,128,8,128,1,float16,float16,63,0.2222656011581421
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,128,1,128,1,float16,float16,1,0.11251039505004883
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,128,8,128,1,float16,fp8,63,0.214247989654541
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,128,1,128,1,float16,fp8,1,0.11950080394744873
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,128,1,128,1,float16,float16,3,0.11268800497055054
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,128,1,128,1,float16,fp8,3,0.12001279592514039
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,128,1,128,1,float16,float16,7,0.11347039937973022
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,128,1,128,1,float16,fp8,7,0.12008960247039795
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,128,1,128,1,float16,float16,15,0.1137120008468628
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,128,1,128,1,float16,fp8,15,0.12005280256271363
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,128,1,128,1,float16,float16,31,0.11499680280685425
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,128,1,128,1,float16,fp8,31,0.12112799882888795
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,128,2,128,1,float16,float16,1,0.14163199663162232
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,128,2,128,1,float16,fp8,1,0.13936959505081176
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,128,2,128,1,float16,float16,3,0.1424831986427307
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,128,2,128,1,float16,fp8,3,0.1404703974723816
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,128,2,128,1,float16,float16,7,0.14172799587249757
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,128,2,128,1,float16,fp8,7,0.13971999883651734
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,128,2,128,1,float16,float16,15,0.14290560483932496
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,128,2,128,1,float16,fp8,15,0.14027040004730223
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,128,2,128,1,float16,float16,31,0.14531999826431274
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,128,4,128,1,float16,float16,1,0.22512800693511964
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,128,4,128,1,float16,float16,3,0.22488958835601808
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,128,4,128,1,float16,fp8,3,0.22584640979766846
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,128,4,128,1,float16,float16,7,0.22459840774536133
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,128,4,128,1,float16,fp8,7,0.2256175994873047
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,128,4,128,1,float16,float16,15,0.22651360034942628
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,128,8,128,1,float16,float16,1,0.20043840408325195
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,128,4,128,1,float16,fp8,15,0.22679359912872316
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,128,4,128,1,float16,float16,31,0.2305743932723999
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,128,4,128,1,float16,fp8,31,0.2296623945236206
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,128,8,128,1,float16,float16,1,0.38288159370422364
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,128,8,128,1,float16,fp8,1,0.38864479064941404
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,128,8,128,1,float16,float16,3,0.38655359745025636
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,128,8,128,1,float16,fp8,3,0.3890575885772705
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,128,8,128,1,float16,float16,7,0.3845792055130005
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,128,8,128,1,float16,fp8,7,0.39093599319458006
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,128,8,128,1,float16,float16,15,0.39181439876556395
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,128,8,128,1,float16,fp8,15,0.3931600093841553
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,1,128,1,float16,float16,1,0.012025599926710128
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,1,128,1,float16,fp8,1,0.012305600196123123
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,128,8,128,1,float16,float16,31,0.40022878646850585
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,1,128,1,float16,float16,3,0.012089599668979645
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,1,128,1,float16,fp8,3,0.012166400253772736
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,128,8,128,1,float16,fp8,31,0.40190877914428713
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,1,128,1,float16,float16,15,0.011659199744462967
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,1,128,1,float16,fp8,7,0.012334399670362473
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,1,128,1,float16,float16,7,0.012051200121641159
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,1,128,1,float16,float16,31,0.012006399780511856
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,1,128,1,float16,fp8,31,0.01241919994354248
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,128,4,128,1,float16,fp8,3,0.11970239877700806
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,1,128,1,float16,float16,63,0.012019199877977371
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,1,128,1,float16,float16,127,0.012015999853610992
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,128,2,128,1,float16,fp8,31,0.1410048007965088
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,1,128,1,float16,float16,255,0.013596799969673157
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,1,128,1,float16,fp8,255,0.014107200503349304
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,128,4,128,1,float16,fp8,1,0.22532639503479004
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,1,128,1,float16,fp8,511,0.01729599982500076
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,1,128,1,float16,float16,1023,0.018172800540924072
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,1,128,1,float16,float16,2047,0.020585599541664123
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,1,128,1,float16,fp8,2047,0.02036159932613373
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,2,128,1,float16,float16,1,0.010288000106811523
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,2,128,1,float16,float16,3,0.010364799946546554
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,2,128,1,float16,fp8,1,0.010289599746465683
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,2,128,1,float16,fp8,3,0.01016160026192665
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,2,128,1,float16,float16,7,0.010169599950313569
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,2,128,1,float16,fp8,7,0.01016160026192665
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,2,128,1,float16,float16,15,0.010310400277376175
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,2,128,1,float16,fp8,15,0.010134399682283402
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,2,128,1,float16,float16,31,0.01019200012087822
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,2,128,1,float16,fp8,31,0.009932799637317658
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,2,128,1,float16,float16,63,0.010206399857997895
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,2,128,1,float16,fp8,63,0.010073599964380264
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,2,128,1,float16,float16,127,0.010494399815797806
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,2,128,1,float16,fp8,127,0.010889600217342376
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,2,128,1,float16,float16,255,0.01199359968304634
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,2,128,1,float16,fp8,255,0.011515200138092041
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,2,128,1,float16,float16,511,0.013896000385284425
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,2,128,1,float16,fp8,511,0.015188799798488617
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,2,128,1,float16,float16,1023,0.016489599645137788
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,2,128,1,float16,fp8,1023,0.015836800634860992
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,2,128,1,float16,float16,2047,0.01868640035390854
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,2,128,1,float16,fp8,2047,0.01839679926633835
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,4,128,1,float16,float16,1,0.010248000174760819
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,4,128,1,float16,fp8,1,0.010124800354242324
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,4,128,1,float16,float16,3,0.010159999877214432
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,4,128,1,float16,fp8,3,0.010188800096511842
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,4,128,1,float16,float16,7,0.010318399965763092
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,4,128,1,float16,fp8,7,0.010267200320959092
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,1,128,1,float16,fp8,63,0.012171199917793274
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,4,128,1,float16,float16,15,0.010342399775981902
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,1,128,1,float16,fp8,127,0.011873599886894227
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,4,128,1,float16,fp8,15,0.010070399940013885
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,4,128,1,float16,fp8,31,0.009915199875831605
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,4,128,1,float16,float16,31,0.01040000021457672
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,4,128,1,float16,float16,63,0.010367999970912933
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,4,128,1,float16,fp8,63,0.009996800124645234
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,4,128,1,float16,float16,127,0.010791999846696853
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,4,128,1,float16,fp8,127,0.010835199803113937
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,4,128,1,float16,float16,255,0.011684799939393998
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,4,128,1,float16,fp8,255,0.01186240017414093
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,4,128,1,float16,fp8,511,0.0151296004652977
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,4,128,1,float16,float16,1023,0.017800000309944154
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,4,128,1,float16,fp8,1023,0.017423999309539796
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,4,128,1,float16,float16,2047,0.02968960106372833
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,8,128,1,float16,float16,1,0.010416000336408614
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,8,128,1,float16,fp8,1,0.010516799986362457
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,8,128,1,float16,float16,3,0.010635200142860412
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,8,128,1,float16,fp8,3,0.01053439974784851
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,8,128,1,float16,float16,7,0.01061440035700798
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,8,128,1,float16,fp8,7,0.010505600273609162
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,8,128,1,float16,float16,15,0.01045759990811348
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,8,128,1,float16,fp8,15,0.010345599800348281
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,8,128,1,float16,float16,31,0.010635200142860412
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,8,128,1,float16,fp8,31,0.01035040020942688
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,8,128,1,float16,float16,63,0.010556799918413162
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,8,128,1,float16,fp8,63,0.010422399640083313
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,8,128,1,float16,float16,127,0.011113599687814713
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,8,128,1,float16,fp8,127,0.011643200367689132
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,8,128,1,float16,float16,255,0.01215839982032776
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,8,128,1,float16,fp8,255,0.012127999961376191
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,8,128,1,float16,float16,511,0.01459839940071106
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,8,128,1,float16,fp8,511,0.015612800419330598
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,8,128,1,float16,float16,1023,0.027508801221847533
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,8,128,1,float16,fp8,1023,0.019966399669647215
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,8,128,1,float16,float16,2047,0.042900800704956055
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,1,128,1,float16,fp8,15,0.012115199863910676
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,8,128,1,float16,fp8,2047,0.0316864013671875
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,1,128,1,float16,float16,1,0.013060800731182098
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,128,8,128,1,float16,float16,511,0.013980799913406372
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,1,128,1,float16,float16,3,0.012921600043773651
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,1,128,1,float16,float16,511,0.016812799870967864
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,1,128,1,float16,fp8,3,0.013627199828624726
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,1,128,1,float16,fp8,7,0.013424000144004822
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,1,128,1,float16,fp8,1023,0.018412800133228303
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,1,128,1,float16,fp8,15,0.01358879953622818
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,1,128,1,float16,float16,31,0.013092799484729767
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,4,128,1,float16,fp8,2047,0.02187040001153946
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,1,128,1,float16,fp8,63,0.01385280042886734
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,1,128,1,float16,fp8,127,0.01387999951839447
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,1,128,1,float16,float16,255,0.015083199739456177
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,1,128,1,float16,fp8,255,0.015225599706172942
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,1,128,1,float16,float16,511,0.017910400032997133
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,1,128,1,float16,fp8,511,0.018435199558734894
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,1,128,1,float16,float16,1023,0.024843199551105498
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,1,128,1,float16,fp8,1023,0.02383359968662262
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,1,128,1,float16,float16,2047,0.036103999614715575
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,1,128,1,float16,fp8,2047,0.029651200771331786
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,2,128,1,float16,float16,1,0.011374399811029435
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,2,128,1,float16,fp8,1,0.011147200316190719
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,2,128,1,float16,float16,3,0.011310400068759918
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,2,128,1,float16,fp8,3,0.011147200316190719
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,2,128,1,float16,float16,7,0.011283200234174728
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,2,128,1,float16,fp8,7,0.01127680018544197
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,2,128,1,float16,float16,15,0.011457599699497223
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,2,128,1,float16,fp8,15,0.011264000087976456
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,2,128,1,float16,float16,31,0.011452800035476685
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,2,128,1,float16,fp8,31,0.011283200234174728
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,1,128,1,float16,fp8,1,0.013265599310398103
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,2,128,1,float16,float16,63,0.011339200288057327
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,2,128,1,float16,float16,127,0.011985599994659424
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,1,128,1,float16,float16,7,0.013092799484729767
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,2,128,1,float16,fp8,63,0.011535999923944473
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,2,128,1,float16,fp8,127,0.012196800112724305
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,1,128,1,float16,float16,15,0.012919999659061432
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,2,128,1,float16,float16,255,0.014097599685192109
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,2,128,1,float16,float16,511,0.015647999942302704
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,128,4,128,1,float16,float16,511,0.013780799508094788
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,2,128,1,float16,fp8,511,0.017127999663352968
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,2,128,1,float16,float16,1023,0.0299919992685318
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,1,128,1,float16,float16,63,0.0133200004696846
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,2,128,1,float16,float16,2047,0.04523360133171082
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,2,128,1,float16,fp8,2047,0.03309440016746521
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,4,128,1,float16,float16,1,0.014207999408245086
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,1,128,1,float16,float16,127,0.013579200208187103
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,4,128,1,float16,float16,3,0.014588800072669984
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,4,128,1,float16,fp8,3,0.014207999408245086
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,4,128,1,float16,float16,7,0.014467200636863709
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,4,128,1,float16,fp8,7,0.014427199959754944
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,4,128,1,float16,float16,15,0.0143327996134758
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,4,128,1,float16,fp8,15,0.01440960019826889
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,4,128,1,float16,float16,31,0.014617599546909332
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,4,128,1,float16,fp8,31,0.014233599603176116
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,4,128,1,float16,float16,63,0.014480000734329224
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,4,128,1,float16,fp8,63,0.013991999626159667
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,4,128,1,float16,float16,127,0.014886400103569031
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,4,128,1,float16,fp8,127,0.015993599593639374
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,4,128,1,float16,float16,255,0.017323200404644013
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,4,128,1,float16,fp8,255,0.017735999822616578
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,4,128,1,float16,float16,511,0.03078559935092926
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,4,128,1,float16,fp8,511,0.02404959946870804
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,4,128,1,float16,float16,1023,0.04738720059394837
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,4,128,1,float16,fp8,1023,0.0358240008354187
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,4,128,1,float16,float16,2047,0.07774080038070678
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,4,128,1,float16,fp8,2047,0.05516160130500793
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,8,128,1,float16,float16,1,0.019897599518299103
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,8,128,1,float16,fp8,1,0.01977919936180115
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,8,128,1,float16,float16,3,0.020153599977493285
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,8,128,1,float16,fp8,3,0.0197952002286911
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,8,128,1,float16,float16,7,0.02016959935426712
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,8,128,1,float16,fp8,7,0.01979999989271164
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,8,128,1,float16,float16,15,0.02016319930553436
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,8,128,1,float16,fp8,15,0.01968639940023422
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,8,128,1,float16,float16,31,0.020207999646663664
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,8,128,1,float16,fp8,31,0.01969279944896698
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,8,128,1,float16,float16,63,0.020329600572586058
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,8,128,1,float16,fp8,63,0.019791999459266664
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,8,128,1,float16,float16,127,0.020508800446987153
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,8,128,1,float16,fp8,127,0.02303680032491684
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,8,128,1,float16,float16,255,0.03495199978351593
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,8,128,1,float16,fp8,255,0.026710399985313417
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,8,128,1,float16,float16,511,0.04923360049724579
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,8,128,1,float16,fp8,511,0.041172799468040464
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,8,128,1,float16,float16,1023,0.08084959983825683
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,8,128,1,float16,fp8,1023,0.05909119844436646
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,8,128,1,float16,float16,2047,0.13960319757461548
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,8,128,1,float16,fp8,2047,0.09813600182533264
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,1,128,1,float16,float16,1,0.011249600350856781
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,4,128,1,float16,fp8,1,0.014155200123786927
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,1,128,1,float16,fp8,1,0.01146719977259636
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,1,128,1,float16,float16,3,0.011142399907112122
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,1,128,1,float16,fp8,3,0.011726400256156922
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,1,128,1,float16,float16,7,0.011028800159692764
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,1,128,1,float16,fp8,7,0.011694400012493134
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,1,128,1,float16,float16,15,0.0112015999853611
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,1,128,1,float16,fp8,15,0.011392000317573547
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,1,128,1,float16,float16,31,0.011214400082826615
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,1,128,1,float16,fp8,31,0.011718399822711945
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,1,128,1,float16,fp8,63,0.011800000071525573
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,1,128,1,float16,float16,127,0.011449600011110306
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,1,128,1,float16,fp8,127,0.011734399944543839
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,1,128,1,float16,float16,255,0.012801599502563477
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,1,128,1,float16,fp8,255,0.01363680064678192
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,1,128,1,float16,fp8,511,0.016867199540138246
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,1,128,1,float16,float16,1023,0.015663999319076537
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,1,128,1,float16,fp8,1023,0.016180799901485445
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,1,128,1,float16,float16,2047,0.01600479930639267
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,1,128,1,float16,fp8,2047,0.01637759953737259
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,2,128,1,float16,float16,1,0.00968799963593483
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,2,128,1,float16,fp8,1,0.009302400052547455
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,2,128,1,float16,float16,3,0.009654399752616883
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,2,128,1,float16,fp8,3,0.009299200028181076
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,2,128,1,float16,float16,7,0.009672000259160995
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,2,128,1,float16,fp8,7,0.00929120033979416
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,2,128,1,float16,float16,15,0.009679999947547913
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,2,128,1,float16,fp8,15,0.009494400024414063
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,2,128,1,float16,float16,31,0.009731200337409974
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,2,128,1,float16,fp8,31,0.009431999921798707
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,2,128,1,float16,float16,63,0.009804800152778625
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,2,128,1,float16,fp8,63,0.009537599980831146
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,2,128,1,float16,float16,127,0.009935999661684037
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,2,128,1,float16,fp8,127,0.01029760017991066
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,2,128,1,float16,float16,255,0.011105599999427795
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,2,128,1,float16,fp8,255,0.010948800295591355
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,2,128,1,float16,float16,511,0.013232000172138214
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,2,128,1,float16,fp8,511,0.012432000041007996
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,2,128,1,float16,float16,1023,0.013828800618648529
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,2,128,1,float16,fp8,1023,0.013067199289798737
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,2,128,1,float16,float16,2047,0.014033600687980652
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,4,128,1,float16,float16,1,0.009540800005197525
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,4,128,1,float16,fp8,1,0.009468799829483033
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,4,128,1,float16,float16,3,0.009620799869298934
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,4,128,1,float16,fp8,3,0.00950080007314682
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,4,128,1,float16,float16,7,0.009564799815416336
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,4,128,1,float16,fp8,7,0.009332799911499023
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,4,128,1,float16,float16,15,0.009556800127029419
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,2,128,1,float16,fp8,1023,0.021855999529361726
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,1,128,1,float16,float16,63,0.011105599999427795
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,4,128,1,float16,float16,31,0.009544000029563904
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,4,128,1,float16,fp8,31,0.00941760018467903
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,4,128,1,float16,float16,63,0.009542399644851684
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,4,128,1,float16,fp8,63,0.009424000233411788
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,4,128,1,float16,float16,127,0.009511999785900116
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,1,128,1,float16,float16,511,0.015787200629711153
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,4,128,1,float16,fp8,127,0.009932799637317658
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,4,128,1,float16,float16,255,0.010894399881362916
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,4,128,1,float16,fp8,255,0.010833600163459777
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,4,128,1,float16,fp8,511,0.01234079971909523
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,4,128,1,float16,float16,1023,0.013275200128555298
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,4,128,1,float16,float16,2047,0.013643200695514678
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,4,128,1,float16,fp8,2047,0.01340319961309433
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,8,128,1,float16,float16,1,0.009921599924564362
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,8,128,1,float16,fp8,1,0.009851200133562088
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,8,128,1,float16,float16,3,0.009972800314426423
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,8,128,1,float16,fp8,3,0.009769599884748459
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,8,128,1,float16,float16,7,0.00979039967060089
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,8,128,1,float16,fp8,7,0.0095551997423172
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,8,128,1,float16,float16,15,0.009774400293827057
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,8,128,1,float16,fp8,15,0.009700799733400345
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,2,128,1,float16,fp8,2047,0.01340319961309433
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,8,128,1,float16,float16,31,0.009782399982213974
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,8,128,1,float16,fp8,31,0.009406399726867676
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,8,128,1,float16,float16,63,0.009935999661684037
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,8,128,1,float16,fp8,63,0.00947680026292801
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,8,128,1,float16,float16,127,0.009995199739933014
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,8,128,1,float16,fp8,127,0.010279999673366546
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,2,128,1,float16,fp8,255,0.012960000336170197
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,8,128,1,float16,float16,255,0.010966400057077408
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,8,128,1,float16,fp8,511,0.012491200119256973
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,8,128,1,float16,float16,1023,0.013928000628948212
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,96,1,128,1,float16,fp8,31,0.013540799915790557
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,8,128,1,float16,float16,2047,0.014228799939155578
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,8,128,1,float16,fp8,2047,0.014097599685192109
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,1,128,1,float16,float16,1,0.01111840009689331
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,1,128,1,float16,fp8,1,0.01157120019197464
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,1,128,1,float16,float16,3,0.011219199746847153
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,1,128,1,float16,fp8,3,0.011668799817562104
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,1,128,1,float16,float16,7,0.011159999668598175
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,1,128,1,float16,fp8,7,0.011691199988126755
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,1,128,1,float16,float16,15,0.011219199746847153
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,1,128,1,float16,fp8,15,0.01151840016245842
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,1,128,1,float16,float16,31,0.011164800077676774
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,4,128,1,float16,float16,511,0.012758399546146392
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,4,128,1,float16,fp8,1023,0.012892800569534301
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,1,128,1,float16,float16,63,0.011054400354623795
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,1,128,1,float16,float16,127,0.011416000127792359
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,1,128,1,float16,fp8,63,0.011648000031709672
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,1,128,1,float16,fp8,127,0.011446399986743927
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,1,128,1,float16,float16,255,0.0127920001745224
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,1,128,1,float16,fp8,255,0.013326400518417358
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,1,128,1,float16,float16,511,0.015694400668144225
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,1,128,1,float16,fp8,511,0.016967999935150146
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,1,128,1,float16,fp8,1023,0.016238400340080263
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,1,128,1,float16,float16,2047,0.015958400070667268
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,1,128,1,float16,fp8,2047,0.01650879979133606
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,2,128,1,float16,fp8,1,0.009296000003814697
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,2,128,1,float16,float16,3,0.009569600224494934
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,2,128,1,float16,fp8,3,0.009430400282144546
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,2,128,1,float16,float16,7,0.009646400064229965
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,2,128,1,float16,fp8,7,0.009335999935865402
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,2,128,1,float16,float16,15,0.009534399956464767
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,8,128,1,float16,fp8,255,0.01109120026230812
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,2,128,1,float16,fp8,15,0.00936639979481697
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,8,128,1,float16,float16,511,0.012863999605178833
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,2,128,1,float16,float16,31,0.009515199810266495
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,2,128,1,float16,fp8,31,0.009406399726867676
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,2,128,1,float16,float16,63,0.009515199810266495
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,2,128,1,float16,fp8,63,0.009305600076913834
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,4,128,1,float16,fp8,15,0.009324800223112106
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,2,128,1,float16,fp8,127,0.009932799637317658
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,2,128,1,float16,float16,255,0.010788799822330475
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,2,128,1,float16,fp8,255,0.010939200222492219
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,2,128,1,float16,float16,511,0.012857599556446076
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,2,128,1,float16,float16,1023,0.013736000657081604
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,2,128,1,float16,fp8,1023,0.013412800431251527
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,2,128,1,float16,float16,2047,0.014270399510860444
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,2,128,1,float16,fp8,2047,0.013775999844074249
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,4,128,1,float16,float16,1,0.009828799962997436
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,4,128,1,float16,fp8,1,0.009547200053930283
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,4,128,1,float16,float16,3,0.009808000177145004
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,4,128,1,float16,fp8,3,0.009576000273227692
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,4,128,1,float16,float16,7,0.009806399792432785
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,4,128,1,float16,fp8,7,0.009539200365543366
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,4,128,1,float16,float16,15,0.009724800288677216
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,4,128,1,float16,fp8,15,0.009455999732017517
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,4,128,1,float16,float16,31,0.009712000191211701
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,4,128,1,float16,fp8,31,0.009494400024414063
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,4,128,1,float16,fp8,63,0.009460800141096116
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,4,128,1,float16,float16,127,0.009702400118112565
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,1,128,1,float16,fp8,31,0.011575999855995178
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,4,128,1,float16,fp8,127,0.010238400101661682
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,4,128,1,float16,fp8,255,0.011443199962377549
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,4,128,1,float16,float16,511,0.013185599446296692
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,4,128,1,float16,fp8,511,0.012667199969291687
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,4,128,1,float16,float16,1023,0.013691200315952301
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,1,128,1,float16,float16,1023,0.015609599649906158
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,4,128,1,float16,float16,2047,0.014399999380111694
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,2,128,1,float16,float16,1,0.00966399982571602
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,4,128,1,float16,fp8,2047,0.014102399349212646
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,8,128,1,float16,float16,1,0.00963359996676445
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,8,128,1,float16,fp8,1,0.009590400010347366
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,8,128,1,float16,float16,3,0.009593600034713745
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,8,128,1,float16,fp8,3,0.009567999839782714
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,8,128,1,float16,float16,7,0.009702400118112565
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,8,128,1,float16,fp8,7,0.009595199674367904
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,8,128,1,float16,float16,15,0.009764800220727921
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,96,8,128,1,float16,fp8,1023,0.013252800703048706
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,8,128,1,float16,fp8,15,0.009721600264310837
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,8,128,1,float16,fp8,31,0.009696000069379807
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,8,128,1,float16,float16,31,0.00963200032711029
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,2,128,1,float16,fp8,511,0.012654399871826172
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,8,128,1,float16,float16,63,0.009752000123262406
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,8,128,1,float16,float16,127,0.009974399954080582
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,8,128,1,float16,fp8,127,0.010284800082445145
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,8,128,1,float16,float16,255,0.011104000359773636
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,8,128,1,float16,fp8,255,0.011070399731397628
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,8,128,1,float16,float16,511,0.013393600285053254
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,8,128,1,float16,fp8,511,0.012671999633312225
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,8,128,1,float16,float16,1023,0.013998399674892425
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,8,128,1,float16,fp8,1023,0.013433599472045898
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,8,128,1,float16,float16,2047,0.015337599813938141
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,8,128,1,float16,fp8,2047,0.014555199444293976
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,1,128,1,float16,float16,1,0.012193600088357926
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,1,128,1,float16,fp8,1,0.012508800625801087
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,1,128,1,float16,float16,3,0.012476799637079239
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,1,128,1,float16,float16,7,0.012486399710178375
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,1,128,1,float16,fp8,7,0.012889599800109864
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,1,128,1,float16,float16,15,0.012510399520397186
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,1,128,1,float16,fp8,15,0.012915199995040894
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,4,128,1,float16,float16,255,0.011032000184059143
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,1,128,1,float16,float16,31,0.012352000176906585
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,1,128,1,float16,fp8,31,0.012747199833393097
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,1,128,1,float16,float16,63,0.012383999675512314
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,1,128,1,float16,fp8,63,0.0126351997256279
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,1,128,1,float16,float16,127,0.012825599312782288
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,1,128,1,float16,fp8,127,0.012651200592517852
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,1,128,1,float16,float16,255,0.014443199336528777
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,1,128,1,float16,fp8,255,0.014300799369812012
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,1,128,1,float16,float16,511,0.01746399998664856
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,1,128,1,float16,fp8,511,0.017459200322628023
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,1,128,1,float16,float16,1023,0.019896000623703003
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,1,128,1,float16,fp8,1023,0.019230400025844575
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,1,128,1,float16,float16,2047,0.023403200507164
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,1,128,1,float16,fp8,2047,0.022655999660491942
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,2,128,1,float16,float16,1,0.011076799780130386
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,2,128,1,float16,float16,3,0.010819199681282043
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,2,128,1,float16,fp8,3,0.010636799782514573
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,2,128,1,float16,float16,7,0.010865599662065507
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,2,128,1,float16,fp8,7,0.0107744000852108
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,2,128,1,float16,float16,15,0.010887999832630158
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,2,128,1,float16,float16,127,0.009705600142478944
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,2,128,1,float16,fp8,15,0.01072480008006096
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,2,128,1,float16,float16,31,0.010732799768447876
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,2,128,1,float16,fp8,31,0.010948800295591355
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,8,128,1,float16,fp8,63,0.009585600346326828
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,2,128,1,float16,float16,127,0.011028800159692764
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,2,128,1,float16,float16,255,0.012366399914026261
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,2,128,1,float16,fp8,127,0.011532799899578094
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,2,128,1,float16,fp8,255,0.012171199917793274
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,2,128,1,float16,float16,511,0.014678399264812469
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,2,128,1,float16,fp8,511,0.015833599865436553
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,2,128,1,float16,float16,1023,0.018991999328136444
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,2,128,1,float16,float16,2047,0.031409600377082826
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,4,128,1,float16,float16,63,0.009591999650001525
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,2,128,1,float16,fp8,2047,0.02300959974527359
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,4,128,1,float16,float16,1,0.01067200005054474
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,4,128,1,float16,fp8,1,0.010820800065994262
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,4,128,1,float16,float16,3,0.010787200182676315
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,4,128,1,float16,fp8,3,0.01085280030965805
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,4,128,1,float16,float16,7,0.010859200358390808
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,4,128,1,float16,fp8,7,0.010726399719715118
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,4,128,1,float16,float16,15,0.010873600095510482
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,4,128,1,float16,fp8,15,0.01096000000834465
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,4,128,1,float16,float16,31,0.010734400153160096
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,4,128,1,float16,fp8,31,0.010969600081443787
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,4,128,1,float16,float16,63,0.010871999710798264
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,4,128,1,float16,fp8,63,0.011030399799346923
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,4,128,1,float16,float16,127,0.01109120026230812
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,4,128,1,float16,fp8,127,0.0118367999792099
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,96,4,128,1,float16,fp8,1023,0.013120000064373017
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,4,128,1,float16,float16,255,0.01231039986014366
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,4,128,1,float16,fp8,255,0.012492799758911132
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,4,128,1,float16,fp8,511,0.01584800034761429
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,4,128,1,float16,float16,1023,0.028148800134658813
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,4,128,1,float16,fp8,1023,0.02035360038280487
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,4,128,1,float16,float16,2047,0.043884798884391785
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,4,128,1,float16,fp8,2047,0.031711998581886294
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,2,128,1,float16,fp8,1,0.01066880002617836
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,8,128,1,float16,float16,1,0.013353599607944489
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,8,128,1,float16,float16,3,0.013444800674915314
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,8,128,1,float16,fp8,3,0.013305599987506866
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,8,128,1,float16,float16,7,0.013406400382518769
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,8,128,1,float16,fp8,7,0.013350400328636169
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,8,128,1,float16,float16,15,0.013409599661827087
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,8,128,1,float16,fp8,15,0.013435199856758118
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,8,128,1,float16,float16,31,0.013324800133705138
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,8,128,1,float16,fp8,31,0.01340160071849823
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,8,128,1,float16,float16,63,0.013460800051689148
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,8,128,1,float16,fp8,63,0.013240000605583191
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,8,128,1,float16,float16,127,0.013955199718475341
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,8,128,1,float16,fp8,127,0.01518079936504364
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,2,128,1,float16,float16,63,0.010886400192975997
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,2,128,1,float16,fp8,63,0.010571199655532836
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,8,128,1,float16,fp8,255,0.01685280054807663
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,8,128,1,float16,fp8,511,0.023192000389099122
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,8,128,1,float16,float16,1023,0.04542720019817352
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,8,128,1,float16,fp8,1023,0.033697599172592164
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,1,128,1,float16,fp8,3,0.012511999905109405
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,8,128,1,float16,float16,2047,0.07573599815368652
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,8,128,1,float16,fp8,2047,0.05379840135574341
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,1,128,1,float16,float16,1,0.011187200248241425
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,1,128,1,float16,fp8,1,0.011745599657297134
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,1,128,1,float16,fp8,3,0.011739200353622437
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,1,128,1,float16,float16,7,0.011291199922561645
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,1,128,1,float16,fp8,7,0.011761599779129028
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,1,128,1,float16,float16,15,0.011299200356006622
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,1,128,1,float16,fp8,15,0.011615999788045884
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,1,128,1,float16,float16,31,0.011259199678897857
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,1,128,1,float16,fp8,31,0.011630400270223617
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,4,128,1,float16,float16,511,0.014948800206184387
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,1,128,1,float16,float16,63,0.011388800293207168
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,1,128,1,float16,fp8,63,0.01170239970088005
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,1,128,1,float16,float16,127,0.011451199650764465
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,1,128,1,float16,fp8,127,0.011737599968910217
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,1,128,1,float16,float16,255,0.012990400195121765
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,1,128,1,float16,fp8,255,0.013473600149154663
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,1,128,1,float16,float16,511,0.016046400368213653
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,1,128,1,float16,fp8,511,0.01687519997358322
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,1,128,1,float16,float16,1023,0.015990400314331056
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,8,128,1,float16,fp8,1,0.01319040060043335
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,1,128,1,float16,fp8,1023,0.016471999883651733
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,1,128,1,float16,fp8,2047,0.016816000640392303
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,2,128,1,float16,fp8,1,0.0095551997423172
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,2,128,1,float16,float16,3,0.009743999689817429
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,2,128,1,float16,fp8,3,0.009644799679517747
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,2,128,1,float16,float16,7,0.009859199821949004
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,2,128,1,float16,fp8,7,0.009654399752616883
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,2,128,1,float16,float16,15,0.009911999851465226
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,8,128,1,float16,float16,255,0.016787199676036833
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,2,128,1,float16,float16,31,0.009854400157928466
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,8,128,1,float16,float16,511,0.02871040105819702
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,2,128,1,float16,fp8,31,0.009536000341176987
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,2,128,1,float16,float16,63,0.00971359983086586
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,2,128,1,float16,fp8,63,0.009379199892282485
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,2,128,1,float16,float16,127,0.009651199728250504
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,2,128,1,float16,float16,255,0.011020799726247787
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,2,128,1,float16,fp8,255,0.011360000073909759
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,2,128,1,float16,float16,511,0.013327999413013459
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,2,128,1,float16,fp8,511,0.012601600587368011
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,2,128,1,float16,float16,1023,0.013609600067138673
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,2,128,1,float16,fp8,1023,0.013100799918174744
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,2,128,1,float16,float16,2047,0.014547200500965118
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,2,128,1,float16,fp8,2047,0.01478240042924881
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,4,128,1,float16,float16,1,0.009692800045013428
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,4,128,1,float16,fp8,1,0.009465599805116654
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,4,128,1,float16,float16,3,0.009651199728250504
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,4,128,1,float16,fp8,3,0.009436800330877303
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,4,128,1,float16,float16,7,0.009556800127029419
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,4,128,1,float16,fp8,7,0.009385599941015243
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,4,128,1,float16,float16,15,0.009571199864149093
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,4,128,1,float16,fp8,15,0.009391999989748
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,4,128,1,float16,float16,31,0.009478399902582169
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,4,128,1,float16,fp8,31,0.009471999853849411
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,4,128,1,float16,float16,63,0.009748800098896027
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,1,128,1,float16,float16,2047,0.016128000617027283
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,4,128,1,float16,fp8,63,0.009470400214195252
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,2,128,1,float16,float16,1,0.009867200255393982
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,4,128,1,float16,float16,255,0.011158400028944016
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,4,128,1,float16,fp8,255,0.0110944002866745
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,4,128,1,float16,float16,511,0.013308799266815186
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,4,128,1,float16,fp8,511,0.012870399653911591
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,4,128,1,float16,fp8,1023,0.013804799318313599
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,4,128,1,float16,float16,2047,0.015238399803638458
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,4,128,1,float16,fp8,2047,0.015190400183200836
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,8,128,1,float16,float16,1,0.009985599666833878
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,2,128,1,float16,fp8,15,0.009668800234794616
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,8,128,1,float16,fp8,1,0.010080000013113022
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,8,128,1,float16,fp8,3,0.009710399806499482
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,8,128,1,float16,float16,7,0.009628800302743911
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,8,128,1,float16,fp8,7,0.009598399698734283
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,8,128,1,float16,float16,15,0.009851200133562088
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,96,2,128,1,float16,fp8,1023,0.01825280040502548
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,8,128,1,float16,fp8,15,0.009809599816799163
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,8,128,1,float16,float16,31,0.010163199901580811
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,8,128,1,float16,fp8,31,0.009804800152778625
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,8,128,1,float16,float16,63,0.009956800192594529
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,8,128,1,float16,fp8,63,0.009937600046396256
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,8,128,1,float16,float16,127,0.010190399736166
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,8,128,1,float16,fp8,127,0.010871999710798264
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,8,128,1,float16,fp8,255,0.011308799684047698
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,1,128,1,float16,float16,3,0.0112015999853611
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,8,128,1,float16,float16,511,0.013545599579811097
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,8,128,1,float16,fp8,511,0.013742400705814362
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,8,128,1,float16,float16,1023,0.014847999811172486
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,8,128,1,float16,fp8,1023,0.01465280055999756
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,8,128,1,float16,fp8,2047,0.016710400581359863
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,1,128,1,float16,float16,1,0.014921599626541137
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,4,128,1,float16,fp8,127,0.010284800082445145
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,1,128,1,float16,fp8,1,0.014763200283050537
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,4,128,1,float16,float16,1023,0.013823999464511872
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,1,128,1,float16,fp8,3,0.014956800639629364
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,1,128,1,float16,float16,7,0.014441600441932679
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,1,128,1,float16,fp8,7,0.015011200308799743
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,1,128,1,float16,float16,15,0.01451839953660965
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,1,128,1,float16,fp8,15,0.014833599328994751
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,8,128,1,float16,float16,3,0.009777600318193436
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,1,128,1,float16,float16,31,0.014379200339317322
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,1,128,1,float16,fp8,31,0.015331199765205384
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,1,128,1,float16,fp8,63,0.015363200008869171
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,2,128,1,float16,fp8,127,0.010129600018262862
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,1,128,1,float16,float16,127,0.015408000349998474
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,1,128,1,float16,fp8,127,0.014846399426460266
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,1,128,1,float16,float16,255,0.01687840074300766
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,1,128,1,float16,fp8,255,0.016678400337696075
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,1,128,1,float16,float16,511,0.021139200031757354
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,1,128,1,float16,fp8,511,0.019844800233840942
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,8,128,1,float16,float16,255,0.011403200030326844
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,1,128,1,float16,float16,1023,0.03391520082950592
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,1,128,1,float16,fp8,1023,0.027112001180648805
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,2,128,1,float16,fp8,1,0.015731200575828552
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,2,128,1,float16,float16,3,0.015542399883270264
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,2,128,1,float16,fp8,3,0.0157600000500679
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,2,128,1,float16,float16,7,0.015447999536991119
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,2,128,1,float16,fp8,7,0.01581919938325882
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,2,128,1,float16,float16,15,0.015449599921703338
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,8,128,1,float16,float16,2047,0.017076799273490907
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,2,128,1,float16,fp8,15,0.015932799875736238
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,2,128,1,float16,fp8,31,0.015382400155067444
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,2,128,1,float16,fp8,63,0.015569600462913512
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,2,128,1,float16,float16,127,0.016073599457740784
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,2,128,1,float16,fp8,127,0.01722240000963211
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,2,128,1,float16,float16,255,0.019222399592399596
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,2,128,1,float16,fp8,255,0.018811200559139252
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,2,128,1,float16,float16,511,0.03243040144443512
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,2,128,1,float16,fp8,511,0.025937598943710328
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,2,128,1,float16,float16,1023,0.05035840272903443
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,1,128,1,float16,float16,3,0.014772799611091614
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,2,128,1,float16,fp8,1023,0.03812640011310577
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,4,128,1,float16,float16,1,0.021406400203704833
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,4,128,1,float16,fp8,1,0.021038399636745454
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,4,128,1,float16,float16,3,0.02141599953174591
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,4,128,1,float16,float16,7,0.02141920030117035
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,4,128,1,float16,fp8,7,0.02130880057811737
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,4,128,1,float16,float16,15,0.021460799872875212
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,1,128,1,float16,float16,63,0.014387199282646179
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,4,128,1,float16,fp8,15,0.02120479941368103
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,4,128,1,float16,fp8,31,0.021243199706077576
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,4,128,1,float16,float16,63,0.021451200544834136
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,4,128,1,float16,fp8,63,0.02120800018310547
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,4,128,1,float16,float16,127,0.022355200350284578
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,4,128,1,float16,fp8,127,0.024571199715137482
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,4,128,1,float16,float16,255,0.035950401425361635
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,2,128,1,float16,float16,1,0.015915200114250183
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,4,128,1,float16,fp8,255,0.02906239926815033
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,4,128,1,float16,float16,511,0.051875197887420656
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,4,128,1,float16,fp8,511,0.04376479983329773
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,2,128,1,float16,float16,31,0.01539359986782074
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,4,128,1,float16,float16,1023,0.08447999954223633
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,4,128,1,float16,fp8,1023,0.06263039708137512
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,2,128,1,float16,float16,63,0.0159840002655983
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,8,128,1,float16,float16,1,0.032292801141738894
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,8,128,1,float16,float16,3,0.03272320032119751
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,8,128,1,float16,fp8,3,0.032311999797821046
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,8,128,1,float16,float16,7,0.03246079981327057
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,8,128,1,float16,float16,15,0.03235040009021759
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,8,128,1,float16,fp8,15,0.032287999987602234
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,8,128,1,float16,float16,31,0.03292160034179688
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,8,128,1,float16,float16,63,0.033371201157569884
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,8,128,1,float16,fp8,63,0.032329601049423215
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,8,128,1,float16,float16,127,0.04154880046844482
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,8,128,1,float16,fp8,127,0.038756799697875974
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,8,128,1,float16,float16,255,0.056888002157211306
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,8,128,1,float16,fp8,255,0.04934079945087433
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,96,4,128,1,float16,float16,127,0.009857600182294845
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,8,128,1,float16,float16,511,0.08697919845581055
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,8,128,1,float16,float16,1023,0.14853760004043579
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,8,128,1,float16,fp8,1023,0.10911999940872193
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,96,1,128,1,float16,float16,1,0.021107199788093566
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,96,1,128,1,float16,fp8,1,0.02178719937801361
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,96,1,128,1,float16,float16,3,0.020636799931526183
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,4,128,1,float16,fp8,3,0.020979200303554536
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,96,1,128,1,float16,float16,7,0.020547200739383698
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,96,1,128,1,float16,fp8,7,0.021779200434684752
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,96,1,128,1,float16,float16,15,0.020744000375270844
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,96,1,128,1,float16,fp8,15,0.021726399660110474
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,96,1,128,1,float16,float16,31,0.020771199464797975
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,96,1,128,1,float16,fp8,31,0.021803200244903564
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,96,1,128,1,float16,fp8,63,0.02192319929599762
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,4,128,1,float16,float16,31,0.02133920043706894
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,96,1,128,1,float16,float16,127,0.02101760059595108
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,96,1,128,1,float16,fp8,127,0.021854400634765625
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,96,1,128,1,float16,float16,255,0.027412799000740052
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,96,1,128,1,float16,fp8,255,0.025307199358940123
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,96,2,128,1,float16,fp8,1,0.023763200640678404
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,96,2,128,1,float16,float16,1,0.023928000032901763
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,96,2,128,1,float16,float16,3,0.02388000041246414
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,96,2,128,1,float16,fp8,3,0.023769600689411162
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,96,2,128,1,float16,float16,7,0.0238319993019104
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,96,2,128,1,float16,fp8,7,0.023683199286460878
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,96,2,128,1,float16,float16,15,0.02391359955072403
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,96,2,128,1,float16,fp8,15,0.023601600527763368
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,8,128,1,float16,fp8,1,0.03248319923877716
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,96,2,128,1,float16,float16,31,0.023766399919986726
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,8,128,1,float16,fp8,7,0.03161279857158661
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,96,2,128,1,float16,fp8,31,0.023889599740505217
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,96,2,128,1,float16,fp8,63,0.023537600040435792
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,96,2,128,1,float16,float16,127,0.027251198887825012
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,96,2,128,1,float16,fp8,127,0.026907199621200563
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,96,2,128,1,float16,float16,255,0.03922399878501892
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,96,2,128,1,float16,fp8,255,0.03154560029506683
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,96,4,128,1,float16,float16,1,0.03529919981956482
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,96,4,128,1,float16,fp8,1,0.03476319909095764
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,96,4,128,1,float16,float16,3,0.0352400004863739
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,96,4,128,1,float16,fp8,3,0.034815999865531924
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,96,4,128,1,float16,float16,7,0.03540639877319336
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,8,128,1,float16,fp8,511,0.07284960150718689
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,96,4,128,1,float16,float16,15,0.035596799850463864
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,96,4,128,1,float16,fp8,15,0.03486399948596954
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,96,4,128,1,float16,fp8,31,0.03483040034770966
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,96,4,128,1,float16,float16,63,0.03831200003623962
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,96,4,128,1,float16,fp8,63,0.035087999701499936
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,96,4,128,1,float16,float16,127,0.04447839856147766
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,96,4,128,1,float16,fp8,127,0.04283359944820404
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,96,1,128,1,float16,float16,63,0.02086720019578934
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,96,4,128,1,float16,fp8,255,0.05292320251464844
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,96,8,128,1,float16,float16,1,0.056492799520492555
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,96,8,128,1,float16,fp8,1,0.05637919902801514
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,96,8,128,1,float16,float16,3,0.056411200761795045
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,96,8,128,1,float16,fp8,3,0.05603839755058289
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,96,8,128,1,float16,float16,7,0.0562175989151001
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,96,8,128,1,float16,fp8,7,0.05604000091552734
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,96,8,128,1,float16,float16,15,0.05666239857673645
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,96,2,128,1,float16,float16,63,0.02391040027141571
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,96,8,128,1,float16,fp8,15,0.05618879795074463
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,96,8,128,1,float16,fp8,31,0.03184320032596588
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,96,8,128,1,float16,float16,31,0.0561519980430603
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,96,8,128,1,float16,float16,63,0.06370239853858947
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,96,8,128,1,float16,fp8,63,0.05857920050621033
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,96,8,128,1,float16,float16,127,0.07092800140380859
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,96,8,128,1,float16,fp8,127,0.07386400103569031
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,96,4,128,1,float16,fp8,7,0.03478879928588867
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,96,8,128,1,float16,fp8,255,0.09036319851875305
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,96,1,128,1,float16,float16,1,0.034329599142074584
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,96,1,128,1,float16,fp8,1,0.03667680025100708
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,96,1,128,1,float16,float16,3,0.03424479961395264
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,96,1,128,1,float16,fp8,3,0.03675200045108795
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,96,1,128,1,float16,float16,7,0.034308800101280214
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,96,1,128,1,float16,fp8,7,0.037115201354026794
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,96,1,128,1,float16,fp8,3,0.022414399683475493
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,96,1,128,1,float16,fp8,15,0.036627200245857236
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,96,1,128,1,float16,float16,31,0.034959998726844785
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,96,1,128,1,float16,float16,63,0.03642399907112122
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,96,4,128,1,float16,float16,255,0.060545599460601805
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,96,1,128,1,float16,fp8,63,0.037257599830627444
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,96,1,128,1,float16,fp8,127,0.03748160004615784
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,96,1,128,1,float16,float16,127,0.041422399878501895
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,96,2,128,1,float16,float16,1,0.0406607985496521
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,96,2,128,1,float16,float16,3,0.040747201442718504
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,96,2,128,1,float16,fp8,3,0.04107199907302857
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,96,2,128,1,float16,float16,7,0.04100799858570099
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,96,2,128,1,float16,fp8,7,0.04105600118637085
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,96,2,128,1,float16,float16,15,0.041198399662971494
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,96,2,128,1,float16,fp8,15,0.040971198678016664
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,96,2,128,1,float16,float16,31,0.041766399145126344
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,96,2,128,1,float16,fp8,31,0.04103519916534424
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,96,2,128,1,float16,float16,63,0.04789919853210449
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,96,8,128,1,float16,fp8,31,0.05626720190048218
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,96,2,128,1,float16,fp8,63,0.04174720048904419
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,96,2,128,1,float16,fp8,127,0.05098720192909241
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,96,2,128,1,float16,float16,127,0.05279359817504883
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,96,4,128,1,float16,float16,1,0.06200640201568604
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,96,4,128,1,float16,fp8,1,0.0630240023136139
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,96,4,128,1,float16,float16,3,0.062191998958587645
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,96,4,128,1,float16,fp8,3,0.06312959790229797
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,96,8,128,1,float16,float16,255,0.09837759733200073
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,96,4,128,1,float16,float16,7,0.06228479743003845
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,96,4,128,1,float16,fp8,7,0.0631168007850647
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,96,4,128,1,float16,float16,31,0.03548640012741089
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,96,4,128,1,float16,fp8,15,0.06283040046691894
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,96,4,128,1,float16,float16,31,0.0657584011554718
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,96,4,128,1,float16,fp8,31,0.06363840103149414
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,96,4,128,1,float16,fp8,63,0.06899039745330811
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,96,1,128,1,float16,float16,15,0.034488001465797426
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,96,4,128,1,float16,float16,127,0.07749279737472534
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,96,4,128,1,float16,fp8,127,0.08097280263900757
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,96,1,128,1,float16,fp8,31,0.03716320097446442
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,96,8,128,1,float16,float16,1,0.10347039699554443
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,96,8,128,1,float16,fp8,1,0.10373439788818359
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,96,8,128,1,float16,float16,3,0.10357120037078857
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,96,8,128,1,float16,fp8,3,0.1029968023300171
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,96,8,128,1,float16,float16,7,0.10339200496673584
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,96,8,128,1,float16,fp8,7,0.10383039712905884
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,96,8,128,1,float16,float16,15,0.1034127950668335
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,96,2,128,1,float16,fp8,1,0.04085600078105926
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,96,8,128,1,float16,float16,63,0.1177664041519165
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,96,8,128,1,float16,fp8,63,0.11338080167770385
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,96,8,128,1,float16,float16,127,0.1289247989654541
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,96,8,128,1,float16,fp8,127,0.13659039735794068
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,1,128,1,float16,fp8,1,0.011823999881744384
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,1,128,1,float16,float16,3,0.01138719990849495
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,1,128,1,float16,fp8,3,0.011838400363922119
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,1,128,1,float16,float16,7,0.011376000195741653
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,1,128,1,float16,fp8,7,0.011921600252389909
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,1,128,1,float16,float16,15,0.011540800333023071
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,1,128,1,float16,fp8,15,0.011806400120258331
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,1,128,1,float16,float16,31,0.011401599645614624
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,1,128,1,float16,fp8,31,0.011814399808645248
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,1,128,1,float16,float16,63,0.01154559999704361
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,1,128,1,float16,fp8,63,0.011924800276756287
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,1,128,1,float16,float16,127,0.011604800075292587
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,1,128,1,float16,fp8,127,0.011948800086975098
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,1,128,1,float16,float16,255,0.013219200074672699
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,1,128,1,float16,fp8,255,0.013521599769592284
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,1,128,1,float16,float16,511,0.015985600650310516
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,1,128,1,float16,fp8,511,0.017015999555587767
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,96,4,128,1,float16,float16,15,0.06267840266227723
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,1,128,1,float16,float16,1023,0.01613759994506836
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,1,128,1,float16,fp8,1023,0.01666560024023056
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,1,128,1,float16,float16,2047,0.016948799788951873
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,1,128,1,float16,fp8,2047,0.017129600048065186
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,2,128,1,float16,float16,1,0.009806399792432785
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,2,128,1,float16,float16,3,0.009883199632167817
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,2,128,1,float16,fp8,3,0.009700799733400345
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,2,128,1,float16,float16,7,0.009894400089979171
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,2,128,1,float16,fp8,7,0.009726399928331375
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,2,128,1,float16,float16,15,0.009931199997663499
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,2,128,1,float16,fp8,15,0.009643200039863586
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,96,8,128,1,float16,fp8,15,0.10325759649276733
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,2,128,1,float16,float16,31,0.010003200173377991
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,2,128,1,float16,float16,63,0.009905599802732468
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,2,128,1,float16,fp8,63,0.009705600142478944
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,2,128,1,float16,float16,127,0.01014079973101616
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,1,128,1,float16,float16,1,0.011388800293207168
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,2,128,1,float16,fp8,127,0.010704000294208527
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,2,128,1,float16,float16,255,0.011383999884128571
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,2,128,1,float16,float16,511,0.013329599797725678
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,2,128,1,float16,fp8,511,0.013764800131320953
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,2,128,1,float16,float16,1023,0.014475199580192565
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,2,128,1,float16,fp8,1023,0.01478559970855713
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,2,128,1,float16,float16,2047,0.016254399716854096
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,2,128,1,float16,fp8,2047,0.016475200653076172
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,4,128,1,float16,float16,1,0.00989919975399971
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,4,128,1,float16,fp8,1,0.009982399642467499
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,4,128,1,float16,float16,3,0.010148800164461135
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,4,128,1,float16,fp8,3,0.010097599774599075
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,4,128,1,float16,float16,7,0.010096000134944915
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,4,128,1,float16,fp8,7,0.010072000324726105
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,4,128,1,float16,float16,15,0.010035199671983719
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,4,128,1,float16,fp8,15,0.009916800260543823
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,4,128,1,float16,float16,31,0.00998080000281334
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,4,128,1,float16,fp8,31,0.009926400333642959
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,4,128,1,float16,float16,63,0.009985599666833878
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,96,4,128,1,float16,float16,63,0.07171199917793274
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,2,128,1,float16,fp8,1,0.009700799733400345
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,4,128,1,float16,fp8,127,0.010732799768447876
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,4,128,1,float16,float16,255,0.011475200206041336
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,4,128,1,float16,fp8,255,0.011432000249624253
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,4,128,1,float16,float16,511,0.013692800700664521
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,4,128,1,float16,fp8,511,0.014120000600814819
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,4,128,1,float16,float16,1023,0.015291200578212738
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,96,8,128,1,float16,float16,31,0.10981279611587524
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,4,128,1,float16,fp8,2047,0.017027199268341064
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,8,128,1,float16,float16,1,0.01005600020289421
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,8,128,1,float16,fp8,1,0.009929600358009338
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,2,128,1,float16,fp8,255,0.011137600243091583
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,8,128,1,float16,float16,3,0.010174400359392165
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,8,128,1,float16,fp8,3,0.010152000188827514
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,8,128,1,float16,float16,7,0.010041599720716476
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,8,128,1,float16,fp8,7,0.010086400061845779
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,8,128,1,float16,float16,15,0.01003360003232956
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,8,128,1,float16,fp8,15,0.010155200213193893
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,8,128,1,float16,float16,31,0.010068800300359726
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,8,128,1,float16,float16,63,0.010331200063228607
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,8,128,1,float16,fp8,63,0.010201600193977357
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,8,128,1,float16,float16,127,0.010627199709415436
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,8,128,1,float16,fp8,127,0.011168000102043153
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,8,128,1,float16,float16,255,0.0118367999792099
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,8,128,1,float16,fp8,255,0.01170559972524643
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,8,128,1,float16,float16,511,0.013801600039005279
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,8,128,1,float16,fp8,511,0.014926399290561675
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,8,128,1,float16,float16,1023,0.01725600063800812
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,8,128,1,float16,fp8,1023,0.01679359972476959
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,8,128,1,float16,float16,2047,0.028803199529647827
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,8,128,1,float16,fp8,2047,0.021848000586032867
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,96,1,128,1,float16,float16,1,0.0610368013381958
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,96,1,128,1,float16,fp8,1,0.06264160275459289
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,4,128,1,float16,float16,127,0.010320000350475311
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,96,1,128,1,float16,float16,3,0.06137279868125915
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,96,1,128,1,float16,fp8,3,0.06300479769706727
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,96,1,128,1,float16,float16,7,0.06130880117416382
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,96,1,128,1,float16,fp8,7,0.06307839751243591
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,96,1,128,1,float16,float16,15,0.06160640120506287
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,96,1,128,1,float16,fp8,15,0.06285920143127441
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,96,1,128,1,float16,float16,31,0.06262879967689514
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,96,1,128,1,float16,fp8,31,0.06422560214996338
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,96,1,128,1,float16,float16,63,0.06477119922637939
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,96,8,128,1,float16,fp8,31,0.10670080184936523
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,96,1,128,1,float16,fp8,63,0.06692799925804138
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,96,2,128,1,float16,float16,1,0.07533599734306336
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,96,2,128,1,float16,fp8,1,0.07191680073738098
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,96,2,128,1,float16,fp8,3,0.0720848023891449
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,96,2,128,1,float16,float16,7,0.07693600058555602
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,96,2,128,1,float16,fp8,7,0.07242559790611267
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,8,128,1,float16,fp8,31,0.01000640019774437
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,96,2,128,1,float16,float16,15,0.07691680192947388
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,96,2,128,1,float16,fp8,15,0.0718608021736145
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,96,2,128,1,float16,float16,31,0.0781216025352478
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,96,2,128,1,float16,fp8,31,0.07457759976387024
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,96,2,128,1,float16,float16,63,0.08229920268058777
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,96,2,128,1,float16,fp8,63,0.07774239778518677
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,96,4,128,1,float16,float16,1,0.11879359483718872
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,96,4,128,1,float16,fp8,1,0.11346880197525025
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,96,4,128,1,float16,float16,3,0.11921600103378296
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,96,4,128,1,float16,fp8,3,0.11426719427108764
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,96,4,128,1,float16,float16,7,0.11965279579162598
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,4,128,1,float16,fp8,63,0.009694399684667588
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,96,4,128,1,float16,fp8,7,0.11399680376052856
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,96,4,128,1,float16,fp8,15,0.11698559522628785
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,96,4,128,1,float16,float16,31,0.12167359590530395
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,96,4,128,1,float16,fp8,31,0.12040799856185913
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,96,4,128,1,float16,float16,63,0.127019202709198
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,96,4,128,1,float16,fp8,63,0.12403199672698975
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,96,8,128,1,float16,float16,1,0.1974239945411682
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,2,128,1,float16,fp8,31,0.009785600006580353
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,96,8,128,1,float16,fp8,1,0.19581279754638672
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,96,8,128,1,float16,float16,3,0.19799679517745972
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,96,2,128,1,float16,float16,3,0.07587839961051941
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,96,8,128,1,float16,fp8,3,0.19524320363998413
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,96,8,128,1,float16,float16,7,0.19886560440063478
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,96,8,128,1,float16,fp8,7,0.19692319631576538
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,96,8,128,1,float16,float16,15,0.20080161094665527
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,96,8,128,1,float16,float16,31,0.20683839321136474
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,96,8,128,1,float16,float16,63,0.22039999961853027
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,96,8,128,1,float16,fp8,63,0.2127552032470703
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,96,1,128,1,float16,float16,1,0.10826879739761353
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,96,1,128,1,float16,fp8,1,0.11649919748306274
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,96,1,128,1,float16,float16,3,0.10812640190124512
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,96,1,128,1,float16,fp8,3,0.11580640077590942
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,96,1,128,1,float16,fp8,7,0.11606240272521973
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,96,1,128,1,float16,float16,15,0.10850720405578614
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,96,1,128,1,float16,fp8,15,0.11612639427185059
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,96,4,128,1,float16,float16,15,0.12053760290145873
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,96,1,128,1,float16,float16,31,0.1099392056465149
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,96,1,128,1,float16,fp8,31,0.11699839830398559
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,96,2,128,1,float16,float16,1,0.13714560270309448
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,96,2,128,1,float16,fp8,1,0.13682880401611328
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,96,2,128,1,float16,float16,3,0.13769279718399047
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,96,2,128,1,float16,float16,7,0.13804639577865602
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,96,2,128,1,float16,fp8,3,0.13717119693756102
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,4,128,1,float16,float16,2047,0.017715199291706084
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,96,2,128,1,float16,float16,15,0.13866720199584961
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,96,2,128,1,float16,fp8,15,0.1368335962295532
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,96,2,128,1,float16,float16,31,0.14061280488967895
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,96,2,128,1,float16,fp8,31,0.13820480108261107
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,96,4,128,1,float16,float16,1,0.22211360931396484
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,96,4,128,1,float16,float16,3,0.22261760234832764
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,96,8,128,1,float16,fp8,15,0.2018336057662964
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,96,8,128,1,float16,fp8,31,0.20672640800476075
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,96,4,128,1,float16,fp8,3,0.22254400253295897
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,96,4,128,1,float16,float16,7,0.22266240119934083
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,96,4,128,1,float16,fp8,7,0.22119519710540772
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,96,4,128,1,float16,float16,15,0.2240544080734253
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,96,4,128,1,float16,fp8,15,0.22404959201812744
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,96,4,128,1,float16,float16,31,0.2283168077468872
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,96,4,128,1,float16,fp8,31,0.22557919025421141
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,96,8,128,1,float16,float16,1,0.3802623987197876
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,96,1,128,1,float16,float16,7,0.10858399868011474
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,96,8,128,1,float16,float16,3,0.38155839443206785
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,96,8,128,1,float16,fp8,3,0.3860320091247559
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,96,8,128,1,float16,float16,7,0.3806591987609863
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,96,8,128,1,float16,fp8,7,0.38770880699157717
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,96,8,128,1,float16,float16,15,0.38851521015167234
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,96,8,128,1,float16,fp8,15,0.3900576114654541
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,1,128,1,float16,float16,1,0.011980800330638886
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,1,128,1,float16,fp8,1,0.012283200025558471
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,1,128,1,float16,float16,3,0.011748799681663513
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,1,128,1,float16,fp8,3,0.012256000190973282
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,1,128,1,float16,float16,7,0.011857599765062333
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,1,128,1,float16,fp8,7,0.012088000029325485
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,96,8,128,1,float16,fp8,31,0.39720959663391114
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,1,128,1,float16,float16,15,0.01194079965353012
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,1,128,1,float16,fp8,15,0.012150400131940842
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,1,128,1,float16,fp8,31,0.012008000165224075
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,1,128,1,float16,float16,63,0.011847999691963196
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,1,128,1,float16,fp8,63,0.012089599668979645
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,1,128,1,float16,float16,127,0.011856000125408172
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,1,128,1,float16,fp8,127,0.012110400199890136
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,1,128,1,float16,float16,255,0.013124799728393555
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,96,2,128,1,float16,fp8,7,0.1364799976348877
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,96,4,128,1,float16,fp8,1023,0.015115199983119965
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,1,128,1,float16,fp8,511,0.016840000450611115
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,1,128,1,float16,float16,1023,0.017028799653053282
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,1,128,1,float16,fp8,1023,0.0174575999379158
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,1,128,1,float16,float16,2047,0.01958560049533844
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,1,128,1,float16,fp8,2047,0.019644799828529357
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,2,128,1,float16,float16,1,0.010233599692583084
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,2,128,1,float16,fp8,1,0.010283199697732925
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,96,4,128,1,float16,fp8,1,0.22212319374084472
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,2,128,1,float16,fp8,3,0.009860800206661224
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,2,128,1,float16,float16,7,0.010443200170993806
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,2,128,1,float16,fp8,7,0.010172799974679948
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,2,128,1,float16,float16,15,0.009878399968147277
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,2,128,1,float16,fp8,15,0.009726399928331375
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,2,128,1,float16,float16,31,0.01000640019774437
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,2,128,1,float16,fp8,31,0.010174400359392165
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,2,128,1,float16,float16,63,0.010416000336408614
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,2,128,1,float16,fp8,63,0.010201600193977357
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,2,128,1,float16,float16,127,0.010654400289058685
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,96,8,128,1,float16,fp8,1,0.38821120262145997
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,2,128,1,float16,fp8,127,0.0110944002866745
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,2,128,1,float16,fp8,255,0.01149279996752739
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,2,128,1,float16,float16,255,0.011771199852228164
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,2,128,1,float16,float16,511,0.013726399838924408
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,2,128,1,float16,fp8,511,0.015150399506092071
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,2,128,1,float16,float16,1023,0.015992000699043274
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,2,128,1,float16,fp8,1023,0.01571680009365082
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,2,128,1,float16,float16,2047,0.018611200153827667
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,2,128,1,float16,fp8,2047,0.018313600122928618
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,4,128,1,float16,float16,1,0.010292799770832061
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,4,128,1,float16,fp8,1,0.009976000338792802
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,4,128,1,float16,float16,3,0.010307200253009796
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,4,128,1,float16,fp8,3,0.009935999661684037
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,4,128,1,float16,float16,7,0.010571199655532836
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,4,128,1,float16,fp8,7,0.009956800192594529
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,4,128,1,float16,float16,15,0.01045759990811348
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,96,8,128,1,float16,float16,31,0.39703199863433836
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,4,128,1,float16,fp8,15,0.01014079973101616
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,4,128,1,float16,float16,31,0.01048479974269867
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,4,128,1,float16,fp8,31,0.010344000160694122
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,4,128,1,float16,float16,63,0.010211200267076493
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,1,128,1,float16,float16,31,0.011928000301122666
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,4,128,1,float16,fp8,127,0.010887999832630158
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,4,128,1,float16,float16,255,0.01210559979081154
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,4,128,1,float16,fp8,255,0.011667200177907944
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,4,128,1,float16,float16,511,0.013940800726413728
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,4,128,1,float16,fp8,511,0.015171200037002563
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,1,128,1,float16,fp8,255,0.013415999710559845
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,1,128,1,float16,float16,511,0.01637600064277649
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,4,128,1,float16,float16,1023,0.017564800381660462
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,4,128,1,float16,fp8,1023,0.01740639954805374
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,4,128,1,float16,float16,2047,0.029071998596191407
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,8,128,1,float16,float16,1,0.010395199805498124
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,4,128,1,float16,fp8,2047,0.02210720032453537
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,8,128,1,float16,fp8,3,0.010155200213193893
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,8,128,1,float16,float16,3,0.01040479987859726
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,8,128,1,float16,float16,7,0.010550399869680404
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,8,128,1,float16,fp8,7,0.010366400331258773
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,2,128,1,float16,float16,3,0.010179200023412705
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,8,128,1,float16,float16,15,0.010555200278759003
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,8,128,1,float16,fp8,15,0.010412800312042236
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,8,128,1,float16,float16,31,0.010462400317192078
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,8,128,1,float16,float16,63,0.010822399705648422
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,8,128,1,float16,float16,127,0.011078400164842605
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,8,128,1,float16,float16,255,0.012289600074291229
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,8,128,1,float16,fp8,255,0.012049599736928939
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,8,128,1,float16,float16,511,0.014467200636863709
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,8,128,1,float16,fp8,511,0.01555040031671524
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,8,128,1,float16,float16,1023,0.027489599585533143
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,8,128,1,float16,fp8,1023,0.019991999864578246
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,8,128,1,float16,float16,2047,0.04273920059204102
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,8,128,1,float16,fp8,2047,0.03139039874076843
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,1,128,1,float16,float16,1,0.011681599915027619
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,1,128,1,float16,fp8,1,0.011427199840545655
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,1,128,1,float16,float16,3,0.011740799993276596
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,1,128,1,float16,fp8,3,0.011473599821329117
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,1,128,1,float16,float16,7,0.011777599900960922
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,1,128,1,float16,fp8,7,0.011552000045776367
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,1,128,1,float16,float16,15,0.011627200245857238
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,1,128,1,float16,fp8,15,0.011524800211191177
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,1,128,1,float16,float16,31,0.011806400120258331
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,1,128,1,float16,fp8,31,0.011436799913644791
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,1,128,1,float16,float16,63,0.01183520033955574
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,4,128,1,float16,fp8,63,0.010159999877214432
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,4,128,1,float16,float16,127,0.010756800323724747
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,1,128,1,float16,float16,127,0.01138240024447441
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,1,128,1,float16,fp8,127,0.012433599680662155
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,1,128,1,float16,float16,255,0.013118399679660797
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,1,128,1,float16,fp8,255,0.012980799376964568
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,1,128,1,float16,fp8,511,0.01713919937610626
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,8,128,1,float16,fp8,1,0.010158400237560272
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,1,128,1,float16,float16,1023,0.02009280025959015
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,1,128,1,float16,fp8,1023,0.0193231999874115
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,1,128,1,float16,float16,2047,0.03297759890556336
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,1,128,1,float16,fp8,2047,0.024247999489307403
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,8,128,1,float16,fp8,127,0.011513599753379821
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,2,128,1,float16,float16,1,0.011547199636697768
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,2,128,1,float16,fp8,1,0.011483199894428253
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,1,128,1,float16,fp8,4095,0.03649600148200989
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,2,128,1,float16,float16,3,0.011468800157308579
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,2,128,1,float16,fp8,3,0.011503999680280685
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,2,128,1,float16,float16,7,0.011603199690580369
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,2,128,1,float16,fp8,7,0.011383999884128571
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,2,128,1,float16,float16,15,0.011740799993276596
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,2,128,1,float16,fp8,15,0.011510399729013443
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,2,128,1,float16,float16,31,0.011816000193357467
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,2,128,1,float16,fp8,63,0.011633600294589996
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,2,128,1,float16,float16,127,0.01210559979081154
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,1,128,1,float16,fp8,63,0.011430399864912033
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,2,128,1,float16,fp8,127,0.012727999687194824
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,2,128,1,float16,float16,255,0.013433599472045898
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,1,128,1,float16,float16,511,0.014979200065135955
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,2,128,1,float16,fp8,255,0.013041600584983826
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,2,128,1,float16,float16,511,0.01599839925765991
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,2,128,1,float16,fp8,511,0.01706400066614151
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,2,128,1,float16,float16,1023,0.02943840026855469
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,2,128,1,float16,fp8,1023,0.021524800360202788
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,2,128,1,float16,fp8,2047,0.033083200454711914
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,8,128,1,float16,fp8,31,0.010196799784898758
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,2,128,1,float16,float16,4095,0.07514879703521729
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,2,128,1,float16,fp8,4095,0.05231680274009705
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,1,128,1,float16,float16,4095,0.0486160010099411
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,4,128,1,float16,fp8,1,0.013998399674892425
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,4,128,1,float16,float16,3,0.014142400026321411
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,4,128,1,float16,fp8,3,0.014056000113487243
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,4,128,1,float16,float16,7,0.014441600441932679
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,4,128,1,float16,fp8,7,0.014113600552082061
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,2,128,1,float16,fp8,31,0.011788800358772278
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,4,128,1,float16,float16,15,0.01432960033416748
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,4,128,1,float16,fp8,15,0.014075200259685516
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,4,128,1,float16,fp8,31,0.014259199798107147
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,4,128,1,float16,float16,31,0.01440960019826889
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,4,128,1,float16,fp8,63,0.014054399728775025
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,4,128,1,float16,float16,127,0.014905600249767304
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,4,128,1,float16,fp8,127,0.015937599539756774
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,4,128,1,float16,float16,255,0.017632000148296356
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,4,128,1,float16,fp8,255,0.017638400197029114
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,4,128,1,float16,fp8,511,0.024107199907302857
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,4,128,1,float16,float16,511,0.030248001217842102
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,4,128,1,float16,float16,1023,0.0475488007068634
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,4,128,1,float16,float16,2047,0.07712479829788207
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,4,128,1,float16,fp8,2047,0.05474720001220703
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,4,128,1,float16,float16,4095,0.1367087960243225
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,4,128,1,float16,fp8,4095,0.09207839965820312
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,8,128,1,float16,float16,1,0.019662399590015412
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,8,128,1,float16,fp8,1,0.019575999677181245
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,8,128,1,float16,fp8,3,0.01956319957971573
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,8,128,1,float16,float16,7,0.019849599897861482
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,8,128,1,float16,fp8,7,0.019636799395084382
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,2,128,1,float16,float16,2047,0.04480000138282776
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,96,8,128,1,float16,fp8,63,0.010564800351858139
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,8,128,1,float16,float16,15,0.02008160054683685
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,8,128,1,float16,fp8,15,0.019739200174808503
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,8,128,1,float16,fp8,31,0.01977120041847229
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,8,128,1,float16,float16,63,0.020342400670051573
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,8,128,1,float16,fp8,63,0.01991039961576462
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,4,128,1,float16,float16,1,0.014127999544143677
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,8,128,1,float16,float16,127,0.020363199710845947
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,8,128,1,float16,fp8,127,0.023188799619674683
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,8,128,1,float16,float16,255,0.03344320058822632
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,8,128,1,float16,fp8,255,0.026499199867248534
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,2,128,1,float16,float16,63,0.011614400148391723
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,8,128,1,float16,float16,511,0.04811199903488159
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,8,128,1,float16,float16,1023,0.08000800013542175
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,4,128,1,float16,float16,63,0.014182400703430176
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,8,128,1,float16,fp8,1023,0.0589743971824646
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,8,128,1,float16,float16,2047,0.13871359825134277
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,1,128,1,float16,float16,1,0.009560000151395798
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,1,128,1,float16,fp8,1,0.009222400188446046
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,8,128,1,float16,float16,4095,0.25420000553131106
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,1,128,1,float16,float16,3,0.009595199674367904
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,4,128,1,float16,fp8,1023,0.03555200099945068
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,1,128,1,float16,float16,7,0.009585600346326828
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,1,128,1,float16,float16,15,0.010062400251626968
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,1,128,1,float16,fp8,7,0.009270399808883667
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,1,128,1,float16,fp8,15,0.009433600306510925
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,1,128,1,float16,float16,31,0.00971359983086586
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,1,128,1,float16,float16,63,0.009752000123262406
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,1,128,1,float16,fp8,63,0.009513600170612336
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,1,128,1,float16,float16,127,0.00981760025024414
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,1,128,1,float16,fp8,127,0.010227199643850327
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,1,128,1,float16,float16,255,0.011017599701881408
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,1,128,1,float16,fp8,255,0.010967999696731567
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,1,128,1,float16,float16,511,0.013148799538612366
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,1,128,1,float16,fp8,511,0.012316799908876418
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,1,128,1,float16,float16,1023,0.013526399433612824
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,1,128,1,float16,fp8,1023,0.013027200102806091
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,1,128,1,float16,float16,2047,0.014028799533843995
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,1,128,1,float16,fp8,2047,0.013627199828624726
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,1,128,1,float16,float16,4095,0.014718399941921234
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,1,128,1,float16,fp8,4095,0.014603200554847717
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,2,128,1,float16,float16,1,0.009508799761533737
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,2,128,1,float16,fp8,1,0.009560000151395798
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,2,128,1,float16,float16,3,0.009654399752616883
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,8,128,1,float16,fp8,511,0.0409168004989624
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,2,128,1,float16,fp8,3,0.009595199674367904
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,2,128,1,float16,float16,7,0.009729599952697754
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,2,128,1,float16,fp8,7,0.009536000341176987
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,2,128,1,float16,float16,15,0.009737599641084671
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,2,128,1,float16,fp8,15,0.009510400146245957
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,2,128,1,float16,float16,31,0.009656000137329101
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,2,128,1,float16,fp8,31,0.009556800127029419
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,2,128,1,float16,float16,63,0.009572800248861313
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,2,128,1,float16,fp8,63,0.009363199770450591
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,2,128,1,float16,float16,127,0.009583999961614608
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,2,128,1,float16,fp8,127,0.009958399832248688
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,8,128,1,float16,fp8,2047,0.09812639951705933
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,2,128,1,float16,float16,255,0.010785599797964096
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,2,128,1,float16,fp8,255,0.011022400110960007
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,2,128,1,float16,float16,511,0.013180799782276154
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,8,128,1,float16,fp8,4095,0.1724063992500305
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,2,128,1,float16,float16,1023,0.013529600203037262
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,2,128,1,float16,float16,2047,0.013617600500583648
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,2,128,1,float16,fp8,1023,0.012948800623416901
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,1,128,1,float16,fp8,3,0.009233599901199341
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,2,128,1,float16,float16,4095,0.014796799421310425
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,2,128,1,float16,fp8,4095,0.014363199472427368
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,4,128,1,float16,float16,1,0.00971359983086586
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,1,128,1,float16,fp8,31,0.009403199702501298
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,4,128,1,float16,float16,3,0.009697599709033966
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,4,128,1,float16,fp8,3,0.009371200203895569
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,4,128,1,float16,float16,7,0.009683199971914292
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,4,128,1,float16,fp8,7,0.009302400052547455
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,4,128,1,float16,float16,15,0.00971359983086586
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,4,128,1,float16,float16,31,0.00974079966545105
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,4,128,1,float16,fp8,31,0.009358400106430053
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,4,128,1,float16,float16,63,0.009603200107812881
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,4,128,1,float16,fp8,63,0.009455999732017517
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,4,128,1,float16,float16,127,0.009756799787282944
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,4,128,1,float16,fp8,127,0.010179200023412705
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,4,128,1,float16,float16,255,0.010883200168609618
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,4,128,1,float16,fp8,255,0.01098880022764206
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,4,128,1,float16,float16,511,0.01305440068244934
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,4,128,1,float16,fp8,511,0.012683199346065521
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,8,128,1,float16,float16,31,0.020193600654602052
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,4,128,1,float16,float16,1023,0.01358720064163208
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,4,128,1,float16,fp8,1023,0.012908799946308136
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,4,128,1,float16,fp8,2047,0.013371199369430542
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,4,128,1,float16,float16,4095,0.014795200526714325
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,4,128,1,float16,fp8,4095,0.01475519984960556
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,8,128,1,float16,float16,1,0.009833600372076035
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,8,128,1,float16,fp8,1,0.009676799923181535
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,8,128,1,float16,float16,3,0.00987040027976036
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,8,128,1,float16,fp8,3,0.009702400118112565
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,8,128,1,float16,float16,7,0.009880000352859497
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,8,128,1,float16,fp8,7,0.009785600006580353
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,8,128,1,float16,float16,15,0.009771200269460678
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,8,128,1,float16,fp8,15,0.009598399698734283
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,2,128,1,float16,fp8,511,0.01210559979081154
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,8,128,1,float16,float16,31,0.009857600182294845
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,8,128,1,float16,fp8,31,0.009480000287294389
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,8,128,1,float16,float16,63,0.009857600182294845
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,8,128,1,float16,fp8,63,0.0095551997423172
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,8,128,1,float16,float16,127,0.009961599856615067
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,8,128,1,float16,fp8,127,0.010163199901580811
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,8,128,1,float16,float16,255,0.010931199789047242
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,8,128,1,float16,fp8,255,0.0112527996301651
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,8,128,1,float16,float16,511,0.012758399546146392
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,8,128,1,float16,fp8,511,0.01276959925889969
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,8,128,1,float16,float16,1023,0.013387200236320496
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,8,128,1,float16,fp8,1023,0.013143999874591828
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,8,128,1,float16,float16,2047,0.014107200503349304
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,8,128,1,float16,fp8,2047,0.014070400595664978
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,8,128,1,float16,float16,4095,0.015414400398731232
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,8,128,1,float16,fp8,4095,0.014684799313545226
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,1,128,1,float16,float16,1,0.00984319970011711
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,4,128,1,float16,fp8,15,0.009247999638319016
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,1,128,1,float16,fp8,1,0.009377600252628326
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,1,128,1,float16,fp8,3,0.009345600008964538
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,1,128,1,float16,fp8,7,0.009403199702501298
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,1,128,1,float16,float16,15,0.009585600346326828
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,1,128,1,float16,fp8,15,0.00928959995508194
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,64,8,128,1,float16,float16,3,0.019793599843978882
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,1,128,1,float16,float16,31,0.00973920002579689
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,1,128,1,float16,float16,63,0.009784000366926194
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,1,128,1,float16,fp8,31,0.00947519987821579
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,1,128,1,float16,fp8,63,0.009452799707651139
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,1,128,1,float16,float16,127,0.009745600074529648
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,1,128,1,float16,fp8,127,0.010091199725866317
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,1,128,1,float16,float16,255,0.010873600095510482
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,1,128,1,float16,fp8,255,0.010878399759531022
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,1,128,1,float16,float16,511,0.012918399274349212
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,1,128,1,float16,fp8,511,0.012459199875593185
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,1,128,1,float16,float16,1023,0.013630400598049163
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,1,128,1,float16,fp8,1023,0.013172799348831176
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,4,128,1,float16,float16,2047,0.013991999626159667
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,1,128,1,float16,fp8,2047,0.013684800267219544
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,1,128,1,float16,float16,4095,0.015214399993419647
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,1,128,1,float16,fp8,4095,0.014787200093269347
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,2,128,1,float16,float16,1,0.00963039994239807
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,2,128,1,float16,fp8,1,0.009470400214195252
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,2,128,1,float16,fp8,3,0.009270399808883667
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,2,128,1,float16,fp8,7,0.009270399808883667
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,2,128,1,float16,float16,7,0.009537599980831146
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,2,128,1,float16,float16,15,0.009471999853849411
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,2,128,1,float16,fp8,15,0.009513600170612336
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,2,128,1,float16,float16,31,0.009536000341176987
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,2,128,1,float16,fp8,31,0.009523200243711472
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,2,128,1,float16,float16,63,0.009520000219345093
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,2,128,1,float16,fp8,63,0.00950080007314682
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,2,128,1,float16,float16,127,0.009646400064229965
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,2,128,1,float16,fp8,127,0.010148800164461135
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,2,128,1,float16,float16,255,0.010900799930095673
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,2,128,1,float16,fp8,255,0.0110895998775959
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,2,128,1,float16,float16,511,0.013030399382114411
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,2,128,1,float16,fp8,511,0.012337599694728852
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,2,128,1,float16,float16,1023,0.013600000739097595
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,2,128,1,float16,fp8,1023,0.01295360028743744
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,2,128,1,float16,float16,2047,0.013947199285030364
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,2,128,1,float16,fp8,2047,0.01329600065946579
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,2,128,1,float16,fp8,2047,0.013025599718093871
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,2,128,1,float16,fp8,4095,0.015072000026702882
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,4,128,1,float16,float16,1,0.009507200121879578
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,64,4,128,1,float16,fp8,1,0.009283199906349182
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,4,128,1,float16,fp8,1,0.009345600008964538
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,4,128,1,float16,float16,3,0.009703999757766724
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,4,128,1,float16,fp8,3,0.009484799951314926
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,1,128,1,float16,float16,3,0.009812799841165542
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,4,128,1,float16,fp8,7,0.009470400214195252
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,1,128,1,float16,float16,7,0.009560000151395798
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,4,128,1,float16,fp8,15,0.00963039994239807
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,4,128,1,float16,float16,31,0.009747199714183807
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,4,128,1,float16,fp8,31,0.009627199918031692
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,4,128,1,float16,float16,63,0.009827200323343277
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,4,128,1,float16,fp8,63,0.009574399888515472
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,4,128,1,float16,float16,127,0.009787199646234512
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,4,128,1,float16,fp8,127,0.010126399993896484
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,4,128,1,float16,float16,255,0.010952000319957734
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,4,128,1,float16,fp8,255,0.010990399867296219
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,4,128,1,float16,float16,511,0.013096000254154205
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,4,128,1,float16,fp8,511,0.012479999661445617
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,4,128,1,float16,float16,1023,0.013633599877357483
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,4,128,1,float16,fp8,1023,0.013099199533462525
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,4,128,1,float16,fp8,2047,0.014248000085353851
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,4,128,1,float16,float16,4095,0.015732799470424653
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,1,128,1,float16,float16,2047,0.01401440054178238
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,4,128,1,float16,fp8,4095,0.014892800152301789
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,8,128,1,float16,float16,1,0.00987040027976036
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,8,128,1,float16,fp8,1,0.00952960029244423
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,8,128,1,float16,float16,3,0.009859199821949004
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,8,128,1,float16,fp8,3,0.009532800316810608
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,2,128,1,float16,float16,3,0.009670399874448777
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,8,128,1,float16,fp8,7,0.009656000137329101
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,8,128,1,float16,fp8,15,0.009641599655151368
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,8,128,1,float16,float16,31,0.009750399738550186
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,8,128,1,float16,fp8,31,0.009616000205278396
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,8,128,1,float16,float16,63,0.009763199836015701
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,8,128,1,float16,fp8,63,0.009518399834632874
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,8,128,1,float16,float16,127,0.009854400157928466
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,8,128,1,float16,fp8,127,0.01061279997229576
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,8,128,1,float16,float16,255,0.011451199650764465
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,8,128,1,float16,fp8,255,0.011036799848079681
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,8,128,1,float16,float16,511,0.01329440027475357
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,2,128,1,float16,float16,4095,0.014870400726795196
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,8,128,1,float16,fp8,511,0.01279360055923462
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,8,128,1,float16,float16,1023,0.013995200395584106
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,8,128,1,float16,float16,2047,0.015305599570274353
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,8,128,1,float16,fp8,2047,0.014127999544143677
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,8,128,1,float16,float16,4095,0.017283199727535246
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,8,128,1,float16,fp8,4095,0.016551999747753142
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,4,128,1,float16,float16,7,0.009827200323343277
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,1,128,1,float16,float16,1,0.010716799646615982
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,4,128,1,float16,float16,15,0.009593600034713745
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,1,128,1,float16,fp8,3,0.010384000092744827
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,1,128,1,float16,float16,7,0.010377600044012069
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,1,128,1,float16,fp8,7,0.010393600165843963
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,1,128,1,float16,float16,15,0.01074879989027977
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,1,128,1,float16,fp8,15,0.010367999970912933
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,1,128,1,float16,float16,31,0.010887999832630158
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,1,128,1,float16,fp8,31,0.010630399733781815
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,1,128,1,float16,float16,63,0.01063840016722679
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,1,128,1,float16,fp8,63,0.010507199913263321
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,1,128,1,float16,float16,127,0.011126399785280228
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,1,128,1,float16,fp8,127,0.01127680018544197
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,1,128,1,float16,float16,255,0.012408000230789185
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,1,128,1,float16,fp8,255,0.012257599830627441
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,1,128,1,float16,float16,511,0.01454080045223236
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,1,128,1,float16,fp8,511,0.015967999398708344
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,1,128,1,float16,float16,1023,0.01727840006351471
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,1,128,1,float16,fp8,1023,0.016519999504089354
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,1,128,1,float16,float16,2047,0.01950239986181259
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,1,128,1,float16,fp8,2047,0.019091199338436126
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,1,128,1,float16,float16,4095,0.03131360113620758
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,8,128,1,float16,float16,7,0.009867200255393982
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,8,128,1,float16,float16,15,0.009782399982213974
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,2,128,1,float16,float16,3,0.01069599986076355
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,2,128,1,float16,fp8,3,0.010907199978828431
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,2,128,1,float16,float16,7,0.010871999710798264
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,2,128,1,float16,fp8,7,0.010734400153160096
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,2,128,1,float16,float16,15,0.010899200290441512
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,2,128,1,float16,fp8,15,0.010761599987745285
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,2,128,1,float16,float16,31,0.011007999628782272
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,2,128,1,float16,fp8,31,0.010764800012111664
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,2,128,1,float16,float16,63,0.010918399691581726
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,2,128,1,float16,fp8,63,0.010632000118494033
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,8,128,1,float16,fp8,1023,0.013443200290203095
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,2,128,1,float16,float16,127,0.011268799751996994
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,2,128,1,float16,fp8,127,0.01199359968304634
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,2,128,1,float16,float16,255,0.012518399953842163
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,2,128,1,float16,fp8,255,0.012392000108957291
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,2,128,1,float16,float16,511,0.014654399454593658
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,2,128,1,float16,fp8,511,0.015787200629711153
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,2,128,1,float16,float16,1023,0.018401600420475006
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,2,128,1,float16,fp8,1023,0.018111999332904815
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,2,128,1,float16,float16,2047,0.03067840039730072
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,2,128,1,float16,fp8,2047,0.02287199944257736
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,1,128,1,float16,float16,3,0.010841599851846694
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,2,128,1,float16,float16,4095,0.04521439969539642
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,4,128,1,float16,float16,1,0.011062400043010711
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,2,128,1,float16,fp8,4095,0.034176000952720643
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,4,128,1,float16,fp8,1,0.011052799969911575
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,4,128,1,float16,float16,3,0.010843200236558914
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,4,128,1,float16,fp8,3,0.011043199896812439
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,4,128,1,float16,float16,7,0.010971199721097946
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,4,128,1,float16,fp8,7,0.010836800187826156
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,4,128,1,float16,float16,15,0.010899200290441512
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,4,128,1,float16,fp8,15,0.010915199667215348
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,4,128,1,float16,float16,31,0.01093600019812584
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,4,128,1,float16,fp8,31,0.010659199953079224
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,64,4,128,1,float16,float16,2047,0.014188799262046813
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,4,128,1,float16,fp8,63,0.01101280003786087
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,4,128,1,float16,float16,127,0.011363200098276138
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,4,128,1,float16,fp8,127,0.011655999720096588
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,4,128,1,float16,float16,255,0.012511999905109405
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,4,128,1,float16,fp8,255,0.012518399953842163
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,4,128,1,float16,float16,511,0.01496960073709488
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,1,128,1,float16,fp8,4095,0.023688000440597535
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,4,128,1,float16,float16,1023,0.02831999957561493
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,2,128,1,float16,float16,1,0.010860799998044967
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,4,128,1,float16,fp8,1023,0.020392000675201416
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,2,128,1,float16,fp8,1,0.0104592002928257
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,4,128,1,float16,fp8,2047,0.031913599371910094
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,4,128,1,float16,float16,4095,0.07383999824523926
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,8,128,1,float16,float16,1,0.013571199774742127
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,8,128,1,float16,fp8,1,0.013223999738693237
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,8,128,1,float16,float16,3,0.013475200533866883
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,8,128,1,float16,fp8,3,0.013406400382518769
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,8,128,1,float16,float16,7,0.01361439973115921
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,8,128,1,float16,fp8,7,0.013400000333786011
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,8,128,1,float16,float16,15,0.013582399487495423
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,8,128,1,float16,fp8,15,0.013409599661827087
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,8,128,1,float16,float16,31,0.013631999492645264
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,8,128,1,float16,fp8,31,0.01345600038766861
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,8,128,1,float16,float16,63,0.01356319934129715
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,8,128,1,float16,fp8,63,0.013300800323486328
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,8,128,1,float16,float16,127,0.013926400244235993
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,8,128,1,float16,fp8,127,0.015039999783039094
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,8,128,1,float16,float16,255,0.017159999907016756
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,8,128,1,float16,fp8,255,0.01677599996328354
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,8,128,1,float16,float16,511,0.02858879864215851
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,8,128,1,float16,fp8,511,0.023179200291633607
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,8,128,1,float16,float16,1023,0.04509919881820679
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,8,128,1,float16,fp8,1023,0.03402880132198334
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,8,128,1,float16,float16,2047,0.07509279847145081
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,8,128,1,float16,float16,4095,0.1334704041481018
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,1,128,1,float16,fp8,1,0.010567999631166457
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,1,128,1,float16,float16,1,0.00958240032196045
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,1,128,1,float16,fp8,1,0.00933919996023178
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,1,128,1,float16,float16,3,0.009678400307893752
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,1,128,1,float16,fp8,3,0.009256000071763993
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,1,128,1,float16,float16,7,0.009470400214195252
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,1,128,1,float16,fp8,7,0.009246399998664856
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,4,128,1,float16,float16,63,0.011099199950695037
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,1,128,1,float16,float16,15,0.009495999664068222
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,1,128,1,float16,float16,31,0.009516800194978714
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,1,128,1,float16,fp8,15,0.009520000219345093
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,1,128,1,float16,fp8,31,0.009246399998664856
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,1,128,1,float16,fp8,63,0.009435199946165086
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,1,128,1,float16,float16,127,0.009724800288677216
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,4,128,1,float16,fp8,511,0.015964800119400026
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,1,128,1,float16,fp8,127,0.010156799852848054
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,1,128,1,float16,fp8,255,0.010992000252008438
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,1,128,1,float16,float16,511,0.013022400438785553
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,4,128,1,float16,float16,2047,0.04366079866886139
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,1,128,1,float16,fp8,511,0.012878400087356568
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,1,128,1,float16,float16,1023,0.013662399351596832
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,1,128,1,float16,fp8,1023,0.01319040060043335
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,1,128,1,float16,float16,2047,0.014206400513648987
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,1,128,1,float16,fp8,2047,0.01364160031080246
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,1,128,1,float16,float16,4095,0.015355199575424194
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,1,128,1,float16,fp8,4095,0.015121600031852723
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,2,128,1,float16,float16,1,0.009755200147628785
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,2,128,1,float16,fp8,1,0.009433600306510925
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,2,128,1,float16,float16,3,0.009804800152778625
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,2,128,1,float16,fp8,3,0.009545599669218063
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,2,128,1,float16,float16,7,0.009784000366926194
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,2,128,1,float16,fp8,7,0.009542399644851684
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,2,128,1,float16,float16,15,0.009652800112962722
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,2,128,1,float16,fp8,15,0.009520000219345093
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,2,128,1,float16,float16,31,0.009614399820566177
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,2,128,1,float16,fp8,31,0.009536000341176987
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,2,128,1,float16,float16,63,0.009652800112962722
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,2,128,1,float16,fp8,63,0.009431999921798707
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,2,128,1,float16,float16,127,0.00968960002064705
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,2,128,1,float16,fp8,127,0.010073599964380264
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,2,128,1,float16,float16,255,0.01101600006222725
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,2,128,1,float16,fp8,255,0.010958400368690491
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,2,128,1,float16,float16,511,0.013275200128555298
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,2,128,1,float16,fp8,511,0.012667199969291687
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,2,128,1,float16,float16,1023,0.01377280056476593
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,2,128,1,float16,fp8,1023,0.013097600638866424
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,2,128,1,float16,float16,2047,0.014454400539398194
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,8,128,1,float16,fp8,2047,0.053595197200775144
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,2,128,1,float16,fp8,2047,0.014027200639247894
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,2,128,1,float16,float16,4095,0.01576800048351288
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,2,128,1,float16,fp8,4095,0.015438400208950043
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,4,128,1,float16,float16,3,0.009761600196361542
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,4,128,1,float16,fp8,3,0.00931359976530075
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,8,128,1,float16,fp8,4095,0.09091359972953797
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,4,128,1,float16,fp8,7,0.00936800017952919
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,4,128,1,float16,float16,15,0.009700799733400345
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,4,128,1,float16,fp8,15,0.009531199932098389
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,4,128,1,float16,float16,31,0.0097680002450943
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,4,128,1,float16,fp8,31,0.009590400010347366
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,4,128,1,float16,float16,63,0.009878399968147277
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,1,128,1,float16,float16,63,0.009726399928331375
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,4,128,1,float16,fp8,63,0.009399999678134919
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,4,128,1,float16,fp8,127,0.009972800314426423
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,4,128,1,float16,float16,127,0.009875199943780898
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,4,128,1,float16,float16,255,0.010862399637699128
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,4,128,1,float16,fp8,255,0.011390399932861329
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,4,128,1,float16,float16,511,0.013187199831008911
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,4,128,1,float16,fp8,511,0.012798400223255157
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,1,128,1,float16,float16,255,0.010849600285291671
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,4,128,1,float16,float16,1023,0.013937599956989288
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,4,128,1,float16,fp8,1023,0.01342719942331314
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,4,128,1,float16,float16,2047,0.015464000403881073
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,4,128,1,float16,float16,4095,0.017556799948215483
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,4,128,1,float16,fp8,4095,0.01735360026359558
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,64,4,128,1,float16,fp8,4095,0.050780802965164185
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,8,128,1,float16,fp8,1,0.00963359996676445
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,8,128,1,float16,fp8,3,0.009600000083446502
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,8,128,1,float16,float16,7,0.009753599762916565
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,8,128,1,float16,fp8,7,0.009755200147628785
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,8,128,1,float16,float16,15,0.009737599641084671
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,8,128,1,float16,fp8,15,0.010022400319576264
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,8,128,1,float16,float16,31,0.009857600182294845
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,8,128,1,float16,fp8,31,0.009694399684667588
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,8,128,1,float16,float16,63,0.009782399982213974
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,8,128,1,float16,fp8,63,0.00992320030927658
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,8,128,1,float16,float16,127,0.010046400129795074
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,8,128,1,float16,fp8,127,0.01069599986076355
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,8,128,1,float16,float16,255,0.0115167997777462
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,4,128,1,float16,float16,1,0.009676799923181535
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,4,128,1,float16,fp8,1,0.009651199728250504
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,8,128,1,float16,fp8,255,0.012323199957609176
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,4,128,1,float16,float16,7,0.009692800045013428
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,8,128,1,float16,float16,511,0.013468800485134125
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,8,128,1,float16,float16,1023,0.015049600601196289
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,8,128,1,float16,fp8,1023,0.014148800075054169
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,8,128,1,float16,fp8,2047,0.01650719940662384
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,8,128,1,float16,float16,4095,0.02860960066318512
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,1,128,1,float16,float16,1,0.013033600151538849
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,1,128,1,float16,fp8,1,0.012571200728416443
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,1,128,1,float16,float16,3,0.012615999579429627
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,1,128,1,float16,fp8,3,0.012639999389648438
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,1,128,1,float16,float16,7,0.012691199779510498
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,1,128,1,float16,fp8,7,0.012630400061607362
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,1,128,1,float16,float16,15,0.01276479959487915
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,1,128,1,float16,fp8,15,0.012763200700283051
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,4,128,1,float16,fp8,2047,0.014603200554847717
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,1,128,1,float16,fp8,31,0.012646399438381195
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,8,128,1,float16,float16,1,0.009771200269460678
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,1,128,1,float16,float16,63,0.013312000036239623
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,8,128,1,float16,float16,3,0.00968639999628067
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,1,128,1,float16,fp8,127,0.013939200341701508
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,1,128,1,float16,float16,255,0.014892800152301789
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,1,128,1,float16,float16,511,0.018291200697422027
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,1,128,1,float16,fp8,511,0.018827199935913086
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,1,128,1,float16,float16,1023,0.03152480125427246
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,1,128,1,float16,fp8,1023,0.023364800214767455
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,1,128,1,float16,float16,2047,0.046491199731826784
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,1,128,1,float16,fp8,2047,0.035132798552513125
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,2,128,1,float16,float16,1,0.015638400614261628
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,2,128,1,float16,fp8,1,0.015614399313926696
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,2,128,1,float16,float16,3,0.015795199573040007
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,8,128,1,float16,fp8,511,0.013302400708198547
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,2,128,1,float16,fp8,3,0.015611200034618378
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,8,128,1,float16,float16,2047,0.016548800468444824
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,2,128,1,float16,float16,7,0.015353600680828094
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,2,128,1,float16,fp8,7,0.015745599567890168
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,2,128,1,float16,float16,15,0.015779200196266174
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,2,128,1,float16,fp8,15,0.015441599488258361
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,2,128,1,float16,float16,31,0.01602399945259094
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,2,128,1,float16,fp8,31,0.015836800634860992
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,2,128,1,float16,float16,63,0.015595200657844543
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,2,128,1,float16,fp8,63,0.01564960032701492
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,2,128,1,float16,float16,127,0.016495999693870545
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,2,128,1,float16,fp8,127,0.017161600291728973
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,2,128,1,float16,float16,255,0.019555200636386872
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,2,128,1,float16,fp8,255,0.019118399918079378
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,2,128,1,float16,float16,511,0.0320032000541687
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,2,128,1,float16,fp8,511,0.02592639923095703
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,2,128,1,float16,float16,1023,0.04911519885063172
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,2,128,1,float16,fp8,1023,0.03743360042572021
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,2,128,1,float16,float16,2047,0.07935839891433716
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,4,128,1,float16,float16,1,0.021073600649833678
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,4,128,1,float16,float16,3,0.02110559940338135
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,4,128,1,float16,fp8,3,0.020878399908542632
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,4,128,1,float16,float16,7,0.021031999588012697
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,1,128,1,float16,float16,31,0.012942400574684144
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,4,128,1,float16,fp8,7,0.020852799713611602
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,4,128,1,float16,float16,15,0.021367999911308288
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,4,128,1,float16,float16,31,0.021372799575328828
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,1,128,1,float16,float16,127,0.01329279989004135
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,1,128,1,float16,fp8,255,0.014681600034236908
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,4,128,1,float16,fp8,63,0.021435199677944182
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,4,128,1,float16,float16,127,0.0222448006272316
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,64,8,128,1,float16,fp8,4095,0.021201600134372712
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,4,128,1,float16,fp8,127,0.0243599995970726
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,4,128,1,float16,float16,255,0.035750401020050046
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,4,128,1,float16,fp8,255,0.02794399857521057
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,4,128,1,float16,float16,511,0.05153120160102844
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,2,128,1,float16,fp8,2047,0.05727199912071228
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,4,128,1,float16,fp8,511,0.04324800074100495
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,4,128,1,float16,fp8,1,0.020849600434303284
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,4,128,1,float16,float16,1023,0.08297600150108338
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,4,128,1,float16,fp8,1023,0.061382400989532473
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,8,128,1,float16,float16,1,0.03249599933624268
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,4,128,1,float16,float16,2047,0.14194560050964355
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,4,128,1,float16,fp8,2047,0.10005600452423095
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,4,128,1,float16,fp8,15,0.02091040015220642
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,8,128,1,float16,fp8,1,0.031769600510597226
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,1,128,1,float16,fp8,63,0.012571200728416443
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,8,128,1,float16,float16,3,0.03243519961833954
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,4,128,1,float16,fp8,31,0.021113599836826324
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,8,128,1,float16,fp8,3,0.03208799958229065
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,8,128,1,float16,float16,7,0.03252640068531036
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,8,128,1,float16,float16,15,0.032467201352119446
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,8,128,1,float16,fp8,15,0.03225600123405457
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,8,128,1,float16,float16,31,0.03261120021343231
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,8,128,1,float16,fp8,31,0.03218559920787811
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,8,128,1,float16,float16,63,0.033081600069999696
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,8,128,1,float16,fp8,63,0.03208479881286621
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,8,128,1,float16,fp8,127,0.03901920020580292
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,8,128,1,float16,float16,127,0.04119040071964264
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,8,128,1,float16,float16,255,0.05647839903831482
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,8,128,1,float16,fp8,255,0.04954720139503479
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,8,128,1,float16,float16,511,0.08649759888648986
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,8,128,1,float16,fp8,511,0.07264159917831421
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,8,128,1,float16,float16,1023,0.14683840274810792
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,1,128,1,float16,float16,1,0.018291200697422027
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,1,128,1,float16,fp8,1,0.018160000443458557
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,1,128,1,float16,float16,3,0.0183119997382164
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,8,128,1,float16,fp8,2047,0.1872063994407654
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,8,128,1,float16,float16,2047,0.264302396774292
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,1,128,1,float16,fp8,7,0.017998400330543517
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,1,128,1,float16,float16,15,0.018108800053596497
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,1,128,1,float16,fp8,15,0.01804479956626892
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,1,128,1,float16,float16,31,0.018347199261188506
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,1,128,1,float16,fp8,31,0.017998400330543517
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,1,128,1,float16,float16,63,0.0182559996843338
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,1,128,1,float16,fp8,63,0.0182559996843338
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,1,128,1,float16,float16,127,0.01887200027704239
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,1,128,1,float16,fp8,127,0.019753600656986236
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,1,128,1,float16,float16,255,0.02290560007095337
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,1,128,1,float16,fp8,255,0.02152799963951111
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,1,128,1,float16,float16,511,0.03592320084571839
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,2,128,1,float16,float16,1,0.023689599335193635
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,2,128,1,float16,float16,3,0.02369920015335083
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,2,128,1,float16,fp8,3,0.023553599417209626
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,2,128,1,float16,float16,7,0.023772799968719484
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,2,128,1,float16,fp8,7,0.023523199558258056
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,2,128,1,float16,float16,15,0.023878400027751923
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,2,128,1,float16,fp8,15,0.023664000630378722
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,2,128,1,float16,float16,31,0.024001599848270418
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,2,128,1,float16,fp8,31,0.02369759976863861
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,2,128,1,float16,float16,63,0.023947200179100035
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,2,128,1,float16,fp8,63,0.023676800727844238
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,8,128,1,float16,fp8,7,0.031860798597335815
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,4,128,1,float16,float16,63,0.02128320038318634
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,2,128,1,float16,fp8,127,0.027182400226593018
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,2,128,1,float16,float16,255,0.03928160071372986
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,2,128,1,float16,fp8,255,0.03149439990520477
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,2,128,1,float16,float16,511,0.05418239831924439
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,2,128,1,float16,fp8,511,0.04708960056304932
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,64,8,128,1,float16,fp8,1023,0.10808320045471191
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,4,128,1,float16,fp8,1,0.034702399373054506
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,4,128,1,float16,float16,3,0.03499360084533691
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,1,128,1,float16,fp8,3,0.01805119961500168
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,1,128,1,float16,float16,7,0.018404799699783325
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,4,128,1,float16,fp8,7,0.03461759984493255
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,4,128,1,float16,fp8,15,0.03468640148639679
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,4,128,1,float16,float16,31,0.03510079979896545
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,4,128,1,float16,fp8,31,0.034753599762916566
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,4,128,1,float16,float16,63,0.036129599809646605
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,4,128,1,float16,fp8,63,0.03485119938850403
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,1,128,1,float16,fp8,511,0.02990880012512207
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,4,128,1,float16,float16,127,0.04350079894065857
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,2,128,1,float16,fp8,1,0.02358720004558563
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,4,128,1,float16,fp8,127,0.042393600940704344
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,4,128,1,float16,float16,255,0.06011679768562317
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,4,128,1,float16,fp8,255,0.05264319777488709
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,4,128,1,float16,float16,511,0.08983039855957031
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,4,128,1,float16,fp8,511,0.07728480100631714
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,8,128,1,float16,float16,1,0.05674880146980286
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,8,128,1,float16,fp8,1,0.05623679757118225
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,8,128,1,float16,float16,3,0.0562720000743866
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,8,128,1,float16,float16,7,0.056411200761795045
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,8,128,1,float16,fp8,7,0.05607360005378723
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,8,128,1,float16,fp8,15,0.055961602926254274
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,8,128,1,float16,float16,31,0.056118398904800415
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,2,128,1,float16,float16,127,0.026183998584747313
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,8,128,1,float16,fp8,31,0.056092798709869385
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,8,128,1,float16,float16,63,0.06314399838447571
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,8,128,1,float16,float16,127,0.0694703996181488
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,4,128,1,float16,float16,1,0.03513759970664978
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,8,128,1,float16,fp8,127,0.07362880110740662
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,8,128,1,float16,float16,255,0.09863680005073547
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,4,128,1,float16,fp8,3,0.03474240005016327
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,4,128,1,float16,float16,7,0.03457440137863159
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,8,128,1,float16,float16,511,0.15758559703826905
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,8,128,1,float16,fp8,511,0.13511040210723876
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,64,1,128,1,float16,float16,1,0.029739201068878174
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,64,1,128,1,float16,fp8,1,0.029553601145744325
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,64,1,128,1,float16,float16,3,0.029686400294303895
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,64,1,128,1,float16,fp8,3,0.029504001140594482
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,64,1,128,1,float16,fp8,7,0.02942720055580139
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,64,1,128,1,float16,float16,15,0.02970399856567383
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,64,1,128,1,float16,fp8,15,0.029499199986457825
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,64,1,128,1,float16,float16,31,0.029683199524879456
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,64,1,128,1,float16,fp8,31,0.029398399591445922
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,8,128,1,float16,fp8,3,0.05613759756088257
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,64,1,128,1,float16,float16,63,0.029846400022506714
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,64,1,128,1,float16,fp8,63,0.029548799991607665
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,64,1,128,1,float16,float16,127,0.03553119897842407
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,8,128,1,float16,float16,15,0.0565775990486145
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,64,1,128,1,float16,fp8,127,0.03296799957752228
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,64,1,128,1,float16,float16,255,0.04518080055713654
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,64,1,128,1,float16,fp8,255,0.038047999143600464
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,64,2,128,1,float16,fp8,1,0.040652799606323245
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,64,2,128,1,float16,float16,3,0.0404559999704361
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,64,2,128,1,float16,fp8,3,0.040782400965690614
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,64,2,128,1,float16,float16,7,0.04067200124263763
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,64,2,128,1,float16,fp8,7,0.040759998559951785
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,64,2,128,1,float16,float16,15,0.04074240028858185
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,64,2,128,1,float16,fp8,15,0.04087679982185364
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,8,128,1,float16,fp8,63,0.057678401470184326
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,64,2,128,1,float16,float16,31,0.04073919951915741
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,64,2,128,1,float16,fp8,31,0.040992000699043275
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,64,2,128,1,float16,float16,63,0.04582720100879669
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,64,2,128,1,float16,float16,127,0.05139840245246887
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,64,2,128,1,float16,fp8,127,0.04918079972267151
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,64,2,128,1,float16,float16,255,0.06702560186386108
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,64,2,128,1,float16,fp8,255,0.05873759984970093
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,64,4,128,1,float16,float16,1,0.06222879886627197
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,64,4,128,1,float16,fp8,1,0.06244959831237793
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,8,128,1,float16,fp8,255,0.08944320082664489
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,64,4,128,1,float16,fp8,3,0.06236799955368042
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,64,4,128,1,float16,float16,15,0.03536800146102905
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,64,4,128,1,float16,float16,7,0.06242079734802246
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,64,4,128,1,float16,fp8,7,0.06233440041542053
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,64,4,128,1,float16,float16,15,0.06281920075416565
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,64,4,128,1,float16,fp8,15,0.06232799887657166
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,64,4,128,1,float16,float16,31,0.063401597738266
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,64,4,128,1,float16,fp8,31,0.06271679997444153
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,64,4,128,1,float16,float16,63,0.06940799951553345
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,64,4,128,1,float16,fp8,63,0.06710559725761414
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,64,1,128,1,float16,float16,7,0.02979680001735687
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,64,4,128,1,float16,float16,127,0.07686560153961182
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,64,4,128,1,float16,float16,255,0.10538079738616943
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,64,4,128,1,float16,fp8,255,0.09639359712600708
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,64,8,128,1,float16,fp8,1,0.10264480113983154
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,64,8,128,1,float16,float16,3,0.1032863974571228
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,64,8,128,1,float16,fp8,3,0.10291199684143067
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,64,8,128,1,float16,float16,7,0.10353440046310425
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,64,8,128,1,float16,fp8,7,0.10267679691314698
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,64,8,128,1,float16,float16,15,0.10305119752883911
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,64,8,128,1,float16,fp8,15,0.10308799743652344
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,64,8,128,1,float16,float16,31,0.10925439596176148
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,64,8,128,1,float16,fp8,31,0.10390239953994751
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,64,8,128,1,float16,fp8,63,0.11243679523468017
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,64,2,128,1,float16,float16,1,0.04069280028343201
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,64,8,128,1,float16,float16,127,0.12780799865722656
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,64,8,128,1,float16,fp8,127,0.136627197265625
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,64,8,128,1,float16,float16,255,0.18133440017700195
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,1,128,1,float16,float16,1,0.010099200159311294
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,1,128,1,float16,fp8,1,0.009942399710416794
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,64,8,128,1,float16,fp8,255,0.16656320095062255
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,1,128,1,float16,float16,3,0.010011199861764908
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,64,2,128,1,float16,fp8,63,0.0409855991601944
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,1,128,1,float16,fp8,3,0.009942399710416794
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,1,128,1,float16,float16,7,0.010103999823331832
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,1,128,1,float16,fp8,15,0.009888000041246413
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,1,128,1,float16,float16,31,0.010169599950313569
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,1,128,1,float16,float16,63,0.010097599774599075
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,1,128,1,float16,fp8,63,0.009905599802732468
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,64,4,128,1,float16,float16,3,0.06149439811706543
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,1,128,1,float16,fp8,127,0.010836800187826156
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,1,128,1,float16,float16,255,0.011529599875211715
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,1,128,1,float16,fp8,255,0.011558400094509124
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,1,128,1,float16,float16,511,0.013683199882507324
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,1,128,1,float16,fp8,511,0.013155199587345123
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,1,128,1,float16,float16,1023,0.014134399592876434
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,1,128,1,float16,fp8,1023,0.013601599633693695
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,64,4,128,1,float16,fp8,127,0.08013759851455689
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,1,128,1,float16,float16,2047,0.014892800152301789
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,1,128,1,float16,fp8,2047,0.015030400454998016
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,1,128,1,float16,float16,4095,0.01688639968633652
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,1,128,1,float16,fp8,4095,0.017859199643135072
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,2,128,1,float16,float16,1,0.01014079973101616
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,64,8,128,1,float16,float16,1,0.10363359451293945
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,2,128,1,float16,float16,3,0.010119999945163726
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,2,128,1,float16,float16,7,0.009916800260543823
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,2,128,1,float16,fp8,7,0.00968799963593483
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,2,128,1,float16,float16,15,0.009920000284910201
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,2,128,1,float16,fp8,15,0.009703999757766724
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,2,128,1,float16,float16,31,0.009756799787282944
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,64,8,128,1,float16,float16,63,0.11648160219192505
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,2,128,1,float16,fp8,31,0.009566400200128555
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,2,128,1,float16,float16,63,0.009747199714183807
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,2,128,1,float16,fp8,63,0.009518399834632874
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,2,128,1,float16,float16,127,0.009921599924564362
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,2,128,1,float16,fp8,127,0.010478399693965912
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,2,128,1,float16,float16,255,0.011291199922561645
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,2,128,1,float16,fp8,255,0.011257600039243698
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,2,128,1,float16,float16,511,0.013433599472045898
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,1,128,1,float16,fp8,7,0.00973920002579689
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,1,128,1,float16,float16,15,0.009920000284910201
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,2,128,1,float16,fp8,511,0.01316159963607788
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,2,128,1,float16,float16,1023,0.014262400567531586
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,2,128,1,float16,fp8,1023,0.01387999951839447
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,2,128,1,float16,float16,2047,0.015719999372959138
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,2,128,1,float16,fp8,2047,0.015932799875736238
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,2,128,1,float16,float16,4095,0.018063999712467194
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,2,128,1,float16,fp8,4095,0.01865279972553253
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,4,128,1,float16,float16,1,0.00998080000281334
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,1,128,1,float16,float16,127,0.010201600193977357
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,4,128,1,float16,float16,3,0.009911999851465226
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,4,128,1,float16,float16,7,0.010087999701499938
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,4,128,1,float16,fp8,7,0.009867200255393982
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,4,128,1,float16,float16,15,0.010011199861764908
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,4,128,1,float16,float16,31,0.010016000270843506
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,4,128,1,float16,fp8,31,0.009956800192594529
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,4,128,1,float16,float16,63,0.010326399654150008
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,4,128,1,float16,fp8,63,0.010035199671983719
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,4,128,1,float16,float16,127,0.010393600165843963
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,4,128,1,float16,fp8,127,0.010699199885129929
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,4,128,1,float16,float16,255,0.01138719990849495
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,4,128,1,float16,fp8,255,0.011283200234174728
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,2,128,1,float16,fp8,1,0.0095040000975132
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,4,128,1,float16,float16,511,0.01364160031080246
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,4,128,1,float16,fp8,511,0.013684800267219544
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,4,128,1,float16,fp8,1023,0.014953599870204925
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,4,128,1,float16,float16,2047,0.017585599422454835
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,4,128,1,float16,fp8,2047,0.016681599617004394
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,4,128,1,float16,float16,4095,0.029043200612068176
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,4,128,1,float16,fp8,4095,0.021550400555133818
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,8,128,1,float16,float16,1,0.009945599734783173
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,8,128,1,float16,fp8,1,0.009911999851465226
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,8,128,1,float16,float16,3,0.010073599964380264
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,8,128,1,float16,fp8,3,0.010107199847698211
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,8,128,1,float16,float16,7,0.010067199915647506
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,8,128,1,float16,fp8,7,0.010011199861764908
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,8,128,1,float16,float16,15,0.010014399886131287
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,8,128,1,float16,fp8,15,0.009945599734783173
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,8,128,1,float16,float16,31,0.010003200173377991
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,1,128,1,float16,fp8,31,0.009841600060462951
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,8,128,1,float16,float16,63,0.01043360009789467
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,8,128,1,float16,fp8,63,0.01013600006699562
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,4,128,1,float16,fp8,1,0.00979359969496727
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,4,128,1,float16,fp8,3,0.00987360030412674
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,8,128,1,float16,float16,127,0.01072160005569458
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,8,128,1,float16,fp8,127,0.010833600163459777
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,4,128,1,float16,fp8,15,0.009864000231027603
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,8,128,1,float16,float16,511,0.013835200667381286
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,8,128,1,float16,fp8,511,0.014926399290561675
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,8,128,1,float16,float16,1023,0.01658080071210861
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,8,128,1,float16,fp8,1023,0.01642400026321411
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,8,128,1,float16,float16,2047,0.02818560004234314
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,8,128,1,float16,fp8,2047,0.021859200298786165
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,8,128,1,float16,float16,4095,0.044180798530578616
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,64,1,128,1,float16,float16,1,0.04796639978885651
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,2,128,1,float16,fp8,3,0.009702400118112565
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,64,1,128,1,float16,fp8,1,0.047417598962783816
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,64,1,128,1,float16,float16,3,0.0474016010761261
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,64,1,128,1,float16,float16,7,0.04792959988117218
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,64,1,128,1,float16,fp8,3,0.0474016010761261
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,64,1,128,1,float16,float16,15,0.048753601312637326
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,64,1,128,1,float16,float16,31,0.05035039782524109
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,64,1,128,1,float16,fp8,15,0.047547200322151185
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,64,1,128,1,float16,fp8,31,0.04760160148143768
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,64,1,128,1,float16,float16,63,0.05608479976654053
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,64,1,128,1,float16,fp8,63,0.049537599086761475
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,64,1,128,1,float16,float16,127,0.060761600732803345
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,64,1,128,1,float16,fp8,127,0.058278399705886844
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,64,2,128,1,float16,float16,1,0.06907520294189454
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,64,2,128,1,float16,fp8,1,0.06992639899253845
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,64,2,128,1,float16,float16,3,0.06861600279808044
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,64,2,128,1,float16,fp8,3,0.06938880085945129
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,64,2,128,1,float16,float16,7,0.06878560185432434
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,64,2,128,1,float16,fp8,7,0.0695967972278595
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,64,2,128,1,float16,float16,15,0.07078239917755128
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,64,2,128,1,float16,fp8,15,0.07013760209083557
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,8,128,1,float16,fp8,31,0.00981760025024414
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,64,2,128,1,float16,float16,31,0.07666720151901245
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,64,2,128,1,float16,fp8,31,0.07135840058326721
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,64,2,128,1,float16,fp8,63,0.07722079753875732
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,64,2,128,1,float16,float16,127,0.08655520081520081
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,8,128,1,float16,float16,255,0.011823999881744384
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,64,2,128,1,float16,fp8,127,0.08916959762573243
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,64,4,128,1,float16,float16,1,0.11125439405441284
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,64,4,128,1,float16,fp8,1,0.11016160249710083
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,64,4,128,1,float16,fp8,3,0.11057920455932617
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,64,4,128,1,float16,fp8,7,0.11089919805526734
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,8,128,1,float16,fp8,4095,0.032232001423835754
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,64,4,128,1,float16,float16,15,0.11577119827270507
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,64,4,128,1,float16,fp8,15,0.11060800552368164
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,64,4,128,1,float16,float16,31,0.11961439847946168
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,64,4,128,1,float16,fp8,31,0.1167072057723999
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,64,1,128,1,float16,fp8,7,0.04707199931144714
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,64,4,128,1,float16,float16,63,0.12446080446243286
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,64,4,128,1,float16,fp8,63,0.12156959772109985
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,64,4,128,1,float16,float16,127,0.13538559675216674
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,64,4,128,1,float16,fp8,127,0.14486080408096313
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,64,8,128,1,float16,float16,1,0.19163999557495118
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,64,8,128,1,float16,fp8,1,0.19061919450759887
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,64,8,128,1,float16,float16,3,0.1924496054649353
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,64,8,128,1,float16,fp8,3,0.19064320325851442
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,64,8,128,1,float16,float16,7,0.19482719898223877
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,64,8,128,1,float16,fp8,7,0.19111520051956177
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,64,8,128,1,float16,float16,15,0.200547194480896
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,64,2,128,1,float16,float16,63,0.07931360006332397
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,64,8,128,1,float16,fp8,15,0.19487520456314086
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,8,128,1,float16,fp8,255,0.011638399958610535
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,64,8,128,1,float16,float16,31,0.20641920566558838
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,64,8,128,1,float16,fp8,31,0.20523519515991212
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,64,4,128,1,float16,float16,3,0.11157280206680298
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,64,8,128,1,float16,float16,63,0.21837279796600342
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,64,8,128,1,float16,float16,127,0.2403104066848755
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,64,8,128,1,float16,fp8,63,0.21256000995635987
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,64,4,128,1,float16,float16,7,0.11216160058975219
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,64,8,128,1,float16,fp8,127,0.2572416067123413
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,64,1,128,1,float16,fp8,1,0.08829119801521301
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,64,1,128,1,float16,float16,3,0.09099519848823548
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,64,1,128,1,float16,fp8,7,0.08864480257034302
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,64,1,128,1,float16,float16,15,0.09069120287895202
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,64,1,128,1,float16,fp8,15,0.08817920088768005
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,64,4,128,1,float16,float16,1023,0.015201599895954132
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,64,1,128,1,float16,float16,63,0.0966480016708374
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,64,1,128,1,float16,fp8,31,0.08953120112419129
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,64,1,128,1,float16,float16,31,0.09283360242843627
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,64,1,128,1,float16,fp8,63,0.09216319918632507
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,64,2,128,1,float16,float16,1,0.13262720108032228
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,64,2,128,1,float16,fp8,1,0.13143520355224608
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,64,2,128,1,float16,fp8,3,0.13198399543762207
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,64,2,128,1,float16,float16,7,0.13340959548950196
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,64,2,128,1,float16,fp8,7,0.1318303942680359
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,64,2,128,1,float16,fp8,15,0.1337007999420166
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,64,2,128,1,float16,float16,15,0.13377439975738525
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,64,2,128,1,float16,float16,31,0.1361199975013733
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,64,2,128,1,float16,fp8,31,0.13463679552078248
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,64,2,128,1,float16,float16,63,0.14246400594711303
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,64,2,128,1,float16,fp8,63,0.13798240423202515
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,64,4,128,1,float16,float16,1,0.21710400581359862
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,64,4,128,1,float16,fp8,1,0.21517601013183593
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,64,4,128,1,float16,float16,3,0.2164639949798584
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,64,4,128,1,float16,float16,7,0.2176624059677124
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,64,4,128,1,float16,fp8,3,0.21493759155273437
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,64,4,128,1,float16,fp8,7,0.21544160842895507
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,64,4,128,1,float16,float16,15,0.21863679885864257
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,64,4,128,1,float16,fp8,15,0.217574405670166
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,64,4,128,1,float16,float16,31,0.22215039730072023
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,64,4,128,1,float16,fp8,31,0.220467209815979
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,64,4,128,1,float16,float16,63,0.23186240196228028
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,64,4,128,1,float16,fp8,63,0.22722399234771729
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,64,1,128,1,float16,float16,1,0.0904304027557373
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,64,8,128,1,float16,float16,1,0.3738159894943237
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,64,8,128,1,float16,float16,3,0.37419040203094484
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,64,8,128,1,float16,fp8,3,0.38135039806365967
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,64,1,128,1,float16,fp8,3,0.08912799954414367
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,64,1,128,1,float16,float16,7,0.09085599780082702
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,64,8,128,1,float16,float16,7,0.37788000106811526
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,64,8,128,1,float16,float16,15,0.3821120023727417
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,64,8,128,1,float16,fp8,15,0.3874063968658447
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,64,8,128,1,float16,float16,31,0.39507360458374025
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,1,128,1,float16,float16,1,0.010023999959230423
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,64,8,128,1,float16,fp8,31,0.3950799942016602
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,1,128,1,float16,fp8,1,0.009855999797582626
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,64,2,128,1,float16,float16,3,0.13283040523529052
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,1,128,1,float16,fp8,3,0.009640000015497207
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,1,128,1,float16,fp8,7,0.009531199932098389
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,64,8,128,1,float16,fp8,63,0.40700478553771974
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,1,128,1,float16,fp8,15,0.009593600034713745
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,1,128,1,float16,float16,31,0.010159999877214432
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,1,128,1,float16,float16,63,0.010168000310659408
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,1,128,1,float16,fp8,63,0.009995199739933014
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,1,128,1,float16,float16,127,0.009932799637317658
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,1,128,1,float16,fp8,127,0.01048479974269867
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,1,128,1,float16,float16,255,0.011505600064992905
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,1,128,1,float16,fp8,255,0.011662399768829346
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,1,128,1,float16,float16,511,0.01385599970817566
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,1,128,1,float16,fp8,511,0.014718399941921234
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,1,128,1,float16,float16,1023,0.014838400483131408
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,1,128,1,float16,fp8,1023,0.01539199948310852
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,1,128,1,float16,float16,2047,0.016728000342845918
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,1,128,1,float16,fp8,2047,0.017054399847984313
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,1,128,1,float16,float16,4095,0.019871999323368073
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,1,128,1,float16,fp8,4095,0.01980320066213608
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,2,128,1,float16,float16,1,0.009944000095129014
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,2,128,1,float16,fp8,1,0.009788800030946732
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,2,128,1,float16,float16,3,0.009948799759149552
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,2,128,1,float16,fp8,3,0.009934400022029877
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,2,128,1,float16,float16,7,0.009969600290060044
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,2,128,1,float16,fp8,7,0.009942399710416794
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,2,128,1,float16,float16,15,0.01000640019774437
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,2,128,1,float16,fp8,15,0.009921599924564362
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,2,128,1,float16,float16,31,0.009926400333642959
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,64,8,128,1,float16,fp8,7,0.3816943883895874
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,2,128,1,float16,fp8,31,0.009982399642467499
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,2,128,1,float16,float16,63,0.010315199941396713
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,1,128,1,float16,float16,3,0.009942399710416794
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,1,128,1,float16,float16,7,0.009931199997663499
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,1,128,1,float16,float16,15,0.009799999743700027
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,2,128,1,float16,fp8,127,0.011033599823713302
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,64,8,128,1,float16,float16,63,0.42049121856689453
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,2,128,1,float16,float16,255,0.011519999802112579
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,2,128,1,float16,fp8,255,0.01136159971356392
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,1,128,1,float16,fp8,31,0.009668800234794616
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,2,128,1,float16,float16,511,0.013758400082588195
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,2,128,1,float16,float16,1023,0.015561600029468537
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,2,128,1,float16,fp8,1023,0.01544319987297058
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,2,128,1,float16,float16,2047,0.01823520064353943
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,2,128,1,float16,float16,4095,0.02980479896068573
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,2,128,1,float16,fp8,4095,0.022678400576114654
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,4,128,1,float16,float16,1,0.010014399886131287
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,4,128,1,float16,fp8,1,0.010163199901580811
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,4,128,1,float16,fp8,3,0.009934400022029877
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,4,128,1,float16,float16,7,0.009947200119495393
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,4,128,1,float16,fp8,7,0.009878399968147277
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,4,128,1,float16,float16,15,0.009961599856615067
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,4,128,1,float16,fp8,15,0.009888000041246413
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,4,128,1,float16,float16,31,0.010027199983596802
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,4,128,1,float16,fp8,31,0.009987200051546097
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,4,128,1,float16,float16,63,0.010164800286293029
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,4,128,1,float16,fp8,63,0.010096000134944915
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,4,128,1,float16,float16,127,0.010553599894046783
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,4,128,1,float16,fp8,127,0.010843200236558914
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,4,128,1,float16,float16,255,0.011468800157308579
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,4,128,1,float16,fp8,255,0.01148959994316101
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,4,128,1,float16,float16,511,0.013886399567127228
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,4,128,1,float16,fp8,511,0.015020799636840821
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,64,8,128,1,float16,fp8,1,0.3797663927078247
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,4,128,1,float16,float16,1023,0.017547200620174407
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,4,128,1,float16,fp8,1023,0.017243200540542604
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,4,128,1,float16,fp8,2047,0.022078399360179902
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,4,128,1,float16,float16,4095,0.044095999002456664
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,4,128,1,float16,fp8,4095,0.03264960050582886
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,8,128,1,float16,float16,1,0.01032480001449585
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,8,128,1,float16,fp8,1,0.010310400277376175
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,2,128,1,float16,fp8,63,0.010060799866914749
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,8,128,1,float16,float16,3,0.010371199995279311
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,8,128,1,float16,fp8,3,0.010320000350475311
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,2,128,1,float16,float16,127,0.010363200306892395
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,8,128,1,float16,fp8,7,0.01029760017991066
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,8,128,1,float16,float16,15,0.010391999781131745
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,8,128,1,float16,fp8,15,0.010275200009346008
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,8,128,1,float16,float16,31,0.010292799770832061
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,8,128,1,float16,fp8,31,0.010155200213193893
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,8,128,1,float16,float16,63,0.010395199805498124
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,8,128,1,float16,fp8,63,0.010224000364542008
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,2,128,1,float16,fp8,2047,0.01778720021247864
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,8,128,1,float16,float16,127,0.010689599812030793
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,8,128,1,float16,fp8,127,0.01130400002002716
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,8,128,1,float16,float16,255,0.011921600252389909
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,8,128,1,float16,float16,511,0.014094400405883788
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,4,128,1,float16,float16,3,0.00995360016822815
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,8,128,1,float16,float16,1023,0.027395200729370118
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,8,128,1,float16,fp8,1023,0.019923199713230134
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,8,128,1,float16,float16,2047,0.042910400032997134
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,8,128,1,float16,fp8,2047,0.031112000346183777
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,1,128,1,float16,float16,1,0.011590400338172912
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,8,128,1,float16,fp8,4095,0.0495743989944458
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,1,128,1,float16,fp8,1,0.011633600294589996
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,1,128,1,float16,float16,3,0.011070399731397628
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,1,128,1,float16,fp8,3,0.01132960021495819
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,1,128,1,float16,float16,7,0.011291199922561645
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,1,128,1,float16,float16,15,0.011336000263690948
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,1,128,1,float16,fp8,15,0.011262399703264236
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,1,128,1,float16,float16,31,0.011503999680280685
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,1,128,1,float16,fp8,31,0.011243200302124024
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,1,128,1,float16,float16,63,0.011428800225257874
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,1,128,1,float16,fp8,63,0.01112319976091385
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,1,128,1,float16,float16,127,0.011519999802112579
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,4,128,1,float16,float16,2047,0.028812798857688903
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,1,128,1,float16,fp8,127,0.011935999989509583
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,1,128,1,float16,float16,255,0.01290079951286316
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,1,128,1,float16,fp8,255,0.012916800379753113
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,1,128,1,float16,float16,511,0.015067200362682342
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,1,128,1,float16,fp8,511,0.016433599591255187
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,1,128,1,float16,float16,1023,0.018824000656604768
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,1,128,1,float16,fp8,1023,0.018811200559139252
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,1,128,1,float16,float16,2047,0.031145599484443665
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,1,128,1,float16,fp8,2047,0.023291200399398804
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,1,128,1,float16,float16,4095,0.04659999907016754
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,8,128,1,float16,float16,7,0.010062400251626968
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,1,128,1,float16,fp8,8191,0.054065597057342527
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,2,128,1,float16,fp8,511,0.014636799693107605
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,2,128,1,float16,float16,1,0.011355199664831162
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,2,128,1,float16,fp8,1,0.011590400338172912
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,2,128,1,float16,float16,3,0.011324799805879592
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,2,128,1,float16,fp8,3,0.011777599900960922
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,2,128,1,float16,float16,7,0.01154400035738945
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,2,128,1,float16,fp8,7,0.011761599779129028
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,8,128,1,float16,fp8,255,0.011760000139474869
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,8,128,1,float16,fp8,511,0.015348799526691437
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,2,128,1,float16,float16,15,0.011742399632930755
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,2,128,1,float16,float16,31,0.011540800333023071
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,2,128,1,float16,fp8,31,0.01167680025100708
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,2,128,1,float16,float16,63,0.011609599739313126
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,2,128,1,float16,fp8,63,0.011745599657297134
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,2,128,1,float16,float16,127,0.012083200365304947
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,2,128,1,float16,fp8,127,0.012603199481964112
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,2,128,1,float16,float16,255,0.013406400382518769
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,2,128,1,float16,fp8,255,0.01326719969511032
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,2,128,1,float16,float16,511,0.015966400504112244
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,2,128,1,float16,fp8,511,0.016976000368595125
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,2,128,1,float16,float16,1023,0.029129600524902342
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,2,128,1,float16,fp8,1023,0.02155359983444214
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,2,128,1,float16,float16,2047,0.04412479996681214
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,1,128,1,float16,fp8,7,0.011287999898195266
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,2,128,1,float16,float16,4095,0.07397119998931885
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,2,128,1,float16,fp8,4095,0.05181760191917419
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,2,128,1,float16,float16,8191,0.133241605758667
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,2,128,1,float16,fp8,8191,0.09005119800567626
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,4,128,1,float16,float16,1,0.013976000249385834
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,4,128,1,float16,fp8,1,0.014161600172519684
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,4,128,1,float16,float16,3,0.014032000303268432
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,4,128,1,float16,fp8,3,0.014243200421333313
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,4,128,1,float16,float16,7,0.014084799587726593
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,4,128,1,float16,fp8,7,0.014286400377750396
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,4,128,1,float16,float16,15,0.014043200016021728
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,4,128,1,float16,fp8,15,0.014116799831390381
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,4,128,1,float16,float16,31,0.014185599982738495
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,4,128,1,float16,fp8,31,0.014256000518798828
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,4,128,1,float16,float16,63,0.014246399700641631
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,4,128,1,float16,fp8,63,0.014345599710941315
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,4,128,1,float16,float16,127,0.014892800152301789
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,4,128,1,float16,fp8,127,0.01583999991416931
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,4,128,1,float16,float16,255,0.01743520051240921
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,4,128,1,float16,fp8,255,0.01740960031747818
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,4,128,1,float16,float16,511,0.02996160089969635
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,4,128,1,float16,fp8,511,0.023951999843120575
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,1,128,1,float16,fp8,4095,0.03448480069637298
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,4,128,1,float16,float16,1023,0.046009600162506104
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,4,128,1,float16,fp8,1023,0.03521440029144287
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,1,128,1,float16,float16,8191,0.07583839893341064
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,4,128,1,float16,fp8,2047,0.05457760095596313
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,4,128,1,float16,float16,4095,0.13530559539794923
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,4,128,1,float16,fp8,4095,0.09188799858093262
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,8,128,1,float16,float16,1,0.019819200038909912
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,4,128,1,float16,float16,8191,0.2521872043609619
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,4,128,1,float16,fp8,8191,0.1676800012588501
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,2,128,1,float16,fp8,15,0.011561600118875503
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,8,128,1,float16,fp8,3,0.019534400105476378
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,8,128,1,float16,float16,7,0.019968000054359437
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,8,128,1,float16,fp8,7,0.019867199659347533
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,8,128,1,float16,float16,15,0.02009119987487793
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,8,128,1,float16,float16,31,0.02017440050840378
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,8,128,1,float16,fp8,31,0.019806399941444397
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,8,128,1,float16,float16,63,0.020366400480270386
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,8,128,1,float16,fp8,63,0.019892799854278564
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,2,128,1,float16,fp8,2047,0.032913601398468016
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,8,128,1,float16,float16,127,0.02029760032892227
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,8,128,1,float16,fp8,127,0.023035199940204622
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,8,128,1,float16,float16,255,0.03402400016784668
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,8,128,1,float16,fp8,255,0.027051201462745665
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,8,128,1,float16,float16,511,0.048563200235366824
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,8,128,1,float16,fp8,511,0.04094879925251007
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,8,128,1,float16,float16,1023,0.07932639718055726
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,8,128,1,float16,fp8,1023,0.058719998598098753
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,8,128,1,float16,float16,2047,0.138264000415802
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,8,128,1,float16,fp8,2047,0.09790719747543335
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,8,128,1,float16,fp8,4095,0.17125920057296753
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,8,128,1,float16,float16,4095,0.25371038913726807
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,4,128,1,float16,float16,2047,0.07628160119056701
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,8,128,1,float16,fp8,8191,0.3216559886932373
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,1,128,1,float16,float16,1,0.009558399766683578
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,8,128,1,float16,float16,8191,0.48779520988464353
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,1,128,1,float16,float16,3,0.009638399630784989
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,1,128,1,float16,fp8,1,0.009561599791049957
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,1,128,1,float16,float16,7,0.009593600034713745
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,1,128,1,float16,fp8,3,0.00942080020904541
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,1,128,1,float16,fp8,15,0.009531199932098389
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,1,128,1,float16,float16,31,0.009700799733400345
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,1,128,1,float16,float16,63,0.009728000313043595
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,1,128,1,float16,fp8,63,0.009435199946165086
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,1,128,1,float16,float16,127,0.009761600196361542
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,1,128,1,float16,fp8,127,0.010164800286293029
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,8,128,1,float16,fp8,15,0.019551999866962433
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,1,128,1,float16,float16,255,0.010756800323724747
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,1,128,1,float16,fp8,255,0.011228799819946289
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,1,128,1,float16,float16,511,0.012971200048923492
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,64,8,128,1,float16,float16,4095,0.07309120297431945
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,1,128,1,float16,fp8,511,0.012110400199890136
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,1,128,1,float16,float16,1023,0.013060800731182098
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,1,128,1,float16,float16,2047,0.013593600690364837
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,1,128,1,float16,fp8,1023,0.012870399653911591
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,1,128,1,float16,float16,4095,0.014560000598430633
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,1,128,1,float16,float16,8191,0.016148799657821657
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,1,128,1,float16,fp8,8191,0.016124799847602844
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,2,128,1,float16,float16,1,0.00971359983086586
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,2,128,1,float16,fp8,1,0.009672000259160995
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,2,128,1,float16,float16,3,0.009836799651384353
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,2,128,1,float16,fp8,3,0.009484799951314926
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,2,128,1,float16,float16,7,0.00966079980134964
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,2,128,1,float16,fp8,7,0.009361600130796432
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,2,128,1,float16,float16,15,0.009665600210428237
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,8,128,1,float16,fp8,1,0.019577600061893463
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,1,128,1,float16,fp8,31,0.009478399902582169
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,2,128,1,float16,fp8,15,0.00931520015001297
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,32,8,128,1,float16,float16,3,0.019574399292469024
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,2,128,1,float16,fp8,31,0.00928800031542778
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,2,128,1,float16,float16,63,0.0094480000436306
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,2,128,1,float16,fp8,63,0.009319999814033508
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,2,128,1,float16,float16,127,0.00947680026292801
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,2,128,1,float16,fp8,127,0.010126399993896484
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,2,128,1,float16,float16,255,0.010915199667215348
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,2,128,1,float16,fp8,255,0.011046399921178817
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,2,128,1,float16,float16,511,0.012990400195121765
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,2,128,1,float16,fp8,511,0.012118399888277055
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,2,128,1,float16,float16,1023,0.013150399923324585
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,2,128,1,float16,fp8,1023,0.012617599964141846
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,2,128,1,float16,float16,2047,0.013457599282264709
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,2,128,1,float16,fp8,2047,0.013131199777126313
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,1,128,1,float16,fp8,2047,0.013387200236320496
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,2,128,1,float16,float16,4095,0.014366400241851807
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,2,128,1,float16,fp8,4095,0.014340800046920777
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,2,128,1,float16,float16,8191,0.01592639982700348
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,2,128,1,float16,fp8,8191,0.016499200463294984
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,4,128,1,float16,fp8,1,0.009406399726867676
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,4,128,1,float16,float16,3,0.00955360010266304
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,4,128,1,float16,fp8,3,0.009401600062847137
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,4,128,1,float16,float16,7,0.009452799707651139
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,4,128,1,float16,fp8,7,0.009435199946165086
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,4,128,1,float16,float16,15,0.009494400024414063
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,4,128,1,float16,fp8,15,0.009329599887132644
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,4,128,1,float16,float16,31,0.009372799843549728
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,4,128,1,float16,fp8,31,0.009270399808883667
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,4,128,1,float16,float16,63,0.009515199810266495
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,4,128,1,float16,fp8,63,0.009268800169229508
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,4,128,1,float16,float16,127,0.00960479974746704
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,4,128,1,float16,fp8,127,0.009961599856615067
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,4,128,1,float16,float16,255,0.010876800119876861
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,4,128,1,float16,fp8,255,0.010795199871063232
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,4,128,1,float16,float16,511,0.012969599664211273
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,4,128,1,float16,fp8,511,0.01223360002040863
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,4,128,1,float16,float16,1023,0.013388800621032714
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,4,128,1,float16,fp8,1023,0.012814399600028992
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,4,128,1,float16,float16,2047,0.013823999464511872
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,4,128,1,float16,fp8,2047,0.013064000010490417
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,4,128,1,float16,float16,4095,0.014526399970054626
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,4,128,1,float16,fp8,4095,0.014638400077819825
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,4,128,1,float16,float16,8191,0.016209599375724793
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,4,128,1,float16,fp8,8191,0.015545600652694702
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,8,128,1,float16,float16,1,0.009718400239944459
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,8,128,1,float16,fp8,1,0.009283199906349182
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,8,128,1,float16,float16,3,0.009710399806499482
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,8,128,1,float16,fp8,3,0.009296000003814697
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,8,128,1,float16,float16,7,0.009737599641084671
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,8,128,1,float16,fp8,7,0.00947519987821579
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,8,128,1,float16,float16,15,0.009855999797582626
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,8,128,1,float16,fp8,15,0.009494400024414063
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,8,128,1,float16,float16,31,0.009838400036096573
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,8,128,1,float16,fp8,31,0.00941760018467903
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,8,128,1,float16,float16,63,0.00989919975399971
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,8,128,1,float16,fp8,63,0.00928800031542778
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,8,128,1,float16,float16,127,0.009939199686050415
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,8,128,1,float16,fp8,127,0.010035199671983719
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,8,128,1,float16,float16,255,0.01093600019812584
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,8,128,1,float16,fp8,255,0.010915199667215348
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,8,128,1,float16,float16,511,0.013166399300098419
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,8,128,1,float16,fp8,511,0.012219200283288956
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,1,128,1,float16,fp8,7,0.009465599805116654
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,8,128,1,float16,float16,1023,0.013537600636482239
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,8,128,1,float16,fp8,1023,0.012772800028324127
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,8,128,1,float16,fp8,2047,0.01403840035200119
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,8,128,1,float16,float16,2047,0.014182400703430176
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,8,128,1,float16,float16,4095,0.01541759967803955
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,2,128,1,float16,float16,31,0.009752000123262406
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,8,128,1,float16,float16,8191,0.018087999522686006
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,8,128,1,float16,fp8,8191,0.01727840006351471
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,1,128,1,float16,float16,1,0.009532800316810608
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,1,128,1,float16,fp8,1,0.009415999799966813
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,1,128,1,float16,fp8,3,0.009326399862766266
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,1,128,1,float16,float16,7,0.00979520007967949
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,1,128,1,float16,fp8,7,0.009308800101280212
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,1,128,1,float16,float16,15,0.009839999675750732
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,1,128,1,float16,fp8,15,0.009335999935865402
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,1,128,1,float16,fp8,31,0.009305600076913834
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,1,128,1,float16,float16,63,0.009758400171995163
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,1,128,1,float16,fp8,63,0.009478399902582169
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,1,128,1,float16,fp8,4095,0.014371199905872345
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,4,128,1,float16,float16,1,0.009550400078296661
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,1,128,1,float16,float16,127,0.009723199903964997
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,1,128,1,float16,fp8,127,0.01016639992594719
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,1,128,1,float16,fp8,255,0.011049599945545196
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,1,128,1,float16,float16,511,0.013049599528312684
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,1,128,1,float16,fp8,511,0.012272000312805176
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,1,128,1,float16,float16,1023,0.013385599851608277
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,1,128,1,float16,fp8,1023,0.012889599800109864
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,1,128,1,float16,float16,2047,0.013838399946689606
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,1,128,1,float16,fp8,2047,0.013288000226020813
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,1,128,1,float16,float16,4095,0.014571200311183929
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,1,128,1,float16,fp8,4095,0.014407999813556671
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,1,128,1,float16,float16,8191,0.01595519930124283
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,2,128,1,float16,float16,1,0.009494400024414063
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,2,128,1,float16,fp8,1,0.00923840031027794
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,2,128,1,float16,float16,3,0.009707199782133103
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,2,128,1,float16,fp8,3,0.009545599669218063
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,2,128,1,float16,float16,7,0.009716799855232239
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,2,128,1,float16,fp8,7,0.009414400160312652
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,2,128,1,float16,float16,15,0.009643200039863586
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,2,128,1,float16,fp8,15,0.009428799897432328
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,2,128,1,float16,float16,31,0.009652800112962722
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,2,128,1,float16,fp8,31,0.009431999921798707
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,2,128,1,float16,float16,63,0.00979200005531311
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,2,128,1,float16,fp8,63,0.00952640026807785
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,2,128,1,float16,float16,127,0.009753599762916565
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,2,128,1,float16,fp8,127,0.010302399843931198
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,2,128,1,float16,float16,255,0.010956799983978272
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,2,128,1,float16,fp8,255,0.01109279990196228
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,8,128,1,float16,fp8,4095,0.014681600034236908
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,2,128,1,float16,float16,511,0.01297439932823181
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,2,128,1,float16,fp8,511,0.012459199875593185
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,1,128,1,float16,float16,3,0.009583999961614608
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,2,128,1,float16,fp8,1023,0.012729600071907043
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,2,128,1,float16,float16,2047,0.013849599659442902
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,2,128,1,float16,fp8,2047,0.013376000523567199
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,1,128,1,float16,float16,31,0.009824000298976898
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,2,128,1,float16,fp8,4095,0.014483200013637542
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,2,128,1,float16,fp8,8191,0.015753600001335143
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,4,128,1,float16,float16,1,0.00958240032196045
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,4,128,1,float16,fp8,1,0.009478399902582169
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,4,128,1,float16,float16,3,0.00968480035662651
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,4,128,1,float16,fp8,3,0.00968639999628067
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,1,128,1,float16,float16,255,0.010931199789047242
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,4,128,1,float16,float16,7,0.009777600318193436
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,4,128,1,float16,fp8,7,0.009676799923181535
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,4,128,1,float16,float16,15,0.009815999865531921
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,4,128,1,float16,fp8,15,0.009668800234794616
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,4,128,1,float16,float16,31,0.009905599802732468
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,4,128,1,float16,fp8,31,0.009587199985980987
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,4,128,1,float16,float16,63,0.009678400307893752
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,4,128,1,float16,fp8,63,0.009495999664068222
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,4,128,1,float16,float16,127,0.009654399752616883
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,4,128,1,float16,fp8,127,0.010232000052928925
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,4,128,1,float16,float16,255,0.010886400192975997
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,1,128,1,float16,fp8,8191,0.016543999314308167
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,4,128,1,float16,fp8,255,0.011128000169992446
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,4,128,1,float16,float16,511,0.013055999577045441
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,4,128,1,float16,float16,1023,0.013387200236320496
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,4,128,1,float16,fp8,511,0.012263999879360199
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,4,128,1,float16,fp8,1023,0.012673600018024445
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,4,128,1,float16,float16,2047,0.01417279988527298
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,4,128,1,float16,fp8,2047,0.013926400244235993
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,4,128,1,float16,float16,4095,0.015676799416542053
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,4,128,1,float16,fp8,4095,0.014819200336933135
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,4,128,1,float16,float16,8191,0.017584000527858735
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,4,128,1,float16,fp8,8191,0.017433600127696992
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,8,128,1,float16,float16,1,0.009836799651384353
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,8,128,1,float16,fp8,1,0.009601599723100662
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,8,128,1,float16,fp8,3,0.00963359996676445
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,8,128,1,float16,float16,7,0.009622400254011154
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,8,128,1,float16,fp8,7,0.00963359996676445
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,8,128,1,float16,float16,15,0.009620799869298934
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,8,128,1,float16,fp8,15,0.00952960029244423
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,8,128,1,float16,float16,31,0.009601599723100662
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,8,128,1,float16,fp8,31,0.00947680026292801
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,8,128,1,float16,float16,63,0.00957920029759407
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,8,128,1,float16,fp8,63,0.009483200311660767
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,8,128,1,float16,float16,127,0.009644799679517747
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,8,128,1,float16,fp8,127,0.010011199861764908
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,8,128,1,float16,float16,255,0.011147200316190719
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,8,128,1,float16,fp8,255,0.010779199749231338
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,8,128,1,float16,float16,511,0.013350400328636169
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,8,128,1,float16,fp8,511,0.012459199875593185
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,8,128,1,float16,float16,1023,0.013787199556827546
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,8,128,1,float16,fp8,1023,0.013240000605583191
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,8,128,1,float16,float16,2047,0.015137599408626556
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,8,128,1,float16,float16,4095,0.017548799514770508
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,8,128,1,float16,fp8,4095,0.016763199865818024
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,2,128,1,float16,float16,1023,0.01343040019273758
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,8,128,1,float16,float16,8191,0.02841919958591461
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,1,128,1,float16,float16,1,0.010704000294208527
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,8,128,1,float16,fp8,8191,0.021275199949741364
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,1,128,1,float16,fp8,1,0.010651200264692306
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,2,128,1,float16,float16,4095,0.014961600303649902
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,1,128,1,float16,fp8,3,0.010667199641466141
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,1,128,1,float16,float16,7,0.010814400017261505
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,1,128,1,float16,float16,15,0.010815999656915664
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,1,128,1,float16,fp8,15,0.01077599972486496
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,1,128,1,float16,float16,31,0.010887999832630158
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,1,128,1,float16,fp8,31,0.010764800012111664
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,1,128,1,float16,float16,63,0.010446400195360184
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,1,128,1,float16,fp8,63,0.010406400263309478
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,1,128,1,float16,float16,127,0.01093439981341362
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,1,128,1,float16,fp8,127,0.011231999844312668
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,1,128,1,float16,float16,255,0.01196800023317337
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,1,128,1,float16,fp8,255,0.012139199674129486
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,1,128,1,float16,fp8,511,0.014953599870204925
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,1,128,1,float16,float16,1023,0.016041600704193117
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,1,128,1,float16,fp8,1023,0.016171200573444365
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,1,128,1,float16,float16,2047,0.01863040030002594
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,8,128,1,float16,float16,3,0.009703999757766724
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,1,128,1,float16,fp8,2047,0.018136000633239745
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,1,128,1,float16,float16,4095,0.03025600016117096
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,1,128,1,float16,fp8,4095,0.02284799963235855
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,1,128,1,float16,float16,8191,0.04515359997749328
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,1,128,1,float16,fp8,8191,0.03392159938812256
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,2,128,1,float16,float16,1,0.010526400059461594
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,32,1,128,1,float16,float16,15,0.00974079966545105
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,2,128,1,float16,fp8,1,0.010475199669599533
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,8,128,1,float16,fp8,2047,0.014276799559593201
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,2,128,1,float16,float16,3,0.010491199791431427
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,2,128,1,float16,fp8,3,0.010807999968528747
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,2,128,1,float16,float16,7,0.01029760017991066
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,2,128,1,float16,float16,15,0.010787200182676315
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,2,128,1,float16,fp8,15,0.010233599692583084
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,2,128,1,float16,float16,31,0.010833600163459777
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,2,128,1,float16,fp8,31,0.010436800122261048
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,2,128,1,float16,float16,63,0.010846400260925293
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,2,128,1,float16,fp8,63,0.010599999874830245
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,2,128,1,float16,float16,127,0.011116799712181092
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,2,128,1,float16,fp8,127,0.01165120005607605
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,2,128,1,float16,float16,255,0.012227199971675873
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,2,128,1,float16,fp8,255,0.012188799679279327
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,1,128,1,float16,float16,3,0.010465600341558457
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,2,128,1,float16,float16,511,0.014299200475215912
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,2,128,1,float16,fp8,511,0.01549919992685318
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,32,2,128,1,float16,float16,8191,0.016545599699020384
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,2,128,1,float16,float16,1023,0.01810719966888428
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,2,128,1,float16,float16,2047,0.029747200012207032
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,2,128,1,float16,fp8,2047,0.022598400712013245
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,2,128,1,float16,fp8,4095,0.03364000022411347
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,2,128,1,float16,float16,8191,0.07536960244178773
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,2,128,1,float16,fp8,8191,0.052374398708343504
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,4,128,1,float16,float16,1,0.01104160025715828
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,4,128,1,float16,fp8,1,0.010416000336408614
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,4,128,1,float16,fp8,3,0.010574399679899215
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,4,128,1,float16,float16,7,0.010683199763298035
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,4,128,1,float16,fp8,7,0.01067039966583252
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,4,128,1,float16,float16,15,0.010755199939012527
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,4,128,1,float16,fp8,15,0.010686399787664414
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,4,128,1,float16,float16,31,0.01061599999666214
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,4,128,1,float16,fp8,31,0.010460799932479859
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,4,128,1,float16,float16,63,0.010806400328874588
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,4,128,1,float16,fp8,63,0.010684800148010255
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,4,128,1,float16,float16,127,0.011260800063610077
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,4,128,1,float16,fp8,127,0.01175680011510849
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,4,128,1,float16,float16,255,0.01226079985499382
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,4,128,1,float16,fp8,255,0.012193600088357926
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,4,128,1,float16,float16,511,0.014473600685596466
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,4,128,1,float16,fp8,511,0.015800000727176668
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,4,128,1,float16,float16,1023,0.02786880135536194
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,4,128,1,float16,fp8,1023,0.020388799905776977
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,4,128,1,float16,float16,2047,0.043198400735855104
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,4,128,1,float16,fp8,2047,0.031462401151657104
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,4,128,1,float16,float16,4095,0.07319999933242798
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,4,128,1,float16,fp8,4095,0.050185602903366086
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,4,128,1,float16,float16,8191,0.1334383964538574
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,1,128,1,float16,fp8,7,0.010545600205659866
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,2,128,1,float16,fp8,1023,0.017417599260807038
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,8,128,1,float16,float16,1,0.01329279989004135
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,8,128,1,float16,fp8,1,0.013305599987506866
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,8,128,1,float16,float16,3,0.013500800728797913
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,2,128,1,float16,float16,4095,0.04508799910545349
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,8,128,1,float16,fp8,7,0.013447999954223633
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,8,128,1,float16,fp8,15,0.01348160058259964
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,8,128,1,float16,float16,15,0.013276800513267517
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,8,128,1,float16,float16,31,0.013654400408267976
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,8,128,1,float16,fp8,31,0.01319040060043335
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,4,128,1,float16,float16,3,0.010838399827480315
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,8,128,1,float16,float16,63,0.01377439945936203
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,8,128,1,float16,fp8,63,0.013276800513267517
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,8,128,1,float16,fp8,127,0.01523520052433014
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,8,128,1,float16,fp8,255,0.016892799735069276
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,1,128,1,float16,float16,511,0.014414399862289429
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,8,128,1,float16,float16,511,0.028059199452400208
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,8,128,1,float16,fp8,511,0.02338239997625351
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,8,128,1,float16,float16,1023,0.04487679898738861
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,8,128,1,float16,fp8,1023,0.03336319923400879
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,8,128,1,float16,float16,2047,0.0749455988407135
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,8,128,1,float16,fp8,2047,0.05371360182762146
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,8,128,1,float16,float16,4095,0.13309600353240966
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,8,128,1,float16,fp8,4095,0.09041919708251953
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,8,128,1,float16,fp8,8191,0.1653887987136841
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,8,128,1,float16,float16,8191,0.25022718906402586
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,1,128,1,float16,float16,1,0.009452799707651139
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,1,128,1,float16,float16,3,0.009471999853849411
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,1,128,1,float16,fp8,1,0.009340800344944
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,1,128,1,float16,fp8,3,0.009259200096130371
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,1,128,1,float16,float16,7,0.009454400092363358
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,1,128,1,float16,fp8,7,0.009404800087213516
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,1,128,1,float16,float16,15,0.009468799829483033
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,1,128,1,float16,fp8,15,0.009470400214195252
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,1,128,1,float16,fp8,31,0.009483200311660767
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,1,128,1,float16,float16,63,0.009672000259160995
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,1,128,1,float16,fp8,63,0.009334400296211243
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,1,128,1,float16,float16,127,0.009574399888515472
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,1,128,1,float16,fp8,127,0.00995199978351593
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,4,128,1,float16,fp8,8191,0.0880895972251892
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,1,128,1,float16,float16,255,0.010777600109577179
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,8,128,1,float16,fp8,3,0.013030399382114411
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,8,128,1,float16,float16,7,0.013391999900341034
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,1,128,1,float16,fp8,511,0.012276799976825714
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,1,128,1,float16,float16,1023,0.013462400436401368
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,1,128,1,float16,fp8,1023,0.012852799892425538
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,1,128,1,float16,float16,2047,0.013916799426078796
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,1,128,1,float16,fp8,2047,0.013628800213336945
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,8,128,1,float16,float16,127,0.013870400190353394
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,1,128,1,float16,float16,4095,0.014846399426460266
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,1,128,1,float16,fp8,4095,0.015057599544525147
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,1,128,1,float16,fp8,8191,0.01619199961423874
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,2,128,1,float16,float16,1,0.009670399874448777
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,2,128,1,float16,fp8,1,0.009494400024414063
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,2,128,1,float16,float16,3,0.009601599723100662
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,2,128,1,float16,fp8,3,0.009452799707651139
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,2,128,1,float16,float16,7,0.009600000083446502
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,2,128,1,float16,fp8,7,0.00955360010266304
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,2,128,1,float16,float16,15,0.009771200269460678
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,2,128,1,float16,fp8,15,0.009571199864149093
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,2,128,1,float16,float16,31,0.009787199646234512
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,2,128,1,float16,fp8,31,0.009532800316810608
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,2,128,1,float16,float16,63,0.00982080027461052
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,2,128,1,float16,fp8,63,0.009436800330877303
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,2,128,1,float16,float16,127,0.009668800234794616
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,2,128,1,float16,fp8,127,0.010188800096511842
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,2,128,1,float16,fp8,7,0.010278400033712387
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,2,128,1,float16,float16,255,0.010844799876213073
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,2,128,1,float16,float16,511,0.01311040073633194
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,1,128,1,float16,float16,31,0.009505599737167358
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,2,128,1,float16,fp8,511,0.012643200159072877
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,2,128,1,float16,float16,1023,0.013755199313163758
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,2,128,1,float16,fp8,1023,0.013167999684810638
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,2,128,1,float16,float16,2047,0.014420799911022186
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,2,128,1,float16,fp8,2047,0.014248000085353851
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,2,128,1,float16,float16,4095,0.015852800011634825
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,2,128,1,float16,fp8,4095,0.015094399452209473
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,2,128,1,float16,float16,8191,0.01812320053577423
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,2,128,1,float16,fp8,8191,0.01748799979686737
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,4,128,1,float16,float16,1,0.009600000083446502
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,4,128,1,float16,fp8,1,0.009507200121879578
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,1,128,1,float16,fp8,255,0.011019200086593628
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,4,128,1,float16,float16,3,0.009564799815416336
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,4,128,1,float16,float16,7,0.009585600346326828
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,4,128,1,float16,fp8,7,0.00968480035662651
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,4,128,1,float16,float16,15,0.009612800180912017
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,4,128,1,float16,float16,31,0.009641599655151368
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,4,128,1,float16,fp8,31,0.0095551997423172
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,32,8,128,1,float16,float16,255,0.016616000235080718
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,4,128,1,float16,float16,63,0.009723199903964997
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,4,128,1,float16,fp8,63,0.009560000151395798
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,1,128,1,float16,float16,8191,0.016809600591659545
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,4,128,1,float16,fp8,127,0.010318399965763092
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,4,128,1,float16,float16,255,0.011596799641847611
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,4,128,1,float16,float16,511,0.013262400031089782
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,4,128,1,float16,fp8,511,0.012425599992275238
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,4,128,1,float16,float16,1023,0.013980799913406372
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,4,128,1,float16,fp8,1023,0.013364799320697784
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,4,128,1,float16,float16,2047,0.015145599842071533
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,4,128,1,float16,fp8,2047,0.014321599900722504
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,4,128,1,float16,float16,4095,0.017430399358272553
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,4,128,1,float16,fp8,4095,0.01690080016851425
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,4,128,1,float16,float16,8191,0.028681600093841554
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,4,128,1,float16,fp8,8191,0.021219199895858763
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,8,128,1,float16,float16,1,0.009752000123262406
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,8,128,1,float16,fp8,1,0.010102400183677673
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,8,128,1,float16,float16,3,0.00987360030412674
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,8,128,1,float16,fp8,3,0.009799999743700027
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,8,128,1,float16,fp8,7,0.009772799909114838
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,8,128,1,float16,float16,15,0.009905599802732468
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,8,128,1,float16,fp8,15,0.009772799909114838
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,2,128,1,float16,fp8,255,0.010913600027561188
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,8,128,1,float16,float16,31,0.009707199782133103
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,8,128,1,float16,float16,63,0.00968960002064705
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,8,128,1,float16,fp8,63,0.009772799909114838
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,8,128,1,float16,float16,127,0.010078399628400802
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,8,128,1,float16,fp8,127,0.010815999656915664
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,8,128,1,float16,float16,255,0.011611200124025344
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,8,128,1,float16,fp8,255,0.011700800061225891
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,8,128,1,float16,float16,511,0.0135903999209404
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,4,128,1,float16,fp8,3,0.009446399658918381
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,8,128,1,float16,fp8,511,0.01342719942331314
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,8,128,1,float16,float16,1023,0.014913600683212281
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,8,128,1,float16,fp8,1023,0.014060799777507783
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,4,128,1,float16,fp8,15,0.00963039994239807
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,8,128,1,float16,fp8,2047,0.016516800224781036
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,8,128,1,float16,fp8,4095,0.021083199977874757
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,8,128,1,float16,float16,8191,0.04356000125408173
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,4,128,1,float16,float16,127,0.009854400157928466
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,1,128,1,float16,float16,1,0.013444800674915314
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,1,128,1,float16,fp8,1,0.013076800107955932
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,1,128,1,float16,float16,3,0.013067199289798737
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,1,128,1,float16,fp8,3,0.01290079951286316
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,1,128,1,float16,float16,7,0.01297439932823181
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,1,128,1,float16,fp8,7,0.012836800515651703
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,1,128,1,float16,float16,15,0.012804800271987915
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,1,128,1,float16,fp8,15,0.012683199346065521
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,1,128,1,float16,float16,31,0.013265599310398103
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,1,128,1,float16,fp8,31,0.013142399489879608
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,1,128,1,float16,float16,63,0.013281600177288055
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,1,128,1,float16,fp8,63,0.013169600069522858
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,1,128,1,float16,float16,127,0.013689599931240082
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,8,128,1,float16,float16,7,0.009878399968147277
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,1,128,1,float16,fp8,127,0.01371839940547943
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,1,128,1,float16,fp8,255,0.01459839940071106
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,1,128,1,float16,float16,255,0.015289600193500518
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,1,128,1,float16,float16,511,0.017479999363422392
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,1,128,1,float16,fp8,511,0.01844319999217987
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,8,128,1,float16,fp8,31,0.00958079993724823
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,1,128,1,float16,float16,1023,0.03043360114097595
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,1,128,1,float16,fp8,1023,0.022912000119686127
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,1,128,1,float16,float16,2047,0.04584319889545441
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,1,128,1,float16,fp8,2047,0.03472479879856109
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,1,128,1,float16,float16,4095,0.07497919797897339
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,1,128,1,float16,fp8,4095,0.054016000032424925
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,1,128,1,float16,float16,511,0.012964800000190735
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,2,128,1,float16,fp8,1,0.015534399449825287
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,2,128,1,float16,float16,3,0.015292799472808838
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,2,128,1,float16,fp8,3,0.015267199277877808
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,8,128,1,float16,float16,2047,0.016275200247764587
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,2,128,1,float16,fp8,7,0.015161600708961488
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,2,128,1,float16,float16,7,0.015438400208950043
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,2,128,1,float16,float16,15,0.015804800391197204
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,2,128,1,float16,fp8,31,0.015659199655056
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,2,128,1,float16,float16,63,0.015857599675655365
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,8,128,1,float16,fp8,8191,0.03211359977722168
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,2,128,1,float16,fp8,63,0.015569600462913512
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,2,128,1,float16,float16,127,0.015771199762821198
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,2,128,1,float16,fp8,127,0.017140799760818483
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,4,128,1,float16,fp8,255,0.011499200016260147
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,2,128,1,float16,float16,255,0.01812160015106201
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,2,128,1,float16,fp8,255,0.01905120015144348
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,2,128,1,float16,fp8,511,0.02595840096473694
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,2,128,1,float16,float16,511,0.03160159885883331
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,2,128,1,float16,float16,1023,0.04809280037879944
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,2,128,1,float16,fp8,1023,0.037092798948287965
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,2,128,1,float16,fp8,2047,0.057145601511001586
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,2,128,1,float16,float16,2047,0.07847679853439331
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,2,128,1,float16,float16,4095,0.13656959533691407
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,4,128,1,float16,float16,1,0.021238400042057036
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,2,128,1,float16,fp8,4095,0.09425280094146729
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,4,128,1,float16,float16,3,0.02125120013952255
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,4,128,1,float16,fp8,1,0.021160000562667848
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,4,128,1,float16,float16,7,0.021254399418830873
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,4,128,1,float16,float16,15,0.021352000534534454
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,4,128,1,float16,fp8,15,0.021211199462413788
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,4,128,1,float16,float16,31,0.02147040069103241
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,4,128,1,float16,fp8,31,0.02128800004720688
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,4,128,1,float16,float16,63,0.02157599925994873
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,4,128,1,float16,fp8,63,0.021427200734615327
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,4,128,1,float16,float16,127,0.021827200055122377
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,4,128,1,float16,fp8,127,0.024564799666404725
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,4,128,1,float16,float16,255,0.0351936012506485
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,4,128,1,float16,fp8,255,0.028376001119613647
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,2,128,1,float16,float16,1,0.015192000567913056
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,4,128,1,float16,float16,511,0.05022720098495483
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,4,128,1,float16,fp8,511,0.04249759912490845
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,32,8,128,1,float16,float16,4095,0.02834559977054596
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,4,128,1,float16,float16,1023,0.08239679932594299
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,2,128,1,float16,float16,31,0.015647999942302704
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,4,128,1,float16,fp8,1023,0.06091039776802063
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,4,128,1,float16,float16,2047,0.14057120084762573
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,8,128,1,float16,float16,1,0.032385599613189694
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,4,128,1,float16,fp8,2047,0.09946240186691284
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,4,128,1,float16,float16,4095,0.25582399368286135
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,4,128,1,float16,fp8,4095,0.1739776015281677
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,8,128,1,float16,fp8,1,0.03224000036716461
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,8,128,1,float16,fp8,3,0.032097598910331725
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,8,128,1,float16,float16,7,0.03235040009021759
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,8,128,1,float16,float16,15,0.032923200726509096
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,8,128,1,float16,fp8,15,0.03205440044403076
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,8,128,1,float16,fp8,7,0.03196159899234772
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,8,128,1,float16,float16,31,0.03291999995708465
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,8,128,1,float16,fp8,31,0.032323199510574344
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,8,128,1,float16,float16,63,0.032913601398468016
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,8,128,1,float16,float16,127,0.039643201231956485
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,8,128,1,float16,fp8,127,0.03922559916973114
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,8,128,1,float16,float16,255,0.05561439990997315
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,8,128,1,float16,fp8,255,0.049200001358985904
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,8,128,1,float16,float16,511,0.08536959886550903
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,4,128,1,float16,fp8,3,0.02112320065498352
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,8,128,1,float16,fp8,511,0.07139679789543152
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,8,128,1,float16,float16,1023,0.14498239755630493
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,8,128,1,float16,fp8,1023,0.10731840133666992
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,8,128,1,float16,float16,2047,0.2636159896850586
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,1,128,1,float16,float16,1,0.017827199399471284
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,8,128,1,float16,float16,4095,0.4918367862701416
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,8,128,1,float16,fp8,4095,0.33402080535888673
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,1,128,1,float16,float16,3,0.018321600556373597
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,1,128,1,float16,fp8,3,0.017633600533008574
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,1,128,1,float16,float16,7,0.018025599420070648
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,1,128,1,float16,fp8,7,0.017518399655818938
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,1,128,1,float16,float16,15,0.01783040016889572
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,1,128,1,float16,fp8,15,0.01778720021247864
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,1,128,1,float16,float16,31,0.017833599448204042
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,1,128,1,float16,fp8,31,0.01785600036382675
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,1,128,1,float16,float16,63,0.017926399409770966
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,1,128,1,float16,fp8,63,0.01791999936103821
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,1,128,1,float16,float16,127,0.018356800079345703
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,2,128,1,float16,fp8,15,0.015263999998569488
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,1,128,1,float16,fp8,127,0.019758400321006776
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,1,128,1,float16,fp8,255,0.02162880003452301
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,1,128,1,float16,float16,511,0.034745600819587705
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,1,128,1,float16,fp8,511,0.02876800000667572
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,1,128,1,float16,float16,1023,0.05173599720001221
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,1,128,1,float16,fp8,1023,0.04007520079612732
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,2,128,1,float16,float16,1,0.02351360023021698
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,2,128,1,float16,fp8,1,0.023526400327682495
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,2,128,1,float16,float16,3,0.023716799914836884
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,8,128,1,float16,fp8,63,0.031832000613212584
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,2,128,1,float16,fp8,3,0.023486399650573732
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,2,128,1,float16,fp8,7,0.02340800017118454
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,2,128,1,float16,float16,15,0.023614400625228883
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,2,128,1,float16,fp8,15,0.02351039946079254
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,2,128,1,float16,float16,31,0.02367199957370758
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,2,128,1,float16,fp8,31,0.02345760017633438
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,8,128,1,float16,fp8,2047,0.18553919792175294
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,2,128,1,float16,float16,63,0.02380319982767105
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,2,128,1,float16,float16,127,0.024324800074100494
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,2,128,1,float16,fp8,63,0.023548799753189086
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,2,128,1,float16,fp8,127,0.0270687997341156
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,2,128,1,float16,float16,255,0.03825120031833649
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,2,128,1,float16,fp8,255,0.0308351993560791
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,2,128,1,float16,float16,511,0.053830397129058835
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,2,128,1,float16,fp8,511,0.046132799983024594
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,2,128,1,float16,float16,1023,0.08614559769630432
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,4,128,1,float16,float16,1,0.03487040102481842
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,2,128,1,float16,fp8,1023,0.06491360068321228
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,4,128,1,float16,fp8,1,0.03447200059890747
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,4,128,1,float16,fp8,3,0.034544000029563905
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,4,128,1,float16,float16,7,0.03474720120429993
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,1,128,1,float16,float16,255,0.021454399824142455
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,4,128,1,float16,fp8,7,0.034534400701522826
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,4,128,1,float16,float16,15,0.03480960130691528
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,4,128,1,float16,float16,31,0.03490239977836609
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,4,128,1,float16,fp8,31,0.03468959927558899
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,8,128,1,float16,float16,3,0.03230240046977997
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,4,128,1,float16,float16,63,0.03511840105056763
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,4,128,1,float16,fp8,63,0.03470720052719116
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,4,128,1,float16,fp8,127,0.04178879857063293
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,2,128,1,float16,float16,7,0.02361599951982498
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,32,4,128,1,float16,fp8,7,0.021031999588012697
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,4,128,1,float16,float16,255,0.05907840132713318
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,4,128,1,float16,fp8,255,0.05225279927253723
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,4,128,1,float16,float16,511,0.0883791983127594
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,4,128,1,float16,fp8,511,0.07578719854354858
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,4,128,1,float16,float16,1023,0.14818400144577026
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,8,128,1,float16,float16,1,0.05612800121307373
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,4,128,1,float16,fp8,1023,0.11063840389251708
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,8,128,1,float16,fp8,1,0.056385600566864015
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,8,128,1,float16,float16,3,0.056150400638580324
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,8,128,1,float16,fp8,3,0.05584319829940796
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,8,128,1,float16,float16,7,0.05646399855613708
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,8,128,1,float16,fp8,7,0.05594559907913208
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,8,128,1,float16,fp8,15,0.056257599592208864
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,8,128,1,float16,float16,31,0.05652639865875244
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,8,128,1,float16,fp8,31,0.05617920160293579
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,8,128,1,float16,float16,63,0.06229280233383179
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,8,128,1,float16,fp8,63,0.05696319937705994
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,8,128,1,float16,float16,127,0.06890559792518616
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,8,128,1,float16,fp8,127,0.07323039770126342
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,4,128,1,float16,float16,3,0.034841600060462954
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,8,128,1,float16,float16,255,0.09580960273742675
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,8,128,1,float16,fp8,255,0.08929759860038758
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,8,128,1,float16,float16,511,0.15782400369644164
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,8,128,1,float16,fp8,511,0.13396639823913575
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,4,128,1,float16,fp8,15,0.034620800614356996
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,1,128,1,float16,float16,1,0.029420799016952513
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,1,128,1,float16,fp8,1,0.02927039861679077
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,8,128,1,float16,fp8,1023,0.20517120361328126
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,1,128,1,float16,float16,3,0.02954559922218323
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,1,128,1,float16,float16,7,0.02948000133037567
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,1,128,1,float16,fp8,7,0.029339200258255003
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,1,128,1,float16,float16,15,0.02954080104827881
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,1,128,1,float16,float16,31,0.02959679961204529
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,1,128,1,float16,float16,63,0.029760000109672547
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,4,128,1,float16,float16,127,0.04329439997673035
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,1,128,1,float16,fp8,63,0.029495999217033386
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,1,128,1,float16,float16,127,0.031174400448799135
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,1,128,1,float16,float16,255,0.04380159974098206
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,1,128,1,float16,fp8,255,0.03684319853782654
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,1,128,1,float16,float16,511,0.05907999873161316
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,1,128,1,float16,fp8,511,0.05270879864692688
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,2,128,1,float16,float16,1,0.04048640131950378
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,2,128,1,float16,fp8,1,0.040321600437164304
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,1,128,1,float16,fp8,1,0.017761600017547608
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,2,128,1,float16,float16,3,0.040329599380493165
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,2,128,1,float16,fp8,3,0.040299201011657716
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,2,128,1,float16,fp8,7,0.040387201309204104
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,2,128,1,float16,float16,15,0.04047679901123047
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,2,128,1,float16,fp8,15,0.04043520092964172
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,2,128,1,float16,float16,31,0.04059520065784454
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,2,128,1,float16,fp8,31,0.04055359959602356
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,2,128,1,float16,float16,63,0.04201599955558777
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,2,128,1,float16,fp8,63,0.04050880074501038
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,8,128,1,float16,float16,1023,0.2735455989837646
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,2,128,1,float16,float16,127,0.0497871994972229
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,1,128,1,float16,fp8,3,0.029254400730133058
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,2,128,1,float16,fp8,127,0.048347198963165285
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,1,128,1,float16,fp8,15,0.029175999760627746
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,2,128,1,float16,fp8,255,0.05834400057792664
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,2,128,1,float16,float16,511,0.09451519846916198
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,2,128,1,float16,fp8,511,0.0836463987827301
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,4,128,1,float16,float16,1,0.06223199963569641
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,4,128,1,float16,fp8,1,0.06227999925613403
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,4,128,1,float16,float16,3,0.06132320165634155
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,4,128,1,float16,float16,7,0.06268479824066162
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,4,128,1,float16,fp8,7,0.06251199841499329
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,4,128,1,float16,float16,15,0.06257759928703308
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,4,128,1,float16,fp8,15,0.062243199348449706
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,4,128,1,float16,float16,31,0.06247040033340454
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,4,128,1,float16,fp8,31,0.06222559809684754
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,4,128,1,float16,float16,63,0.06802240014076233
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,4,128,1,float16,fp8,63,0.0632752001285553
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,32,8,128,1,float16,float16,15,0.05627999901771545
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,4,128,1,float16,float16,127,0.07376639842987061
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,2,128,1,float16,float16,7,0.040387201309204104
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,4,128,1,float16,float16,255,0.10234880447387695
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,4,128,1,float16,fp8,255,0.0960864007472992
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,4,128,1,float16,float16,511,0.15969280004501343
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,4,128,1,float16,fp8,511,0.14252159595489503
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,8,128,1,float16,float16,1,0.10346879959106445
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,8,128,1,float16,fp8,1,0.10309439897537231
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,8,128,1,float16,float16,3,0.10327680110931396
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,8,128,1,float16,fp8,3,0.10303679704666138
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,8,128,1,float16,float16,7,0.10357600450515747
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,8,128,1,float16,fp8,7,0.10330560207366943
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,8,128,1,float16,float16,15,0.10354880094528199
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,8,128,1,float16,fp8,15,0.10314719676971436
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,8,128,1,float16,float16,31,0.10684959888458252
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,8,128,1,float16,fp8,31,0.1034000039100647
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,1,128,1,float16,fp8,31,0.029425600171089174
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,8,128,1,float16,float16,63,0.11571199893951416
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,8,128,1,float16,fp8,63,0.1122431993484497
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,8,128,1,float16,float16,127,0.1275488018989563
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,4,128,1,float16,fp8,3,0.06234880089759827
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,8,128,1,float16,fp8,127,0.13649439811706543
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,1,128,1,float16,fp8,127,0.03273600041866302
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,1,128,1,float16,float16,1,0.010019200295209885
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,8,128,1,float16,float16,255,0.17781120538711548
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,1,128,1,float16,fp8,1,0.009976000338792802
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,1,128,1,float16,float16,3,0.009924799948930741
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,1,128,1,float16,fp8,3,0.009964799880981446
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,8,128,1,float16,float16,511,0.3029567956924438
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,8,128,1,float16,fp8,511,0.2560847997665405
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,1,128,1,float16,float16,7,0.010310400277376175
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,1,128,1,float16,fp8,7,0.010009600222110749
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,1,128,1,float16,float16,31,0.010279999673366546
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,1,128,1,float16,fp8,31,0.01000479981303215
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,1,128,1,float16,fp8,15,0.009547200053930283
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,1,128,1,float16,float16,15,0.010335999727249145
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,1,128,1,float16,float16,127,0.009815999865531921
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,1,128,1,float16,float16,255,0.010969600081443787
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,1,128,1,float16,fp8,255,0.010937599837779999
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,1,128,1,float16,float16,511,0.013235199451446533
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,4,128,1,float16,fp8,127,0.0795952022075653
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,1,128,1,float16,float16,1023,0.013592000305652618
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,1,128,1,float16,fp8,1023,0.01316480040550232
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,1,128,1,float16,float16,2047,0.014446400105953217
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,1,128,1,float16,fp8,2047,0.014259199798107147
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,1,128,1,float16,float16,4095,0.015838399529457092
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,1,128,1,float16,fp8,4095,0.01539520025253296
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,1,128,1,float16,float16,8191,0.01823520064353943
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,1,128,1,float16,fp8,8191,0.01809120029211044
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,2,128,1,float16,float16,1,0.009531199932098389
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,2,128,1,float16,fp8,1,0.009355200082063675
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,2,128,1,float16,float16,255,0.06564000248908997
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,2,128,1,float16,fp8,3,0.00957759991288185
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,2,128,1,float16,fp8,7,0.00955199971795082
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,2,128,1,float16,float16,7,0.009998399764299393
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,2,128,1,float16,float16,15,0.009761600196361542
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,2,128,1,float16,fp8,15,0.009788800030946732
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,2,128,1,float16,float16,31,0.009799999743700027
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,2,128,1,float16,fp8,31,0.009758400171995163
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,2,128,1,float16,float16,63,0.009855999797582626
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,2,128,1,float16,fp8,63,0.009724800288677216
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,2,128,1,float16,float16,127,0.009769599884748459
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,2,128,1,float16,fp8,127,0.010239999741315842
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,2,128,1,float16,float16,255,0.011404799669981003
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,2,128,1,float16,fp8,255,0.011376000195741653
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,2,128,1,float16,float16,511,0.013075199723243714
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,2,128,1,float16,fp8,511,0.01297439932823181
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,2,128,1,float16,float16,1023,0.014081600308418273
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,32,8,128,1,float16,fp8,255,0.166103994846344
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,2,128,1,float16,float16,2047,0.015531200170516967
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,2,128,1,float16,fp8,2047,0.014580799639225006
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,2,128,1,float16,float16,4095,0.01788640022277832
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,2,128,1,float16,fp8,4095,0.017047999799251555
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,2,128,1,float16,float16,8191,0.029201599955558776
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,2,128,1,float16,fp8,8191,0.02165919989347458
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,4,128,1,float16,float16,1,0.010086400061845779
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,1,128,1,float16,float16,63,0.010115200281143188
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,1,128,1,float16,fp8,63,0.010080000013113022
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,4,128,1,float16,float16,3,0.009883199632167817
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,4,128,1,float16,fp8,3,0.010060799866914749
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,4,128,1,float16,float16,7,0.01016160026192665
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,1,128,1,float16,fp8,127,0.010515200346708298
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,4,128,1,float16,fp8,7,0.009862399846315383
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,4,128,1,float16,float16,15,0.010075200349092484
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,4,128,1,float16,fp8,15,0.009907200187444686
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,4,128,1,float16,fp8,31,0.009924799948930741
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,4,128,1,float16,float16,63,0.010132800042629241
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,4,128,1,float16,fp8,63,0.009864000231027603
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,4,128,1,float16,float16,127,0.010492800176143647
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,4,128,1,float16,float16,255,0.011631999909877778
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,4,128,1,float16,fp8,255,0.011769600212574005
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,4,128,1,float16,float16,511,0.013728000223636627
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,1,128,1,float16,fp8,511,0.012703999876976013
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,4,128,1,float16,fp8,511,0.013793599605560303
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,4,128,1,float16,float16,1023,0.015156799554824829
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,4,128,1,float16,float16,2047,0.016655999422073364
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,4,128,1,float16,fp8,2047,0.016697600483894348
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,4,128,1,float16,float16,4095,0.028352001309394838
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,4,128,1,float16,fp8,4095,0.021060800552368163
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,4,128,1,float16,float16,8191,0.043854400515556335
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,4,128,1,float16,fp8,8191,0.03266879916191101
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,8,128,1,float16,float16,1,0.010003200173377991
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,8,128,1,float16,fp8,1,0.009777600318193436
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,8,128,1,float16,float16,3,0.009915199875831605
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,8,128,1,float16,fp8,3,0.009728000313043595
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,8,128,1,float16,float16,7,0.009857600182294845
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,8,128,1,float16,fp8,7,0.009911999851465226
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,8,128,1,float16,float16,15,0.010127999633550645
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,8,128,1,float16,fp8,15,0.009916800260543823
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,8,128,1,float16,float16,31,0.01011039987206459
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,8,128,1,float16,fp8,31,0.009967999905347824
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,8,128,1,float16,float16,63,0.01021760031580925
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,2,128,1,float16,fp8,1023,0.013897599279880523
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,8,128,1,float16,fp8,63,0.009852799773216247
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,8,128,1,float16,float16,127,0.010294400155544281
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,8,128,1,float16,fp8,127,0.011068800091743469
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,8,128,1,float16,float16,255,0.011559999734163284
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,8,128,1,float16,fp8,255,0.011806400120258331
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,8,128,1,float16,float16,511,0.01356160044670105
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,8,128,1,float16,fp8,511,0.015184000134468079
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,8,128,1,float16,float16,1023,0.016332800686359405
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,8,128,1,float16,fp8,1023,0.01661760061979294
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,4,128,1,float16,fp8,1,0.00963359996676445
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,8,128,1,float16,float16,2047,0.02863360047340393
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,8,128,1,float16,fp8,2047,0.021908800303936004
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,4,128,1,float16,float16,31,0.010054399818181991
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,4,128,1,float16,fp8,127,0.010787200182676315
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,8,128,1,float16,float16,4095,0.04339520037174225
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,8,128,1,float16,fp8,4095,0.03197920024394989
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,4,128,1,float16,fp8,1023,0.014444799721240997
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,32,1,128,1,float16,float16,1,0.04698080122470856
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,32,1,128,1,float16,fp8,1,0.04676479995250702
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,8,128,1,float16,fp8,8191,0.05124959945678711
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,32,1,128,1,float16,float16,3,0.04660800099372864
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,32,1,128,1,float16,fp8,7,0.04695680141448975
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,32,1,128,1,float16,float16,7,0.04720959961414337
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,32,1,128,1,float16,float16,15,0.0471455991268158
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,2,128,1,float16,float16,3,0.00968639999628067
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,32,1,128,1,float16,float16,31,0.04668160080909729
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,32,1,128,1,float16,fp8,31,0.04711039960384369
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,32,1,128,1,float16,fp8,15,0.047070398926734924
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,32,1,128,1,float16,float16,63,0.05295199751853943
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,32,1,128,1,float16,float16,127,0.05651999711990356
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,32,1,128,1,float16,fp8,127,0.05664799809455871
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,32,1,128,1,float16,float16,255,0.071806401014328
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,32,2,128,1,float16,float16,1,0.06775040030479432
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,32,2,128,1,float16,fp8,1,0.06824960112571717
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,32,2,128,1,float16,float16,3,0.06880319714546204
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,32,2,128,1,float16,fp8,3,0.06809599995613098
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,32,2,128,1,float16,float16,7,0.06803200244903565
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,32,2,128,1,float16,fp8,7,0.06867520213127136
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,32,2,128,1,float16,float16,15,0.06776000261306762
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,32,2,128,1,float16,float16,31,0.06947680115699768
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,32,2,128,1,float16,fp8,31,0.06858559846878051
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,32,2,128,1,float16,float16,63,0.07643679976463318
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,32,2,128,1,float16,fp8,63,0.07363200187683105
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,32,2,128,1,float16,float16,127,0.08352959752082825
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,32,8,128,1,float16,float16,8191,0.07321919798851013
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,32,2,128,1,float16,fp8,127,0.08707839846611024
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,32,2,128,1,float16,float16,255,0.1094704031944275
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,32,4,128,1,float16,float16,1,0.10904480218887329
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,32,4,128,1,float16,fp8,1,0.10941280126571655
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,32,4,128,1,float16,float16,3,0.11003999710083008
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,32,4,128,1,float16,fp8,3,0.10926079750061035
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,32,4,128,1,float16,float16,7,0.10973119735717773
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,32,4,128,1,float16,float16,15,0.1097823977470398
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,32,4,128,1,float16,fp8,15,0.10961600542068481
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,32,4,128,1,float16,float16,31,0.11701439619064331
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,32,4,128,1,float16,fp8,31,0.1106495976448059
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,32,4,128,1,float16,float16,63,0.12217439413070678
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,32,4,128,1,float16,fp8,63,0.12032959461212159
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,32,4,128,1,float16,float16,127,0.13218239545822144
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,32,4,128,1,float16,fp8,127,0.1441759943962097
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,32,4,128,1,float16,float16,255,0.1855232000350952
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,32,4,128,1,float16,fp8,255,0.17576160430908203
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,32,8,128,1,float16,float16,1,0.19108320474624635
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,32,8,128,1,float16,fp8,1,0.1904415965080261
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,32,8,128,1,float16,float16,3,0.19103039503097535
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,32,1,128,1,float16,fp8,255,0.06617439985275268
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,32,8,128,1,float16,fp8,3,0.19011679887771607
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,32,8,128,1,float16,float16,7,0.1911120057106018
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,32,8,128,1,float16,fp8,7,0.1911520004272461
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,32,8,128,1,float16,fp8,15,0.1906159996986389
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,32,8,128,1,float16,float16,31,0.20400960445404054
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,32,8,128,1,float16,fp8,31,0.20333759784698485
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,32,8,128,1,float16,float16,63,0.21628639698028565
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,32,8,128,1,float16,fp8,63,0.21005120277404785
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,32,2,128,1,float16,fp8,255,0.104694402217865
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,32,1,128,1,float16,fp8,3,0.04722720086574554
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,32,8,128,1,float16,float16,127,0.2361407995223999
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,32,8,128,1,float16,fp8,127,0.256060791015625
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,32,4,128,1,float16,fp8,7,0.11053919792175293
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,32,1,128,1,float16,float16,1,0.07917919754981995
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,32,8,128,1,float16,float16,255,0.34000959396362307
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,32,1,128,1,float16,float16,3,0.07974560260772705
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,32,8,128,1,float16,fp8,255,0.3171135902404785
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,32,1,128,1,float16,float16,7,0.07950400114059449
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,32,1,128,1,float16,fp8,3,0.0805679976940155
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,32,1,128,1,float16,fp8,7,0.08039839863777161
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,32,1,128,1,float16,fp8,15,0.0802016019821167
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,32,1,128,1,float16,float16,31,0.08652480244636536
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,32,1,128,1,float16,fp8,31,0.08066239953041077
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,32,1,128,1,float16,float16,63,0.09159039855003356
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,32,1,128,1,float16,fp8,63,0.08805599808692932
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,32,1,128,1,float16,float16,127,0.0973039984703064
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,32,1,128,1,float16,fp8,127,0.10121279954910278
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,32,2,128,1,float16,float16,1,0.12119840383529663
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,32,2,128,1,float16,fp8,1,0.12066400051116943
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,32,1,128,1,float16,fp8,63,0.047111999988555905
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,32,2,128,1,float16,float16,3,0.12139519453048705
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,32,2,128,1,float16,fp8,3,0.12131359577178955
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,32,2,128,1,float16,float16,7,0.12172160148620606
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,32,2,128,1,float16,fp8,7,0.12149120569229126
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,32,2,128,1,float16,float16,15,0.125873601436615
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,32,2,128,1,float16,fp8,15,0.0684943974018097
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,32,2,128,1,float16,fp8,15,0.12182879447937012
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,32,2,128,1,float16,float16,31,0.13149600028991698
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,32,2,128,1,float16,fp8,31,0.12728480100631714
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,32,2,128,1,float16,float16,63,0.13702720403671265
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,32,2,128,1,float16,fp8,63,0.13417439460754393
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,32,2,128,1,float16,float16,127,0.14740959405899048
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,32,2,128,1,float16,fp8,127,0.1587056040763855
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,32,4,128,1,float16,float16,1,0.20488319396972657
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,32,4,128,1,float16,fp8,1,0.2036128044128418
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,32,4,128,1,float16,float16,3,0.20781118869781495
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,32,4,128,1,float16,fp8,3,0.2031615972518921
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,32,4,128,1,float16,float16,7,0.20896320343017577
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,32,4,128,1,float16,fp8,7,0.20306720733642578
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,32,4,128,1,float16,float16,15,0.21539680957794188
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,32,4,128,1,float16,fp8,15,0.2066175937652588
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,32,4,128,1,float16,float16,31,0.2189728021621704
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,32,4,128,1,float16,float16,63,0.2288815975189209
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,32,4,128,1,float16,fp8,63,0.225435209274292
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,32,1,128,1,float16,fp8,1,0.08023520112037659
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,32,4,128,1,float16,float16,127,0.2454927921295166
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,32,4,128,1,float16,fp8,127,0.27278881072998046
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,32,8,128,1,float16,float16,1,0.36727199554443357
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,32,8,128,1,float16,fp8,1,0.3667504072189331
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,32,8,128,1,float16,float16,3,0.36844959259033205
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,32,8,128,1,float16,fp8,3,0.36718881130218506
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,32,8,128,1,float16,float16,7,0.37172799110412597
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,32,8,128,1,float16,float16,15,0.3805056095123291
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,32,8,128,1,float16,fp8,15,0.3848432064056396
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,32,8,128,1,float16,float16,31,0.3909775972366333
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,32,8,128,1,float16,fp8,31,0.3947904109954834
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,32,8,128,1,float16,float16,15,0.19476159811019897
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,32,8,128,1,float16,float16,63,0.4165823936462402
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,32,8,128,1,float16,fp8,63,0.40566401481628417
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,1,128,1,float16,float16,1,0.010137599706649781
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,1,128,1,float16,fp8,1,0.009752000123262406
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,1,128,1,float16,float16,3,0.00984480008482933
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,1,128,1,float16,fp8,3,0.009678400307893752
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,32,8,128,1,float16,float16,127,0.45054078102111816
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,1,128,1,float16,fp8,7,0.009712000191211701
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,1,128,1,float16,fp8,15,0.009609600156545639
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,32,8,128,1,float16,fp8,127,0.49814720153808595
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,1,128,1,float16,float16,31,0.009774400293827057
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,1,128,1,float16,float16,15,0.009932799637317658
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,1,128,1,float16,float16,63,0.009854400157928466
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,1,128,1,float16,fp8,31,0.010078399628400802
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,1,128,1,float16,float16,127,0.010075200349092484
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,1,128,1,float16,float16,255,0.011673600226640702
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,1,128,1,float16,fp8,255,0.01117279976606369
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,1,128,1,float16,float16,511,0.013340799510478974
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,1,128,1,float16,fp8,511,0.013340799510478974
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,1,128,1,float16,float16,1023,0.01430879980325699
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,32,4,128,1,float16,fp8,31,0.21773600578308105
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,1,128,1,float16,float16,2047,0.015625600516796113
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,1,128,1,float16,float16,4095,0.018118399381637573
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,1,128,1,float16,fp8,2047,0.015894399583339693
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,1,128,1,float16,fp8,4095,0.018588800728321076
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,1,128,1,float16,float16,8191,0.029660800099372865
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,1,128,1,float16,fp8,8191,0.023835200071334838
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,32,1,128,1,float16,float16,15,0.08067520260810852
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,2,128,1,float16,fp8,1,0.009932799637317658
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,2,128,1,float16,float16,3,0.009881599992513656
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,2,128,1,float16,fp8,3,0.009963200241327286
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,2,128,1,float16,float16,7,0.009908799827098847
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,2,128,1,float16,fp8,7,0.00995519980788231
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,2,128,1,float16,float16,15,0.009913600236177444
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,2,128,1,float16,fp8,15,0.00998080000281334
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,2,128,1,float16,float16,31,0.009991999715566635
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,2,128,1,float16,fp8,31,0.009779199957847595
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,32,8,128,1,float16,fp8,7,0.36916959285736084
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,2,128,1,float16,fp8,63,0.00995199978351593
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,2,128,1,float16,float16,127,0.010419200360774993
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,2,128,1,float16,fp8,127,0.010871999710798264
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,2,128,1,float16,float16,255,0.011750400066375732
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,2,128,1,float16,fp8,255,0.011635199934244157
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,2,128,1,float16,fp8,511,0.013782399892807006
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,2,128,1,float16,float16,1023,0.015110400319099427
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,1,128,1,float16,float16,7,0.009774400293827057
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,2,128,1,float16,fp8,1023,0.015044799447059632
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,2,128,1,float16,float16,2047,0.017824000120162962
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,1,128,1,float16,fp8,63,0.009585600346326828
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,2,128,1,float16,fp8,2047,0.01696320027112961
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,1,128,1,float16,fp8,127,0.010380800068378448
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,2,128,1,float16,float16,4095,0.02937760055065155
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,2,128,1,float16,fp8,4095,0.021792000532150267
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,2,128,1,float16,float16,8191,0.04437440037727356
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,1,128,1,float16,fp8,1023,0.014169600605964661
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,4,128,1,float16,fp8,1,0.009857600182294845
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,4,128,1,float16,float16,3,0.010224000364542008
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,4,128,1,float16,float16,7,0.010232000052928925
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,4,128,1,float16,fp8,7,0.010249599814414978
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,4,128,1,float16,float16,15,0.01027199998497963
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,2,128,1,float16,float16,1,0.009934400022029877
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,4,128,1,float16,fp8,15,0.010267200320959092
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,4,128,1,float16,fp8,31,0.010227199643850327
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,4,128,1,float16,float16,63,0.010436800122261048
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,4,128,1,float16,fp8,63,0.010239999741315842
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,4,128,1,float16,float16,127,0.010654400289058685
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,2,128,1,float16,float16,63,0.010124800354242324
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,4,128,1,float16,fp8,127,0.011376000195741653
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,4,128,1,float16,float16,255,0.011905600130558015
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,4,128,1,float16,fp8,255,0.011727999895811081
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,4,128,1,float16,float16,511,0.01395999938249588
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,4,128,1,float16,fp8,511,0.014990399777889251
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,4,128,1,float16,float16,1023,0.016620799899101257
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,4,128,1,float16,fp8,1023,0.01666560024023056
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,4,128,1,float16,float16,2047,0.02858720123767853
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,4,128,1,float16,fp8,2047,0.021980799734592438
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,4,128,1,float16,float16,4095,0.04366079866886139
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,4,128,1,float16,fp8,4095,0.03203200101852417
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,4,128,1,float16,fp8,8191,0.05143200159072876
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,8,128,1,float16,float16,1,0.010142400115728378
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,8,128,1,float16,fp8,1,0.010248000174760819
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,8,128,1,float16,float16,3,0.01032319962978363
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,8,128,1,float16,fp8,3,0.010305599868297577
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,8,128,1,float16,float16,7,0.010150399804115296
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,8,128,1,float16,fp8,7,0.010142400115728378
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,8,128,1,float16,float16,15,0.010299199819564819
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,8,128,1,float16,fp8,15,0.010158400237560272
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,8,128,1,float16,float16,31,0.010313600301742554
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,8,128,1,float16,fp8,31,0.010230399668216705
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,8,128,1,float16,float16,63,0.010467199981212616
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,4,128,1,float16,float16,1,0.010168000310659408
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,8,128,1,float16,fp8,63,0.010337600111961364
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,8,128,1,float16,float16,127,0.01067200005054474
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,8,128,1,float16,fp8,127,0.011233600229024887
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,4,128,1,float16,fp8,3,0.010086400061845779
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,8,128,1,float16,fp8,255,0.011691199988126755
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,8,128,1,float16,float16,511,0.014324800670146942
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,8,128,1,float16,fp8,511,0.015641599893569946
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,8,128,1,float16,float16,1023,0.027063998579978942
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,8,128,1,float16,fp8,1023,0.01993599981069565
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,8,128,1,float16,float16,2047,0.0426144003868103
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,4,128,1,float16,float16,31,0.01021760031580925
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,8,128,1,float16,fp8,2047,0.03105440139770508
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,8,128,1,float16,float16,4095,0.07210879921913146
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,8,128,1,float16,fp8,4095,0.049404799938201904
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,8,128,1,float16,float16,8191,0.13256479501724244
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,1,128,1,float16,float16,1,0.011009600013494492
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,1,128,1,float16,float16,3,0.011180800199508668
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,1,128,1,float16,fp8,1,0.011116799712181092
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,1,128,1,float16,fp8,3,0.011184000223875046
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,1,128,1,float16,float16,7,0.01114879995584488
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,1,128,1,float16,fp8,7,0.011196800321340562
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,1,128,1,float16,float16,15,0.011177600175142289
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,1,128,1,float16,fp8,15,0.011563199758529662
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,1,128,1,float16,fp8,31,0.010972800105810166
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,1,128,1,float16,float16,63,0.011580800265073776
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,1,128,1,float16,float16,127,0.01151840016245842
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,1,128,1,float16,fp8,127,0.012014400213956833
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,1,128,1,float16,float16,255,0.013068799674510957
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,1,128,1,float16,fp8,255,0.012972800433635712
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,4,128,1,float16,float16,8191,0.07424799799919128
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,1,128,1,float16,float16,511,0.015243199467658997
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,1,128,1,float16,fp8,511,0.016249600052833556
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,1,128,1,float16,float16,1023,0.01803999990224838
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,1,128,1,float16,fp8,1023,0.018343999981880188
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,1,128,1,float16,float16,2047,0.030395200848579405
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,1,128,1,float16,fp8,2047,0.023390400409698486
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,1,128,1,float16,float16,4095,0.04541600048542023
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,1,128,1,float16,fp8,4095,0.033932799100875856
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,1,128,1,float16,fp8,8191,0.053491199016571046
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,1,128,1,float16,float16,16383,0.13617600202560426
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,2,128,1,float16,fp8,8191,0.03333120048046112
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,1,128,1,float16,fp8,16383,0.09133599996566773
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,2,128,1,float16,float16,1,0.011446399986743927
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,8,128,1,float16,float16,255,0.011992000043392181
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,2,128,1,float16,fp8,1,0.01170239970088005
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,2,128,1,float16,fp8,3,0.01120320037007332
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,2,128,1,float16,fp8,7,0.011260800063610077
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,2,128,1,float16,float16,15,0.011404799669981003
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,2,128,1,float16,fp8,15,0.011433599889278412
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,2,128,1,float16,float16,31,0.011371199786663056
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,2,128,1,float16,fp8,31,0.011337599903345107
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,2,128,1,float16,float16,63,0.011294399946928024
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,8,128,1,float16,fp8,8191,0.08755519986152649
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,2,128,1,float16,fp8,63,0.011313600093126297
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,2,128,1,float16,float16,127,0.011828800290822982
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,32,2,128,1,float16,float16,511,0.01390720009803772
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,2,128,1,float16,fp8,127,0.012468799948692322
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,1,128,1,float16,float16,31,0.011212799698114395
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,2,128,1,float16,fp8,255,0.013038399815559387
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,1,128,1,float16,fp8,63,0.010971199721097946
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,2,128,1,float16,fp8,511,0.016673600673675536
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,2,128,1,float16,float16,511,0.015695999562740325
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,2,128,1,float16,float16,1023,0.02885119915008545
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,2,128,1,float16,fp8,1023,0.021374399960041045
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,2,128,1,float16,float16,2047,0.044206398725509646
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,2,128,1,float16,fp8,2047,0.03234719932079315
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,2,128,1,float16,fp8,4095,0.05153120160102844
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,2,128,1,float16,float16,8191,0.13311840295791627
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,2,128,1,float16,fp8,8191,0.09001439809799194
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,2,128,1,float16,float16,16383,0.25025439262390137
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,1,128,1,float16,float16,8191,0.07566720247268677
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,2,128,1,float16,fp8,16383,0.1659119963645935
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,4,128,1,float16,float16,1,0.01419840008020401
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,4,128,1,float16,fp8,3,0.013865600526332855
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,4,128,1,float16,float16,3,0.014316800236701965
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,4,128,1,float16,float16,7,0.013980799913406372
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,4,128,1,float16,fp8,7,0.01404000073671341
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,4,128,1,float16,float16,15,0.01427839994430542
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,4,128,1,float16,fp8,15,0.014028799533843995
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,4,128,1,float16,float16,31,0.014116799831390381
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,4,128,1,float16,fp8,31,0.014144000411033631
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,2,128,1,float16,float16,3,0.01159520000219345
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,4,128,1,float16,float16,63,0.015025599300861359
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,4,128,1,float16,fp8,63,0.01412159949541092
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,2,128,1,float16,float16,7,0.011721599847078323
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,4,128,1,float16,float16,127,0.014424000680446625
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,4,128,1,float16,float16,255,0.017334400117397307
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,4,128,1,float16,fp8,255,0.017774400115013123
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,4,128,1,float16,float16,511,0.029718399047851562
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,4,128,1,float16,fp8,511,0.02412319928407669
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,4,128,1,float16,float16,1023,0.04597600102424622
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,4,128,1,float16,fp8,1023,0.034703999757766724
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,4,128,1,float16,float16,2047,0.07637439966201783
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,4,128,1,float16,fp8,2047,0.05417119860649109
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,4,128,1,float16,float16,4095,0.1346160054206848
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,4,128,1,float16,fp8,4095,0.09099040031433106
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,4,128,1,float16,float16,8191,0.2525887966156006
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,4,128,1,float16,fp8,8191,0.16757919788360595
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,2,128,1,float16,float16,255,0.01348160058259964
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,8,128,1,float16,float16,1,0.019577600061893463
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,8,128,1,float16,fp8,1,0.019649599492549897
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,8,128,1,float16,float16,3,0.019652800261974336
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,4,128,1,float16,float16,16383,0.4879136085510254
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,8,128,1,float16,fp8,3,0.01967200040817261
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,4,128,1,float16,fp8,16383,0.3184864044189453
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,2,128,1,float16,float16,4095,0.07419840097427369
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,8,128,1,float16,float16,15,0.019991999864578246
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,8,128,1,float16,fp8,7,0.019388799369335175
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,8,128,1,float16,fp8,31,0.019817599654197694
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,8,128,1,float16,float16,31,0.019871999323368073
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,8,128,1,float16,fp8,15,0.01935359984636307
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,8,128,1,float16,fp8,63,0.019968000054359437
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,8,128,1,float16,float16,127,0.02033119946718216
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,8,128,1,float16,fp8,127,0.02297919988632202
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,8,128,1,float16,fp8,255,0.02667360007762909
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,8,128,1,float16,float16,255,0.03354719877243042
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,8,128,1,float16,float16,511,0.0490880012512207
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,8,128,1,float16,fp8,511,0.040166398882865904
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,8,128,1,float16,fp8,1023,0.05818079710006714
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,8,128,1,float16,float16,1023,0.07942079901695251
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,4,128,1,float16,fp8,1,0.014078399538993836
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,8,128,1,float16,float16,2047,0.13797760009765625
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,8,128,1,float16,float16,4095,0.25318078994750975
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,8,128,1,float16,fp8,4095,0.1700559973716736
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,4,128,1,float16,fp8,127,0.01594880074262619
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,8,128,1,float16,float16,8191,0.4865104198455811
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,8,128,1,float16,fp8,8191,0.32239999771118166
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,1,128,1,float16,float16,1,0.009775999933481216
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,1,128,1,float16,fp8,1,0.009329599887132644
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,1,128,1,float16,float16,3,0.009771200269460678
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,1,128,1,float16,fp8,3,0.009244800359010697
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,1,128,1,float16,float16,7,0.009691199660301209
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,8,128,1,float16,fp8,16383,0.6173583984375
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,1,128,1,float16,float16,15,0.00965920016169548
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,8,128,1,float16,float16,16383,0.9557344436645507
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,1,128,1,float16,fp8,15,0.009321600198745728
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,1,128,1,float16,float16,31,0.009703999757766724
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,8,128,1,float16,float16,7,0.01968960016965866
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,1,128,1,float16,float16,63,0.00968799963593483
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,1,128,1,float16,float16,127,0.009729599952697754
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,1,128,1,float16,fp8,31,0.009489600360393525
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,1,128,1,float16,fp8,127,0.010249599814414978
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,1,128,1,float16,float16,255,0.010982400178909302
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,1,128,1,float16,fp8,255,0.011007999628782272
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,1,128,1,float16,fp8,511,0.011953599750995636
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,1,128,1,float16,float16,1023,0.01313599944114685
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,1,128,1,float16,fp8,1023,0.012539200484752655
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,8,128,1,float16,float16,63,0.019857600331306458
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,1,128,1,float16,fp8,2047,0.013371199369430542
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,1,128,1,float16,float16,4095,0.014345599710941315
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,1,128,1,float16,float16,8191,0.015860800445079804
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,1,128,1,float16,fp8,8191,0.015961599349975587
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,1,128,1,float16,float16,16383,0.018223999440670012
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,1,128,1,float16,fp8,16383,0.01874080002307892
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,2,128,1,float16,float16,1,0.009636799991130828
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,2,128,1,float16,fp8,1,0.009363199770450591
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,2,128,1,float16,float16,3,0.009544000029563904
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,2,128,1,float16,fp8,3,0.009454400092363358
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,2,128,1,float16,float16,7,0.009708800166845322
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,16,8,128,1,float16,fp8,2047,0.09672160148620605
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,2,128,1,float16,fp8,7,0.00952960029244423
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,2,128,1,float16,fp8,15,0.009550400078296661
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,2,128,1,float16,float16,15,0.009251199662685394
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,2,128,1,float16,float16,31,0.009771200269460678
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,2,128,1,float16,fp8,31,0.009390400350093841
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,2,128,1,float16,float16,63,0.009787199646234512
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,2,128,1,float16,fp8,63,0.009431999921798707
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,2,128,1,float16,float16,127,0.00965920016169548
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,2,128,1,float16,fp8,127,0.01005600020289421
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,2,128,1,float16,float16,255,0.010871999710798264
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,2,128,1,float16,fp8,255,0.010996799916028976
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,2,128,1,float16,float16,511,0.013035200536251068
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,2,128,1,float16,fp8,511,0.012140800058841706
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,2,128,1,float16,float16,1023,0.013232000172138214
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,2,128,1,float16,fp8,1023,0.012612800300121307
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,2,128,1,float16,float16,2047,0.013433599472045898
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,2,128,1,float16,fp8,2047,0.013174399733543396
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,2,128,1,float16,float16,4095,0.014323200285434722
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,2,128,1,float16,fp8,4095,0.014182400703430176
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,2,128,1,float16,float16,8191,0.015772800147533416
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,2,128,1,float16,fp8,8191,0.015494400262832641
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,2,128,1,float16,float16,16383,0.01767839938402176
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,2,128,1,float16,fp8,16383,0.01786080002784729
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,4,128,1,float16,float16,1,0.009513600170612336
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,4,128,1,float16,fp8,1,0.009319999814033508
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,4,128,1,float16,float16,3,0.009593600034713745
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,4,128,1,float16,fp8,3,0.009384000301361084
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,4,128,1,float16,float16,7,0.009513600170612336
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,4,128,1,float16,fp8,7,0.009385599941015243
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,4,128,1,float16,float16,15,0.009596800059080124
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,4,128,1,float16,fp8,15,0.009359999746084213
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,4,128,1,float16,float16,31,0.009635200351476669
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,4,128,1,float16,fp8,31,0.00923520028591156
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,1,128,1,float16,fp8,7,0.009249600023031235
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,4,128,1,float16,fp8,63,0.009195200353860854
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,4,128,1,float16,float16,127,0.00942080020904541
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,4,128,1,float16,fp8,127,0.010102400183677673
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,4,128,1,float16,float16,255,0.010678400099277497
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,4,128,1,float16,fp8,255,0.010995200276374817
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,4,128,1,float16,float16,511,0.01297920048236847
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,4,128,1,float16,fp8,511,0.012289600074291229
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,4,128,1,float16,float16,1023,0.013152000308036805
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,4,128,1,float16,fp8,1023,0.012727999687194824
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,4,128,1,float16,fp8,2047,0.013275200128555298
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,4,128,1,float16,float16,4095,0.014777599275112152
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,1,128,1,float16,float16,2047,0.013568000495433807
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,4,128,1,float16,fp8,4095,0.014318400621414184
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,4,128,1,float16,fp8,8191,0.015880000591278077
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,4,128,1,float16,float16,16383,0.01886879950761795
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,4,128,1,float16,fp8,16383,0.01866399943828583
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,8,128,1,float16,float16,1,0.009561599791049957
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,8,128,1,float16,fp8,1,0.009384000301361084
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,8,128,1,float16,float16,3,0.009399999678134919
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,8,128,1,float16,fp8,3,0.009363199770450591
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,8,128,1,float16,float16,7,0.00990239977836609
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,8,128,1,float16,fp8,7,0.00939520001411438
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,8,128,1,float16,float16,15,0.009438399970531464
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,8,128,1,float16,fp8,15,0.009443199634552002
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,8,128,1,float16,float16,31,0.009697599709033966
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,8,128,1,float16,fp8,31,0.009465599805116654
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,8,128,1,float16,float16,63,0.009643200039863586
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,8,128,1,float16,fp8,63,0.009468799829483033
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,8,128,1,float16,float16,127,0.00960479974746704
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,8,128,1,float16,fp8,127,0.009976000338792802
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,8,128,1,float16,float16,255,0.010785599797964096
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,8,128,1,float16,fp8,255,0.011121600121259689
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,8,128,1,float16,float16,511,0.013120000064373017
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,8,128,1,float16,fp8,511,0.012390399724245072
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,8,128,1,float16,float16,1023,0.013275200128555298
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,4,128,1,float16,float16,63,0.00966079980134964
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,8,128,1,float16,fp8,1023,0.012943999469280243
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,8,128,1,float16,fp8,2047,0.013551999628543854
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,8,128,1,float16,float16,4095,0.015374399721622467
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,8,128,1,float16,fp8,4095,0.01478399932384491
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,1,128,1,float16,fp8,63,0.009319999814033508
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,8,128,1,float16,float16,8191,0.017659200727939604
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,1,128,1,float16,float16,511,0.012980799376964568
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,8,128,1,float16,fp8,8191,0.01744000017642975
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,1,128,1,float16,fp8,4095,0.014236800372600555
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,8,128,1,float16,fp8,16383,0.022726400196552275
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,1,128,1,float16,float16,1,0.009651199728250504
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,1,128,1,float16,fp8,1,0.009484799951314926
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,1,128,1,float16,fp8,3,0.009487999975681305
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,1,128,1,float16,float16,15,0.009784000366926194
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,1,128,1,float16,fp8,15,0.009353599697351455
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,1,128,1,float16,float16,31,0.009574399888515472
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,1,128,1,float16,fp8,31,0.009480000287294389
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,1,128,1,float16,float16,63,0.009785600006580353
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,1,128,1,float16,fp8,63,0.009511999785900116
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,1,128,1,float16,float16,127,0.009707199782133103
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,1,128,1,float16,fp8,127,0.01011200025677681
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,1,128,1,float16,float16,255,0.01074879989027977
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,1,128,1,float16,fp8,255,0.010958400368690491
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,1,128,1,float16,float16,511,0.012889599800109864
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,8,128,1,float16,float16,2047,0.014180800318717957
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,1,128,1,float16,fp8,511,0.012025599926710128
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,1,128,1,float16,float16,1023,0.013342399895191193
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,1,128,1,float16,fp8,1023,0.01273760050535202
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,4,128,1,float16,float16,2047,0.013566400110721587
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,1,128,1,float16,float16,2047,0.013583999872207642
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,1,128,1,float16,fp8,2047,0.013096000254154205
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,1,128,1,float16,float16,4095,0.014481599628925323
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,1,128,1,float16,fp8,4095,0.014137600362300873
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,1,128,1,float16,float16,8191,0.01595519930124283
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,8,128,1,float16,float16,16383,0.02929919958114624
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,1,128,1,float16,float16,16383,0.018027199804782866
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,1,128,1,float16,fp8,16383,0.01817920058965683
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,1,128,1,float16,float16,3,0.009721600264310837
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,1,128,1,float16,float16,7,0.009651199728250504
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,2,128,1,float16,fp8,1,0.009283199906349182
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,1,128,1,float16,fp8,7,0.009412799775600434
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,2,128,1,float16,fp8,3,0.009193599969148637
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,2,128,1,float16,fp8,7,0.009433600306510925
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,2,128,1,float16,fp8,15,0.009251199662685394
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,2,128,1,float16,float16,31,0.009495999664068222
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,2,128,1,float16,fp8,31,0.009344000369310379
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,2,128,1,float16,float16,63,0.009558399766683578
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,2,128,1,float16,fp8,63,0.009465599805116654
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,2,128,1,float16,float16,127,0.009619200229644775
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,2,128,1,float16,fp8,127,0.010302399843931198
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,2,128,1,float16,float16,255,0.010889600217342376
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,2,128,1,float16,fp8,255,0.011020799726247787
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,2,128,1,float16,float16,511,0.012863999605178833
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,2,128,1,float16,fp8,511,0.012272000312805176
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,2,128,1,float16,float16,1023,0.01327040046453476
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,2,128,1,float16,fp8,1023,0.012656000256538392
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,2,128,1,float16,float16,2047,0.013692800700664521
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,2,128,1,float16,fp8,2047,0.013193599879741669
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,2,128,1,float16,float16,4095,0.014713600277900696
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,2,128,1,float16,fp8,4095,0.01419840008020401
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,2,128,1,float16,float16,8191,0.016209599375724793
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,2,128,1,float16,fp8,8191,0.015868799388408662
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,2,128,1,float16,float16,16383,0.018849599361419677
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,2,128,1,float16,fp8,16383,0.018334400653839112
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,4,128,1,float16,float16,1,0.009537599980831146
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,4,128,1,float16,fp8,1,0.009364800155162811
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,4,128,1,float16,float16,3,0.00963039994239807
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,4,128,1,float16,fp8,3,0.009435199946165086
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,16,4,128,1,float16,float16,8191,0.01613440066576004
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,4,128,1,float16,fp8,7,0.009747199714183807
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,4,128,1,float16,float16,15,0.009665600210428237
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,2,128,1,float16,float16,3,0.009446399658918381
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,4,128,1,float16,fp8,15,0.00958079993724823
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,4,128,1,float16,float16,31,0.009600000083446502
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,4,128,1,float16,fp8,31,0.009566400200128555
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,4,128,1,float16,float16,63,0.009647999703884125
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,4,128,1,float16,float16,127,0.009724800288677216
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,4,128,1,float16,fp8,127,0.01027199998497963
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,4,128,1,float16,float16,255,0.010932800173759461
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,4,128,1,float16,fp8,255,0.011193600296974183
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,4,128,1,float16,float16,511,0.012928000092506409
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,4,128,1,float16,fp8,511,0.012444800138473511
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,4,128,1,float16,float16,1023,0.01337919980287552
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,4,128,1,float16,fp8,1023,0.012880000472068786
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,4,128,1,float16,float16,2047,0.01406719982624054
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,4,128,1,float16,fp8,2047,0.01393119990825653
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,4,128,1,float16,float16,4095,0.015348799526691437
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,4,128,1,float16,fp8,4095,0.014800000190734863
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,4,128,1,float16,float16,8191,0.017975999414920805
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,4,128,1,float16,fp8,8191,0.017366400361061095
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,4,128,1,float16,float16,16383,0.02940160036087036
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,4,128,1,float16,fp8,16383,0.022628800570964815
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,8,128,1,float16,float16,1,0.009796799719333648
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,8,128,1,float16,fp8,1,0.009523200243711472
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,8,128,1,float16,float16,3,0.009788800030946732
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,8,128,1,float16,float16,7,0.009745600074529648
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,8,128,1,float16,fp8,7,0.009520000219345093
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,8,128,1,float16,float16,15,0.00952960029244423
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,8,128,1,float16,fp8,15,0.009444800019264222
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,1,128,1,float16,fp8,8191,0.01632159948348999
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,8,128,1,float16,float16,31,0.009636799991130828
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,4,128,1,float16,float16,7,0.009619200229644775
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,8,128,1,float16,float16,63,0.009583999961614608
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,8,128,1,float16,fp8,63,0.009542399644851684
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,2,128,1,float16,float16,1,0.009635200351476669
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,8,128,1,float16,fp8,127,0.010424000024795533
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,8,128,1,float16,fp8,255,0.011475200206041336
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,2,128,1,float16,float16,7,0.009465599805116654
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,2,128,1,float16,float16,15,0.009409599751234055
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,8,128,1,float16,fp8,511,0.012563200294971466
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,4,128,1,float16,fp8,63,0.009683199971914292
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,8,128,1,float16,fp8,1023,0.013687999546527862
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,8,128,1,float16,float16,2047,0.01536639928817749
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,8,128,1,float16,fp8,2047,0.014339199662208557
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,8,128,1,float16,float16,4095,0.017399999499320983
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,8,128,1,float16,fp8,4095,0.017041599750518797
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,8,128,1,float16,float16,8191,0.02834399938583374
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,8,128,1,float16,float16,16383,0.04339680075645447
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,8,128,1,float16,fp8,16383,0.03253119885921478
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,1,128,1,float16,float16,1,0.010355199873447418
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,1,128,1,float16,fp8,1,0.01045600026845932
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,1,128,1,float16,float16,3,0.01056319996714592
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,1,128,1,float16,fp8,3,0.010411199927330018
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,1,128,1,float16,float16,7,0.010655999928712846
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,8,128,1,float16,fp8,3,0.009550400078296661
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,1,128,1,float16,fp8,7,0.01064160019159317
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,1,128,1,float16,float16,15,0.010548800230026245
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,1,128,1,float16,fp8,15,0.010553599894046783
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,1,128,1,float16,float16,31,0.010492800176143647
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,1,128,1,float16,fp8,31,0.010471999645233154
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,8,128,1,float16,fp8,31,0.009556800127029419
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,1,128,1,float16,float16,63,0.010284800082445145
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,1,128,1,float16,fp8,63,0.010547199845314026
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,1,128,1,float16,float16,127,0.01075040027499199
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,1,128,1,float16,fp8,127,0.011588799953460693
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,1,128,1,float16,float16,255,0.012204799801111221
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,1,128,1,float16,fp8,255,0.011984000355005265
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,8,128,1,float16,float16,511,0.013230399787425995
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,1,128,1,float16,float16,511,0.014281600713729858
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,8,128,1,float16,float16,1023,0.013808000087738036
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,1,128,1,float16,fp8,1023,0.015255999565124512
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,1,128,1,float16,float16,2047,0.017763200402259826
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,1,128,1,float16,fp8,2047,0.01727840006351471
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,1,128,1,float16,float16,4095,0.029576000571250916
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,1,128,1,float16,fp8,4095,0.021907199919223786
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,8,128,1,float16,fp8,8191,0.02126079946756363
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,1,128,1,float16,fp8,8191,0.03331519961357117
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,1,128,1,float16,float16,16383,0.07578080296516418
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,1,128,1,float16,fp8,16383,0.052223998308181765
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,2,128,1,float16,float16,1,0.010639999806880952
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,2,128,1,float16,float16,3,0.010927999764680863
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,2,128,1,float16,fp8,3,0.010931199789047242
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,2,128,1,float16,float16,7,0.010999999940395355
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,2,128,1,float16,fp8,7,0.010940799862146378
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,2,128,1,float16,float16,15,0.010675200074911118
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,2,128,1,float16,fp8,15,0.010452800244092942
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,2,128,1,float16,float16,31,0.010688000172376633
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,2,128,1,float16,fp8,31,0.010715200006961823
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,2,128,1,float16,float16,63,0.01077279970049858
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,2,128,1,float16,fp8,63,0.010763200372457505
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,2,128,1,float16,float16,127,0.011124800145626067
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,2,128,1,float16,fp8,127,0.011633600294589996
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,2,128,1,float16,float16,255,0.012467200309038163
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,2,128,1,float16,fp8,255,0.01231520026922226
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,2,128,1,float16,float16,511,0.014604799449443817
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,2,128,1,float16,fp8,511,0.015532800555229187
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,2,128,1,float16,float16,1023,0.017179200053215028
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,8,128,1,float16,float16,127,0.009814400225877762
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,2,128,1,float16,fp8,1023,0.017081600427627564
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,2,128,1,float16,float16,2047,0.029500800371170043
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,2,128,1,float16,fp8,2047,0.02237119972705841
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,2,128,1,float16,float16,4095,0.044326400756835936
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,2,128,1,float16,fp8,4095,0.03334720134735107
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,1,128,1,float16,fp8,511,0.014414399862289429
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,1,128,1,float16,float16,1023,0.015403200685977936
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,2,128,1,float16,float16,8191,0.0741312026977539
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,2,128,1,float16,fp8,16383,0.09023039937019348
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,4,128,1,float16,float16,1,0.010966400057077408
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,4,128,1,float16,float16,3,0.010945600271224976
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,4,128,1,float16,fp8,1,0.010716799646615982
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,1,128,1,float16,float16,8191,0.04511840045452118
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,4,128,1,float16,float16,7,0.010711999982595444
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,4,128,1,float16,fp8,7,0.010985600203275681
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,4,128,1,float16,fp8,15,0.011022400110960007
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,2,128,1,float16,fp8,1,0.010649599879980088
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,4,128,1,float16,float16,31,0.011017599701881408
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,4,128,1,float16,fp8,31,0.010791999846696853
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,4,128,1,float16,float16,63,0.011078400164842605
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,4,128,1,float16,fp8,63,0.010623999685049058
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,4,128,1,float16,float16,127,0.011151999980211259
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,4,128,1,float16,fp8,127,0.011484800279140473
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,4,128,1,float16,float16,255,0.012583999335765839
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,4,128,1,float16,fp8,255,0.012636800110340119
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,4,128,1,float16,float16,511,0.014630399644374847
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,4,128,1,float16,fp8,511,0.016201600432395935
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,4,128,1,float16,fp8,1023,0.020366400480270386
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,4,128,1,float16,float16,1023,0.027823999524116516
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,4,128,1,float16,float16,2047,0.042910400032997134
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,4,128,1,float16,fp8,2047,0.03157280087471008
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,4,128,1,float16,fp8,4095,0.050102400779724124
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,4,128,1,float16,float16,4095,0.07262399792671204
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,4,128,1,float16,fp8,8191,0.08814560174942017
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,4,128,1,float16,float16,8191,0.13172160387039183
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,4,128,1,float16,float16,16383,0.2501728057861328
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,8,128,1,float16,float16,1,0.013240000605583191
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,8,128,1,float16,float16,3,0.013372799754142762
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,4,128,1,float16,fp8,16383,0.16329920291900635
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,8,128,1,float16,fp8,1,0.013158400356769562
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,8,128,1,float16,fp8,3,0.01311040073633194
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,8,128,1,float16,fp8,7,0.013470399379730224
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,8,128,1,float16,float16,7,0.01316159963607788
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,8,128,1,float16,float16,15,0.013315199315547943
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,8,128,1,float16,fp8,15,0.013425600528717042
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,8,128,1,float16,float16,31,0.013299199938774108
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,16,8,128,1,float16,float16,255,0.011383999884128571
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,8,128,1,float16,float16,63,0.01358720064163208
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,8,128,1,float16,fp8,63,0.013275200128555298
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,8,128,1,float16,float16,127,0.013921600580215455
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,8,128,1,float16,fp8,127,0.01526080071926117
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,8,128,1,float16,fp8,255,0.01674239933490753
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,8,128,1,float16,float16,511,0.027980801463127137
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,8,128,1,float16,fp8,511,0.023337599635124207
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,8,128,1,float16,float16,1023,0.04476960003376007
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,2,128,1,float16,float16,16383,0.13512320518493653
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,8,128,1,float16,fp8,1023,0.03346399962902069
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,8,128,1,float16,float16,2047,0.07484959959983825
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,4,128,1,float16,fp8,3,0.010979200154542923
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,8,128,1,float16,fp8,2047,0.05353119969367981
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,8,128,1,float16,float16,4095,0.1330415964126587
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,4,128,1,float16,float16,15,0.01080320030450821
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,8,128,1,float16,fp8,8191,0.1649824023246765
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,1,128,1,float16,float16,1,0.009465599805116654
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,1,128,1,float16,fp8,1,0.009464000165462495
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,8,128,1,float16,fp8,16383,0.3133568048477173
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,8,128,1,float16,float16,16383,0.487716817855835
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,1,128,1,float16,float16,3,0.009692800045013428
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,1,128,1,float16,float16,7,0.009651199728250504
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,1,128,1,float16,fp8,3,0.009331200271844864
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,1,128,1,float16,fp8,7,0.009323199838399887
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,1,128,1,float16,fp8,15,0.009201599657535553
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,1,128,1,float16,float16,31,0.00957920029759407
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,1,128,1,float16,float16,63,0.009801600128412247
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,1,128,1,float16,fp8,63,0.009321600198745728
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,1,128,1,float16,float16,127,0.009668800234794616
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,1,128,1,float16,fp8,127,0.010225600004196167
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,1,128,1,float16,float16,255,0.010966400057077408
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,1,128,1,float16,fp8,255,0.010900799930095673
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,1,128,1,float16,float16,511,0.012915199995040894
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,1,128,1,float16,fp8,511,0.012540799379348756
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,1,128,1,float16,float16,1023,0.013736000657081604
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,1,128,1,float16,fp8,1023,0.013286399841308593
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,1,128,1,float16,float16,2047,0.013972799479961395
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,1,128,1,float16,fp8,2047,0.013436800241470337
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,1,128,1,float16,float16,4095,0.015015999972820281
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,8,128,1,float16,float16,255,0.016865600645542145
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,1,128,1,float16,float16,8191,0.016383999586105348
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,2,128,1,float16,fp8,8191,0.05199679732322693
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,1,128,1,float16,fp8,8191,0.015855999290943147
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,1,128,1,float16,float16,16383,0.019075199961662292
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,2,128,1,float16,float16,1,0.009620799869298934
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,2,128,1,float16,fp8,1,0.009292799979448318
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,8,128,1,float16,fp8,4095,0.09071679711341858
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,2,128,1,float16,fp8,3,0.009412799775600434
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,2,128,1,float16,float16,7,0.00950239971280098
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,2,128,1,float16,fp8,7,0.009305600076913834
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,2,128,1,float16,float16,15,0.009755200147628785
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,2,128,1,float16,fp8,15,0.009531199932098389
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,2,128,1,float16,float16,31,0.009667199850082398
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,2,128,1,float16,fp8,31,0.009414400160312652
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,2,128,1,float16,float16,63,0.009651199728250504
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,2,128,1,float16,fp8,63,0.009443199634552002
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,1,128,1,float16,float16,15,0.009696000069379807
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,2,128,1,float16,float16,127,0.00971359983086586
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,1,128,1,float16,fp8,31,0.009096000343561172
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,2,128,1,float16,fp8,127,0.01037440001964569
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,2,128,1,float16,fp8,255,0.011124800145626067
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,2,128,1,float16,fp8,511,0.0124719999730587
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,2,128,1,float16,float16,1023,0.013582399487495423
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,2,128,1,float16,fp8,1023,0.013070400059223174
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,2,128,1,float16,float16,2047,0.014207999408245086
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,2,128,1,float16,fp8,2047,0.013799999654293061
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,2,128,1,float16,float16,4095,0.015150399506092071
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,2,128,1,float16,fp8,4095,0.014502400159835815
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,2,128,1,float16,float16,8191,0.017849600315093993
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,2,128,1,float16,fp8,8191,0.01735839992761612
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,8,128,1,float16,fp8,31,0.013177600502967835
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,2,128,1,float16,float16,16383,0.029606398940086365
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,4,128,1,float16,float16,1,0.009588800370693207
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,2,128,1,float16,fp8,16383,0.022443200647830962
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,4,128,1,float16,fp8,1,0.009412799775600434
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,4,128,1,float16,float16,3,0.009743999689817429
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,4,128,1,float16,fp8,3,0.009438399970531464
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,4,128,1,float16,float16,7,0.009759999811649323
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,4,128,1,float16,fp8,7,0.009446399658918381
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,4,128,1,float16,float16,15,0.00979359969496727
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,4,128,1,float16,fp8,15,0.009497600048780442
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,1,128,1,float16,fp8,16383,0.01841440051794052
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,4,128,1,float16,float16,31,0.009812799841165542
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,4,128,1,float16,float16,63,0.009811200201511383
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,4,128,1,float16,fp8,63,0.009515199810266495
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,4,128,1,float16,float16,127,0.009931199997663499
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,4,128,1,float16,fp8,127,0.010463999956846238
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,2,128,1,float16,float16,3,0.009657599776983262
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,4,128,1,float16,fp8,255,0.011215999722480774
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,4,128,1,float16,float16,511,0.01295199990272522
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,4,128,1,float16,fp8,511,0.01239520013332367
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,16,8,128,1,float16,float16,8191,0.25074241161346433
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,4,128,1,float16,fp8,1023,0.013760000467300415
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,4,128,1,float16,fp8,2047,0.014302399754524232
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,4,128,1,float16,float16,2047,0.0150736004114151
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,4,128,1,float16,float16,4095,0.01759520024061203
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,4,128,1,float16,fp8,4095,0.01677599996328354
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,4,128,1,float16,float16,8191,0.028377598524093627
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,4,128,1,float16,fp8,8191,0.021113599836826324
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,4,128,1,float16,float16,16383,0.043961599469184875
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,4,128,1,float16,fp8,16383,0.03272800147533417
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,8,128,1,float16,float16,1,0.010012800246477127
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,8,128,1,float16,fp8,1,0.009777600318193436
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,8,128,1,float16,float16,3,0.009859199821949004
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,8,128,1,float16,fp8,3,0.00979359969496727
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,2,128,1,float16,float16,255,0.01117599979043007
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,8,128,1,float16,fp8,7,0.009769599884748459
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,2,128,1,float16,float16,511,0.013191999495029449
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,8,128,1,float16,fp8,15,0.009830400347709656
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,8,128,1,float16,float16,31,0.00971359983086586
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,8,128,1,float16,fp8,31,0.00971520021557808
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,8,128,1,float16,float16,63,0.009929600358009338
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,8,128,1,float16,fp8,63,0.01013600006699562
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,8,128,1,float16,float16,127,0.010159999877214432
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,8,128,1,float16,fp8,127,0.01090560033917427
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,8,128,1,float16,float16,255,0.011548800021409988
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,8,128,1,float16,fp8,255,0.011495999991893768
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,8,128,1,float16,float16,511,0.013739199936389923
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,8,128,1,float16,fp8,511,0.01319040060043335
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,8,128,1,float16,float16,1023,0.014815999567508698
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,1,128,1,float16,fp8,4095,0.014929600059986115
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,8,128,1,float16,fp8,1023,0.014187200367450714
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,8,128,1,float16,float16,2047,0.016289600729942323
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,8,128,1,float16,fp8,2047,0.01660960018634796
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,8,128,1,float16,float16,4095,0.02815839946269989
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,8,128,1,float16,fp8,4095,0.021115200221538545
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,8,128,1,float16,float16,8191,0.04413760006427765
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,4,128,1,float16,fp8,31,0.009547200053930283
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,8,128,1,float16,fp8,16383,0.050800001621246337
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,4,128,1,float16,float16,255,0.011081600189208984
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,1,128,1,float16,float16,1,0.013102400302886962
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,1,128,1,float16,fp8,1,0.012601600587368011
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,1,128,1,float16,fp8,3,0.012745599448680877
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,1,128,1,float16,float16,7,0.012484800070524216
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,1,128,1,float16,fp8,7,0.012435200065374375
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,1,128,1,float16,float16,15,0.012481600046157837
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,1,128,1,float16,fp8,15,0.012366399914026261
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,1,128,1,float16,float16,31,0.012464000284671784
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,1,128,1,float16,fp8,31,0.01273919939994812
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,1,128,1,float16,float16,63,0.012763200700283051
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,1,128,1,float16,fp8,63,0.012785600125789642
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,8,128,1,float16,float16,7,0.009787199646234512
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,1,128,1,float16,float16,127,0.01313440054655075
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,8,128,1,float16,float16,15,0.010011199861764908
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,1,128,1,float16,float16,255,0.014884799718856812
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,1,128,1,float16,fp8,127,0.013966399431228637
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,1,128,1,float16,float16,511,0.0168272003531456
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,1,128,1,float16,fp8,511,0.018489600718021394
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,1,128,1,float16,float16,1023,0.030096000432968138
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,1,128,1,float16,float16,2047,0.04481599926948547
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,1,128,1,float16,fp8,2047,0.03392800092697144
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,1,128,1,float16,float16,4095,0.07489280104637146
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,1,128,1,float16,fp8,4095,0.054155200719833374
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,1,128,1,float16,float16,8191,0.13390239477157592
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,8,128,1,float16,fp8,8191,0.03225440084934235
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,1,128,1,float16,fp8,8191,0.09215520024299621
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,2,128,1,float16,float16,1,0.015462400019168853
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,8,128,1,float16,float16,16383,0.07340800166130065
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,2,128,1,float16,float16,3,0.015320000052452088
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,2,128,1,float16,fp8,3,0.01520799994468689
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,2,128,1,float16,float16,7,0.015862399339675905
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,2,128,1,float16,fp8,7,0.015294399857521058
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,16,4,128,1,float16,float16,1023,0.013792000710964203
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,2,128,1,float16,fp8,15,0.015238399803638458
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,2,128,1,float16,float16,31,0.015374399721622467
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,2,128,1,float16,fp8,31,0.01523520052433014
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,2,128,1,float16,float16,63,0.015334400534629821
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,2,128,1,float16,fp8,63,0.015430399775505066
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,2,128,1,float16,float16,127,0.015358400344848634
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,2,128,1,float16,fp8,127,0.017107200622558594
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,2,128,1,float16,float16,255,0.017844800651073457
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,2,128,1,float16,fp8,255,0.018995200097560883
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,2,128,1,float16,float16,511,0.031204798817634584
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,2,128,1,float16,fp8,511,0.025630399584770203
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,2,128,1,float16,float16,1023,0.04769119918346405
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,2,128,1,float16,fp8,1023,0.036550399661064145
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,1,128,1,float16,fp8,255,0.01483680009841919
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,2,128,1,float16,fp8,2047,0.05651199817657471
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,2,128,1,float16,float16,2047,0.07771840095520019
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,2,128,1,float16,float16,4095,0.13619359731674194
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,1,128,1,float16,fp8,1023,0.023281599581241607
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,4,128,1,float16,float16,1,0.021406400203704833
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,2,128,1,float16,fp8,4095,0.09340320229530334
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,4,128,1,float16,fp8,1,0.021240000426769257
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,4,128,1,float16,float16,3,0.021209600567817687
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,4,128,1,float16,float16,7,0.021294400095939636
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,4,128,1,float16,fp8,3,0.020895999670028687
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,4,128,1,float16,fp8,7,0.021081599593162536
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,4,128,1,float16,float16,15,0.02140959948301315
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,4,128,1,float16,fp8,15,0.020960000157356263
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,4,128,1,float16,float16,31,0.021355199813842773
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,4,128,1,float16,fp8,31,0.021163199841976166
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,4,128,1,float16,float16,63,0.02126239985227585
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,4,128,1,float16,float16,127,0.021932800114154816
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,4,128,1,float16,fp8,127,0.024251200258731842
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,4,128,1,float16,float16,255,0.035390400886535646
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,4,128,1,float16,fp8,255,0.027779200673103334
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,4,128,1,float16,float16,511,0.04936000108718872
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,4,128,1,float16,fp8,511,0.04197919964790344
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,4,128,1,float16,float16,1023,0.08146880269050598
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,4,128,1,float16,fp8,1023,0.05947999954223633
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,2,128,1,float16,fp8,1,0.01544319987297058
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,4,128,1,float16,float16,2047,0.1400048017501831
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,4,128,1,float16,fp8,2047,0.09884639978408813
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,1,128,1,float16,float16,3,0.013033600151538849
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,2,128,1,float16,float16,15,0.01549600064754486
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,4,128,1,float16,fp8,4095,0.1732416033744812
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,4,128,1,float16,float16,4095,0.2558527946472168
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,8,128,1,float16,float16,1,0.03225759863853454
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,8,128,1,float16,fp8,1,0.03172959983348846
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,8,128,1,float16,float16,3,0.03222239911556244
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,8,128,1,float16,fp8,3,0.03207040131092072
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,8,128,1,float16,float16,7,0.032451200485229495
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,4,128,1,float16,fp8,8191,0.32286241054534914
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,8,128,1,float16,float16,15,0.03293440043926239
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,8,128,1,float16,fp8,7,0.031744000315666196
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,8,128,1,float16,float16,31,0.03288320004940033
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,8,128,1,float16,fp8,15,0.031998398900032046
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,8,128,1,float16,fp8,31,0.031937599182128906
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,8,128,1,float16,float16,63,0.03259040117263794
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,8,128,1,float16,fp8,63,0.03203360140323639
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,8,128,1,float16,fp8,127,0.03903360068798065
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,8,128,1,float16,float16,255,0.05551519989967346
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,8,128,1,float16,fp8,255,0.048844799399375916
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,8,128,1,float16,float16,511,0.08407199978828431
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,8,128,1,float16,float16,1023,0.14429759979248047
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,2,128,1,float16,fp8,8191,0.1703711986541748
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,2,128,1,float16,float16,8191,0.254531192779541
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,8,128,1,float16,fp8,1023,0.10636320114135742
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,4,128,1,float16,fp8,63,0.02106720060110092
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,8,128,1,float16,float16,2047,0.2627487897872925
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,8,128,1,float16,float16,4095,0.4899439811706543
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,8,128,1,float16,fp8,4095,0.33341920375823975
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,1,128,1,float16,float16,1,0.0176816001534462
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,1,128,1,float16,fp8,1,0.01757120043039322
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,1,128,1,float16,float16,3,0.017713600397109987
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,1,128,1,float16,fp8,3,0.01767359972000122
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,1,128,1,float16,float16,7,0.017936000227928163
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,1,128,1,float16,fp8,7,0.017828799784183502
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,8,128,1,float16,fp8,8191,0.6447535991668701
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,4,128,1,float16,float16,8191,0.4884335994720459
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,1,128,1,float16,fp8,15,0.017576000094413756
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,1,128,1,float16,float16,31,0.018267199397087097
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,1,128,1,float16,fp8,31,0.017975999414920805
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,1,128,1,float16,float16,63,0.01780160069465637
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,1,128,1,float16,fp8,63,0.01754239946603775
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,1,128,1,float16,float16,127,0.01785600036382675
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,8,128,1,float16,float16,127,0.040299201011657716
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,1,128,1,float16,fp8,127,0.019303999841213226
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,1,128,1,float16,fp8,255,0.021220800280570985
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,1,128,1,float16,float16,511,0.03355039954185486
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,1,128,1,float16,fp8,511,0.02829279899597168
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,8,128,1,float16,fp8,511,0.07156320214271546
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,1,128,1,float16,fp8,1023,0.03953759968280792
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,1,128,1,float16,float16,2047,0.08046879768371581
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,1,128,1,float16,fp8,2047,0.059539198875427246
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,2,128,1,float16,float16,1,0.023815999925136565
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,2,128,1,float16,fp8,1,0.023337599635124207
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,2,128,1,float16,float16,3,0.023715199530124666
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,2,128,1,float16,fp8,3,0.02340960055589676
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,2,128,1,float16,float16,7,0.023769600689411162
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,2,128,1,float16,float16,15,0.023771199584007262
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,8,128,1,float16,fp8,2047,0.18571360111236573
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,2,128,1,float16,fp8,15,0.023558400571346283
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,2,128,1,float16,fp8,31,0.023526400327682495
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,2,128,1,float16,float16,63,0.023992000520229338
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,2,128,1,float16,fp8,63,0.02359199970960617
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,2,128,1,float16,float16,127,0.02428479939699173
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,2,128,1,float16,fp8,127,0.027107200026512145
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,2,128,1,float16,float16,255,0.0373744010925293
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,2,128,1,float16,fp8,255,0.030852800607681273
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,2,128,1,float16,float16,511,0.052527999877929686
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,16,8,128,1,float16,float16,8191,0.9551808357238769
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,1,128,1,float16,float16,15,0.017764799296855927
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,2,128,1,float16,fp8,511,0.04548799991607666
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,2,128,1,float16,fp8,1023,0.06397119760513306
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,2,128,1,float16,float16,1023,0.09109920263290405
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,2,128,1,float16,float16,2047,0.1433135986328125
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,4,128,1,float16,float16,1,0.0348143994808197
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,4,128,1,float16,fp8,1,0.034745600819587705
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,2,128,1,float16,fp8,2047,0.10369759798049927
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,4,128,1,float16,fp8,3,0.03470079898834229
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,1,128,1,float16,float16,255,0.020878399908542632
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,4,128,1,float16,float16,7,0.034806400537490845
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,4,128,1,float16,fp8,7,0.03467040061950684
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,4,128,1,float16,float16,15,0.03476319909095764
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,4,128,1,float16,fp8,15,0.034702399373054506
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,4,128,1,float16,float16,31,0.035051199793815616
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,4,128,1,float16,fp8,31,0.034544000029563905
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,1,128,1,float16,float16,1023,0.05098559856414795
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,4,128,1,float16,fp8,63,0.034904000163078305
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,4,128,1,float16,float16,127,0.04270879924297333
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,4,128,1,float16,fp8,127,0.04201439917087555
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,4,128,1,float16,fp8,255,0.051734399795532224
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,4,128,1,float16,float16,511,0.08704959750175476
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,4,128,1,float16,float16,1023,0.14770079851150514
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,2,128,1,float16,fp8,7,0.023425599932670592
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,4,128,1,float16,fp8,1023,0.1103775978088379
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,8,128,1,float16,float16,1,0.0559503972530365
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,4,128,1,float16,float16,2047,0.2632175922393799
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,8,128,1,float16,fp8,1,0.055851197242736815
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,4,128,1,float16,fp8,2047,0.18768800497055055
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,8,128,1,float16,fp8,3,0.05597119927406311
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,8,128,1,float16,float16,3,0.05605760216712952
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,8,128,1,float16,float16,7,0.056734400987625125
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,8,128,1,float16,float16,15,0.056790399551391604
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,8,128,1,float16,fp8,15,0.056276798248291016
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,8,128,1,float16,float16,31,0.05675519704818725
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,8,128,1,float16,fp8,31,0.05624480247497558
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,8,128,1,float16,float16,63,0.061887997388839724
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,8,128,1,float16,fp8,63,0.05669119954109192
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,8,128,1,float16,float16,127,0.06779519915580749
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,8,128,1,float16,fp8,127,0.07311519980430603
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,8,128,1,float16,float16,255,0.09585919976234436
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,8,128,1,float16,fp8,255,0.0885807991027832
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,4,128,1,float16,float16,3,0.035148799419403076
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,8,128,1,float16,float16,511,0.15436480045318604
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,4,128,1,float16,float16,63,0.03551200032234192
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,8,128,1,float16,fp8,511,0.13304320573806763
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,8,128,1,float16,float16,1023,0.2715440034866333
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,4,128,1,float16,fp8,511,0.07487040162086486
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,8,128,1,float16,fp8,1023,0.20404160022735596
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,1,128,1,float16,float16,1,0.029022398591041564
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,1,128,1,float16,float16,3,0.029047998785972595
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,8,128,1,float16,float16,2047,0.4991151809692383
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,1,128,1,float16,fp8,1,0.029022398591041564
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,1,128,1,float16,fp8,3,0.028934401273727418
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,8,128,1,float16,fp8,2047,0.3656896114349365
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,1,128,1,float16,fp8,7,0.029078400135040282
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,1,128,1,float16,float16,7,0.02922079861164093
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,2,128,1,float16,float16,31,0.023662400245666505
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,1,128,1,float16,fp8,15,0.02943840026855469
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,1,128,1,float16,fp8,31,0.029016000032424927
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,1,128,1,float16,float16,63,0.029289600253105164
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,1,128,1,float16,float16,127,0.030160000920295714
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,1,128,1,float16,fp8,127,0.03245440125465393
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,8,128,1,float16,fp8,7,0.05594559907913208
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,1,128,1,float16,fp8,255,0.03655360043048859
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,1,128,1,float16,fp8,511,0.05168799757957458
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,1,128,1,float16,float16,511,0.05848960280418396
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,1,128,1,float16,fp8,1023,0.07027040123939514
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,2,128,1,float16,float16,1,0.04047360122203827
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,2,128,1,float16,fp8,1,0.04042559862136841
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,2,128,1,float16,float16,3,0.04050880074501038
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,2,128,1,float16,fp8,3,0.040398401021957395
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,2,128,1,float16,float16,7,0.04039520025253296
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,2,128,1,float16,fp8,7,0.0402976006269455
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,2,128,1,float16,float16,15,0.04057439863681793
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,2,128,1,float16,fp8,15,0.04041439890861511
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,2,128,1,float16,float16,31,0.04072319865226746
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,2,128,1,float16,fp8,31,0.04048640131950378
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,2,128,1,float16,float16,63,0.040996798872947694
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,16,4,128,1,float16,float16,255,0.05865439772605896
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,2,128,1,float16,fp8,63,0.04046559929847717
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,2,128,1,float16,float16,127,0.0485727995634079
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,2,128,1,float16,fp8,127,0.047440001368522645
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,2,128,1,float16,float16,255,0.06518080234527587
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,2,128,1,float16,fp8,255,0.05791199803352356
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,2,128,1,float16,float16,511,0.09328160285949708
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,2,128,1,float16,fp8,511,0.08264319896697998
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,1,128,1,float16,float16,15,0.029187199473381043
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,2,128,1,float16,float16,1023,0.1542688012123108
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,4,128,1,float16,float16,1,0.061153602600097653
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,2,128,1,float16,fp8,1023,0.1184880018234253
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,4,128,1,float16,fp8,1,0.06130399703979492
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,4,128,1,float16,fp8,3,0.061612802743911746
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,4,128,1,float16,float16,7,0.0620576024055481
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,4,128,1,float16,float16,15,0.061238402128219606
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,1,128,1,float16,fp8,63,0.029462400078773498
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,4,128,1,float16,fp8,15,0.061615997552871705
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,4,128,1,float16,float16,31,0.06143519878387451
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,1,128,1,float16,float16,255,0.04256480038166046
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,4,128,1,float16,float16,63,0.06719840168952942
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,4,128,1,float16,fp8,63,0.06234239935874939
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,4,128,1,float16,float16,127,0.07465280294418335
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,4,128,1,float16,fp8,127,0.07976639866828919
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,4,128,1,float16,float16,255,0.10214079618453979
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,4,128,1,float16,fp8,255,0.0961184024810791
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,1,128,1,float16,float16,1023,0.09192320108413696
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,4,128,1,float16,float16,511,0.15881439447402954
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,4,128,1,float16,fp8,511,0.14088319540023803
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,8,128,1,float16,float16,1,0.10282399654388427
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,4,128,1,float16,fp8,1023,0.21174399852752684
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,8,128,1,float16,float16,3,0.10292799472808838
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,8,128,1,float16,fp8,3,0.10348639488220215
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,8,128,1,float16,float16,7,0.1028704047203064
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,8,128,1,float16,fp8,7,0.1023743987083435
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,8,128,1,float16,float16,15,0.10287200212478638
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,8,128,1,float16,fp8,15,0.10280799865722656
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,8,128,1,float16,float16,31,0.10462559461593628
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,8,128,1,float16,fp8,31,0.1028175950050354
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,8,128,1,float16,float16,63,0.11374560594558716
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,8,128,1,float16,fp8,63,0.11163519620895386
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,8,128,1,float16,float16,127,0.12472480535507202
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,8,128,1,float16,fp8,127,0.13603520393371582
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,8,128,1,float16,float16,255,0.1763856053352356
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,8,128,1,float16,fp8,255,0.16741119623184203
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,1,128,1,float16,float16,31,0.029329600930213928
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,4,128,1,float16,float16,3,0.06133599877357483
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,4,128,1,float16,fp8,7,0.0616096019744873
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,1,128,1,float16,float16,1,0.010145600140094756
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,8,128,1,float16,fp8,511,0.25391039848327634
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,1,128,1,float16,fp8,1,0.009998399764299393
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,4,128,1,float16,fp8,31,0.061972802877426146
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,1,128,1,float16,fp8,3,0.010049600154161453
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,1,128,1,float16,float16,3,0.010017599910497665
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,1,128,1,float16,fp8,7,0.010020799934864044
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,8,128,1,float16,float16,1023,0.52183518409729
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,1,128,1,float16,float16,15,0.009984000027179718
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,1,128,1,float16,fp8,15,0.009860800206661224
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,8,128,1,float16,fp8,1023,0.3942640066146851
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,1,128,1,float16,fp8,31,0.009774400293827057
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,1,128,1,float16,fp8,63,0.009799999743700027
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,1,128,1,float16,float16,31,0.009835200011730194
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,1,128,1,float16,float16,127,0.010236799716949463
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,1,128,1,float16,fp8,127,0.01043199971318245
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,1,128,1,float16,float16,255,0.011406400054693223
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,1,128,1,float16,fp8,255,0.011327999830245971
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,1,128,1,float16,float16,511,0.013155199587345123
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,1,128,1,float16,fp8,511,0.012718400359153748
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,1,128,1,float16,float16,1023,0.013542400300502777
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,1,128,1,float16,fp8,1023,0.01327040046453476
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,1,128,1,float16,fp8,2047,0.014151999354362487
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,1,128,1,float16,float16,4095,0.015775999426841734
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,1,128,1,float16,fp8,4095,0.014737600088119506
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,1,128,1,float16,float16,8191,0.018134400248527527
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,1,128,1,float16,fp8,8191,0.017679999768733978
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,1,128,1,float16,float16,16383,0.030142399668693542
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,1,128,1,float16,fp8,16383,0.022988800704479218
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,2,128,1,float16,float16,1,0.009571199864149093
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,2,128,1,float16,fp8,1,0.009436800330877303
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,2,128,1,float16,float16,3,0.00981760025024414
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,2,128,1,float16,fp8,3,0.009683199971914292
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,2,128,1,float16,float16,7,0.009782399982213974
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,2,128,1,float16,fp8,7,0.009673599898815156
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,2,128,1,float16,float16,15,0.009815999865531921
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,2,128,1,float16,fp8,15,0.00957920029759407
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,2,128,1,float16,float16,31,0.009636799991130828
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,2,128,1,float16,fp8,31,0.0094480000436306
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,2,128,1,float16,float16,63,0.009672000259160995
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,8,128,1,float16,float16,511,0.2922976016998291
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,2,128,1,float16,fp8,63,0.009406399726867676
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,2,128,1,float16,fp8,127,0.010358399897813796
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,2,128,1,float16,float16,127,0.010292799770832061
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,2,128,1,float16,float16,255,0.011097600311040878
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,2,128,1,float16,fp8,255,0.011407999694347382
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,2,128,1,float16,float16,511,0.013236799836158752
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,1,128,1,float16,float16,7,0.010153599828481675
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,2,128,1,float16,fp8,511,0.012806400656700134
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,2,128,1,float16,float16,1023,0.014169600605964661
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,2,128,1,float16,fp8,1023,0.013526399433612824
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,2,128,1,float16,fp8,2047,0.014289599657058717
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,2,128,1,float16,float16,4095,0.017686399817466735
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,2,128,1,float16,fp8,4095,0.016795200109481812
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,2,128,1,float16,float16,8191,0.028835201263427736
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,1,128,1,float16,float16,63,0.010294400155544281
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,2,128,1,float16,fp8,8191,0.02129279971122742
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,2,128,1,float16,float16,16383,0.04423680007457733
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,4,128,1,float16,float16,1,0.009991999715566635
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,2,128,1,float16,fp8,16383,0.03332160115242004
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,4,128,1,float16,fp8,1,0.009867200255393982
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,4,128,1,float16,float16,1023,0.27792959213256835
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,4,128,1,float16,float16,3,0.009963200241327286
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,4,128,1,float16,float16,7,0.010195200145244599
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,4,128,1,float16,fp8,3,0.010038399696350097
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,4,128,1,float16,fp8,7,0.009886399656534196
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,4,128,1,float16,fp8,15,0.010212799906730652
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,4,128,1,float16,float16,31,0.010067199915647506
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,4,128,1,float16,fp8,31,0.009761600196361542
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,4,128,1,float16,float16,15,0.009825599938631057
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,4,128,1,float16,float16,127,0.010233599692583084
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,4,128,1,float16,fp8,127,0.010873600095510482
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,4,128,1,float16,float16,255,0.011606399714946748
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,4,128,1,float16,fp8,255,0.011561600118875503
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,4,128,1,float16,float16,511,0.013510400056838989
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,4,128,1,float16,fp8,511,0.013494400680065155
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,4,128,1,float16,float16,1023,0.015166400372982025
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,4,128,1,float16,fp8,1023,0.014123199880123139
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,4,128,1,float16,float16,2047,0.01647839993238449
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,4,128,1,float16,fp8,2047,0.016707199811935424
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,4,128,1,float16,float16,4095,0.028630399703979494
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,4,128,1,float16,fp8,4095,0.02099519968032837
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,4,128,1,float16,float16,8191,0.043612799048423766
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,4,128,1,float16,fp8,8191,0.032416000962257385
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,4,128,1,float16,float16,16383,0.07387359738349915
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,4,128,1,float16,fp8,16383,0.05050240159034729
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,8,128,1,float16,float16,1,0.009934400022029877
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,8,128,1,float16,float16,3,0.010307200253009796
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,8,128,1,float16,fp8,3,0.009736000001430512
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,8,128,1,float16,float16,7,0.010185600072145463
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,8,128,1,float16,fp8,7,0.009763199836015701
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,8,128,1,float16,float16,15,0.010198400169610978
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,8,128,1,float16,fp8,15,0.009804800152778625
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,8,128,1,float16,float16,31,0.010308799892663955
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,8,128,1,float16,float16,63,0.010203199833631516
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,8,128,1,float16,fp8,63,0.010038399696350097
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,8,128,1,float16,float16,127,0.010620799660682679
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,8,128,1,float16,fp8,127,0.011049599945545196
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,8,128,1,float16,float16,255,0.011886399984359742
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,8,128,1,float16,fp8,255,0.011804799735546111
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,8,128,1,float16,float16,511,0.013937599956989288
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,2,128,1,float16,float16,2047,0.01557600051164627
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,8,128,1,float16,fp8,511,0.014769600331783294
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,8,128,1,float16,fp8,1023,0.016476799547672272
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,8,128,1,float16,float16,2047,0.02821280062198639
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,8,128,1,float16,fp8,2047,0.021670399606227873
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,16,8,128,1,float16,fp8,1,0.10234719514846802
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,8,128,1,float16,float16,4095,0.043382400274276735
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,1,128,1,float16,float16,2047,0.01430719941854477
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,8,128,1,float16,fp8,4095,0.032014399766922
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,4,128,1,float16,float16,63,0.010227199643850327
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,4,128,1,float16,fp8,63,0.009995199739933014
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,8,128,1,float16,fp8,8191,0.05086719989776611
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,8,128,1,float16,float16,16383,0.13211519718170167
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,8,128,1,float16,fp8,16383,0.08793600201606751
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,1,128,1,float16,float16,1,0.04685760140419006
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,1,128,1,float16,fp8,1,0.04635519981384277
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,1,128,1,float16,fp8,3,0.04656639993190766
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,1,128,1,float16,float16,3,0.04654879868030548
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,1,128,1,float16,fp8,7,0.046675199270248414
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,1,128,1,float16,fp8,15,0.046751999855041505
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,1,128,1,float16,float16,31,0.04701279997825623
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,8,128,1,float16,fp8,1,0.01016160026192665
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,1,128,1,float16,fp8,31,0.04675360023975372
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,1,128,1,float16,float16,63,0.048110398650169375
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,1,128,1,float16,fp8,63,0.04675680100917816
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,8,128,1,float16,fp8,31,0.010019200295209885
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,1,128,1,float16,float16,127,0.05564320087432861
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,1,128,1,float16,fp8,127,0.054360002279281616
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,1,128,1,float16,fp8,255,0.06535840034484863
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,1,128,1,float16,float16,511,0.10011039972305298
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,1,128,1,float16,fp8,511,0.0900496006011963
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,2,128,1,float16,float16,1,0.06788480281829834
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,2,128,1,float16,fp8,1,0.06823199987411499
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,2,128,1,float16,float16,3,0.06817920207977295
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,2,128,1,float16,fp8,3,0.06830080151557923
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,2,128,1,float16,float16,7,0.06808639764785766
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,2,128,1,float16,fp8,7,0.06825439929962158
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,2,128,1,float16,float16,15,0.06808800101280213
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,2,128,1,float16,fp8,15,0.06836959719657898
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,2,128,1,float16,float16,31,0.06808480024337768
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,2,128,1,float16,fp8,31,0.06863999962806702
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,8,128,1,float16,float16,8191,0.07358880043029785
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,2,128,1,float16,float16,63,0.07505120038986206
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,2,128,1,float16,float16,127,0.08224959969520569
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,2,128,1,float16,fp8,127,0.08681600093841553
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,2,128,1,float16,float16,255,0.11184320449829102
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,2,128,1,float16,fp8,255,0.10414719581604004
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,2,128,1,float16,float16,511,0.16505279541015624
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,1,128,1,float16,float16,7,0.04645760059356689
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,1,128,1,float16,float16,15,0.04684639871120453
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,4,128,1,float16,float16,1,0.10972479581832886
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,4,128,1,float16,fp8,1,0.10965119600296021
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,4,128,1,float16,float16,3,0.10952479839324951
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,4,128,1,float16,fp8,3,0.1101807951927185
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,4,128,1,float16,float16,7,0.10936160087585449
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,4,128,1,float16,fp8,7,0.1095471978187561
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,4,128,1,float16,fp8,15,0.1094655990600586
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,4,128,1,float16,float16,31,0.11493760347366333
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,1,128,1,float16,float16,255,0.07084320187568664
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,4,128,1,float16,float16,63,0.12079520225524902
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,4,128,1,float16,fp8,63,0.12069920301437378
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,4,128,1,float16,float16,127,0.1309167981147766
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,4,128,1,float16,fp8,127,0.14485280513763427
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,16,8,128,1,float16,float16,1023,0.016548800468444824
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,4,128,1,float16,float16,255,0.18367040157318115
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,4,128,1,float16,fp8,255,0.1759168028831482
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,8,128,1,float16,float16,1,0.19343680143356323
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,4,128,1,float16,float16,511,0.30195839405059816
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,2,128,1,float16,fp8,63,0.0710591971874237
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,8,128,1,float16,fp8,1,0.19101120233535768
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,8,128,1,float16,float16,3,0.1924239993095398
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,8,128,1,float16,fp8,3,0.1900480031967163
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,8,128,1,float16,float16,7,0.19348160028457642
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,8,128,1,float16,fp8,7,0.18958239555358886
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,8,128,1,float16,float16,15,0.19337120056152343
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,8,128,1,float16,fp8,15,0.19026559591293335
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,8,128,1,float16,float16,31,0.20413920879364014
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,8,128,1,float16,fp8,31,0.1990288019180298
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,8,128,1,float16,float16,63,0.2140752077102661
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,8,128,1,float16,fp8,63,0.2082911968231201
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,4,128,1,float16,float16,15,0.11002720594406128
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,8,128,1,float16,fp8,127,0.2561840057373047
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,8,128,1,float16,float16,255,0.33925600051879884
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,4,128,1,float16,fp8,31,0.11006879806518555
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,8,128,1,float16,fp8,255,0.31405439376831057
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,16,1,128,1,float16,float16,1,0.07888479828834534
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,16,1,128,1,float16,fp8,1,0.07895519733428955
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,16,1,128,1,float16,fp8,3,0.0791920006275177
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,16,1,128,1,float16,float16,7,0.079312002658844
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,16,1,128,1,float16,float16,3,0.07897440195083619
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,8,128,1,float16,fp8,511,0.4832304000854492
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,16,1,128,1,float16,float16,15,0.07920640110969543
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,16,1,128,1,float16,fp8,15,0.07963200211524964
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,16,1,128,1,float16,fp8,31,0.07929919958114624
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,16,1,128,1,float16,float16,31,0.08055359721183777
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,16,1,128,1,float16,float16,63,0.08940160274505615
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,16,1,128,1,float16,float16,127,0.09480159878730773
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,16,1,128,1,float16,fp8,127,0.0999888002872467
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,4,128,1,float16,fp8,511,0.2648112058639526
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,16,1,128,1,float16,float16,255,0.11856640577316284
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,16,1,128,1,float16,fp8,255,0.11613919734954833
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,16,2,128,1,float16,fp8,1,0.12002719640731811
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,16,2,128,1,float16,float16,1,0.11980479955673218
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,16,2,128,1,float16,float16,3,0.11970399618148804
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,16,2,128,1,float16,fp8,3,0.11999520063400268
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,16,2,128,1,float16,float16,7,0.11931519508361817
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,16,2,128,1,float16,fp8,7,0.12056000232696533
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,16,2,128,1,float16,float16,15,0.11989279985427856
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,16,2,128,1,float16,fp8,15,0.12046240568161011
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,16,2,128,1,float16,float16,31,0.12867039442062378
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,16,2,128,1,float16,fp8,31,0.12200959920883178
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,16,2,128,1,float16,float16,63,0.1346016049385071
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,16,2,128,1,float16,fp8,63,0.13265279531478882
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,8,128,1,float16,float16,127,0.23315041065216063
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,16,2,128,1,float16,fp8,127,0.15736000537872313
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,16,2,128,1,float16,float16,255,0.19691840410232545
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,16,2,128,1,float16,fp8,255,0.19043680429458618
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,16,4,128,1,float16,fp8,1,0.20199038982391357
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,16,4,128,1,float16,float16,1,0.20413439273834227
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,16,1,128,1,float16,fp8,7,0.07931680083274842
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,16,4,128,1,float16,float16,3,0.20279040336608886
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,16,4,128,1,float16,fp8,3,0.20424480438232423
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,16,4,128,1,float16,float16,7,0.20353760719299316
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,16,4,128,1,float16,float16,15,0.21011199951171874
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,16,1,128,1,float16,fp8,63,0.08407359719276428
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,16,4,128,1,float16,fp8,15,0.2038719892501831
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,16,4,128,1,float16,float16,31,0.2184175968170166
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,16,4,128,1,float16,fp8,31,0.21412959098815917
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,16,4,128,1,float16,float16,63,0.22636640071868896
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,16,4,128,1,float16,fp8,63,0.2244623899459839
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,16,4,128,1,float16,float16,127,0.24286239147186278
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,16,4,128,1,float16,fp8,127,0.2718544006347656
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,16,4,128,1,float16,float16,255,0.3468751907348633
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,16,4,128,1,float16,fp8,255,0.33079841136932375
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,2,128,1,float16,fp8,511,0.15055840015411376
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,16,8,128,1,float16,fp8,1,0.3647855997085571
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,16,8,128,1,float16,float16,3,0.36489760875701904
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,16,2,128,1,float16,float16,127,0.1437824010848999
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,16,8,128,1,float16,float16,7,0.36973440647125244
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,16,8,128,1,float16,fp8,7,0.3623759984970093
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,16,8,128,1,float16,float16,15,0.3772176027297974
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,16,8,128,1,float16,fp8,15,0.37290880680084226
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,16,8,128,1,float16,float16,31,0.38769280910491943
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,16,8,128,1,float16,float16,511,0.5820543766021729
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,16,8,128,1,float16,fp8,31,0.3901855945587158
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,16,8,128,1,float16,float16,63,0.4131152153015137
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,16,8,128,1,float16,fp8,63,0.40185117721557617
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,16,8,128,1,float16,float16,127,0.4439712047576904
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,1,128,1,float16,float16,1,0.00986879989504814
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,1,128,1,float16,float16,3,0.00973920002579689
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,1,128,1,float16,fp8,1,0.009921599924564362
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,16,8,128,1,float16,fp8,127,0.49520001411437986
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,1,128,1,float16,fp8,3,0.009678400307893752
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,1,128,1,float16,float16,7,0.009640000015497207
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,1,128,1,float16,float16,15,0.00987199991941452
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,1,128,1,float16,fp8,15,0.009560000151395798
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,1,128,1,float16,fp8,7,0.010020799934864044
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,1,128,1,float16,float16,31,0.00982080027461052
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,1,128,1,float16,fp8,31,0.00968480035662651
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,1,128,1,float16,float16,63,0.010019200295209885
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,1,128,1,float16,fp8,63,0.00968799963593483
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,1,128,1,float16,float16,127,0.009835200011730194
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,1,128,1,float16,fp8,127,0.010441599786281586
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,1,128,1,float16,float16,255,0.011097600311040878
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,1,128,1,float16,fp8,255,0.011470399796962738
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,1,128,1,float16,float16,511,0.013291199505329133
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,16,8,128,1,float16,fp8,255,0.6116672039031983
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,1,128,1,float16,fp8,511,0.012774400413036346
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,1,128,1,float16,float16,1023,0.014032000303268432
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,1,128,1,float16,fp8,1023,0.013867199420928955
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,1,128,1,float16,fp8,2047,0.014553600549697876
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,1,128,1,float16,float16,4095,0.017878399789333345
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,1,128,1,float16,fp8,4095,0.01727679967880249
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,1,128,1,float16,float16,2047,0.015475200116634369
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,1,128,1,float16,float16,8191,0.029667198657989502
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,1,128,1,float16,fp8,16383,0.03299840092658997
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,2,128,1,float16,float16,1,0.009811200201511383
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,2,128,1,float16,fp8,1,0.009935999661684037
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,16,8,128,1,float16,float16,1,0.3677072048187256
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,2,128,1,float16,fp8,3,0.01003199964761734
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,2,128,1,float16,float16,7,0.009787199646234512
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,2,128,1,float16,float16,15,0.009940800070762635
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,2,128,1,float16,fp8,15,0.009921599924564362
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,2,128,1,float16,float16,31,0.009918399900197983
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,16,8,128,1,float16,fp8,3,0.3619632005691528
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,2,128,1,float16,fp8,31,0.009888000041246413
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,2,128,1,float16,float16,63,0.01000479981303215
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,2,128,1,float16,fp8,63,0.009851200133562088
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,2,128,1,float16,float16,127,0.010390400141477584
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,2,128,1,float16,fp8,127,0.010921599715948105
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,2,128,1,float16,float16,255,0.011596799641847611
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,2,128,1,float16,fp8,255,0.011579199880361556
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,2,128,1,float16,fp8,511,0.013739199936389923
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,2,128,1,float16,float16,1023,0.015163199603557586
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,2,128,1,float16,fp8,1023,0.01412159949541092
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,2,128,1,float16,float16,2047,0.016473600268363954
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,2,128,1,float16,fp8,2047,0.016575999557971954
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,2,128,1,float16,float16,4095,0.028808000683784484
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,2,128,1,float16,fp8,4095,0.02106879949569702
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,2,128,1,float16,float16,8191,0.043798398971557614
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,2,128,1,float16,fp8,8191,0.032579201459884646
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,16,8,128,1,float16,float16,255,0.6551104068756104
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,2,128,1,float16,float16,16383,0.07436800003051758
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,4,128,1,float16,float16,1,0.01029760017991066
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,2,128,1,float16,fp8,16383,0.05443680286407471
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,4,128,1,float16,fp8,1,0.01010880023241043
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,4,128,1,float16,fp8,3,0.009984000027179718
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,4,128,1,float16,float16,3,0.01013600006699562
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,4,128,1,float16,float16,7,0.010147199779748917
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,4,128,1,float16,fp8,7,0.010076799988746643
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,1,128,1,float16,fp8,8191,0.02174399942159653
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,4,128,1,float16,float16,15,0.010134399682283402
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,1,128,1,float16,float16,16383,0.04437119960784912
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,4,128,1,float16,fp8,15,0.009774400293827057
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,4,128,1,float16,fp8,31,0.01003360003232956
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,4,128,1,float16,float16,63,0.010020799934864044
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,4,128,1,float16,fp8,63,0.009969600290060044
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,4,128,1,float16,float16,127,0.010260800272226334
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,4,128,1,float16,fp8,127,0.0108255997300148
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,4,128,1,float16,float16,255,0.011614400148391723
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,4,128,1,float16,fp8,255,0.011577600240707397
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,4,128,1,float16,float16,511,0.013897599279880523
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,4,128,1,float16,fp8,511,0.014899200201034546
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,4,128,1,float16,float16,1023,0.01634719967842102
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,4,128,1,float16,fp8,1023,0.01639840006828308
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,4,128,1,float16,float16,2047,0.02847520112991333
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,4,128,1,float16,fp8,2047,0.021934400498867034
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,4,128,1,float16,float16,4095,0.043721601366996765
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,4,128,1,float16,fp8,4095,0.03173600137233734
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,4,128,1,float16,float16,8191,0.07346879839897155
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,4,128,1,float16,fp8,8191,0.05092319846153259
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,4,128,1,float16,fp8,16383,0.08819519877433776
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,8,128,1,float16,float16,1,0.01048320010304451
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,8,128,1,float16,fp8,1,0.010078399628400802
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,8,128,1,float16,float16,3,0.010100799798965453
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,8,128,1,float16,fp8,3,0.01003040000796318
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,8,128,1,float16,float16,7,0.01005920022726059
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,16,4,128,1,float16,fp8,7,0.20239999294281005
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,8,128,1,float16,fp8,7,0.010044799745082855
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,8,128,1,float16,fp8,15,0.01016639992594719
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,8,128,1,float16,float16,31,0.010281600058078766
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,8,128,1,float16,fp8,31,0.010180799663066864
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,8,128,1,float16,float16,63,0.010417599976062775
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,8,128,1,float16,fp8,63,0.010259199887514114
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,8,128,1,float16,float16,127,0.010809600353240967
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,8,128,1,float16,fp8,127,0.011195199936628342
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,8,128,1,float16,float16,255,0.0118367999792099
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,8,128,1,float16,fp8,255,0.01173119992017746
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,8,128,1,float16,float16,511,0.013979199528694152
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,8,128,1,float16,fp8,511,0.015369600057601929
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,8,128,1,float16,float16,1023,0.02733440101146698
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,2,128,1,float16,float16,3,0.009984000027179718
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,8,128,1,float16,fp8,1023,0.019908800721168518
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,8,128,1,float16,float16,2047,0.042905598878860474
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,2,128,1,float16,fp8,7,0.010089600086212158
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,8,128,1,float16,float16,4095,0.07284640073776245
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,8,128,1,float16,fp8,4095,0.05001760125160217
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,8,128,1,float16,float16,8191,0.13281919956207275
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,8,128,1,float16,fp8,8191,0.0877664029598236
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,2,128,1,float16,float16,511,0.013523200154304504
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,8,128,1,float16,fp8,16383,0.16268800497055053
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,1,128,1,float16,float16,1,0.01146719977259636
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,1,128,1,float16,fp8,1,0.011257600039243698
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,4,128,1,float16,float16,16383,0.1343600034713745
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,1,128,1,float16,float16,3,0.010982400178909302
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,1,128,1,float16,float16,7,0.011521600186824799
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,1,128,1,float16,float16,15,0.01106399968266487
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,1,128,1,float16,fp8,7,0.011023999750614166
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,1,128,1,float16,float16,31,0.011222399771213531
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,1,128,1,float16,fp8,31,0.011084800213575363
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,1,128,1,float16,float16,63,0.011292800307273865
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,1,128,1,float16,fp8,63,0.011315199732780456
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,1,128,1,float16,float16,127,0.011398400366306304
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,1,128,1,float16,fp8,127,0.01228640004992485
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,1,128,1,float16,float16,255,0.013035200536251068
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,1,128,1,float16,fp8,255,0.01316480040550232
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,1,128,1,float16,float16,511,0.014900800585746766
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,1,128,1,float16,fp8,511,0.016513599455356597
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,1,128,1,float16,float16,1023,0.017582400143146514
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,1,128,1,float16,fp8,1023,0.01794240027666092
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,1,128,1,float16,float16,2047,0.02982560098171234
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,1,128,1,float16,fp8,2047,0.02344159930944443
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,1,128,1,float16,float16,4095,0.045465600490570066
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,1,128,1,float16,fp8,4095,0.033697599172592164
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,1,128,1,float16,float16,8191,0.07565439939498901
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,1,128,1,float16,fp8,8191,0.0525488018989563
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,1,128,1,float16,float16,16383,0.13598400354385376
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,1,128,1,float16,fp8,16383,0.09069120287895202
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,4,128,1,float16,float16,31,0.010078399628400802
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,1,128,1,float16,fp8,32767,0.16651519536972045
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,2,128,1,float16,float16,1,0.011603199690580369
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,2,128,1,float16,float16,3,0.011697600036859513
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,2,128,1,float16,fp8,1,0.01148959994316101
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,2,128,1,float16,fp8,3,0.01156959980726242
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,2,128,1,float16,float16,7,0.01159999966621399
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,2,128,1,float16,fp8,7,0.01162559986114502
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,2,128,1,float16,float16,15,0.011582399904727935
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,2,128,1,float16,fp8,15,0.011588799953460693
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,2,128,1,float16,float16,31,0.011622399836778641
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,2,128,1,float16,fp8,31,0.011604800075292587
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,2,128,1,float16,float16,63,0.011617600172758102
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,2,128,1,float16,fp8,63,0.011608000099658965
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,2,128,1,float16,float16,127,0.012033600360155106
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,2,128,1,float16,fp8,127,0.012537600100040435
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,2,128,1,float16,float16,255,0.01324319988489151
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,2,128,1,float16,fp8,255,0.01332319974899292
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,2,128,1,float16,float16,511,0.01579360067844391
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,2,128,1,float16,fp8,511,0.01706240028142929
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,8,128,1,float16,float16,16383,0.2487760066986084
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,2,128,1,float16,float16,1023,0.028708800673484802
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,1,128,1,float16,fp8,3,0.01157120019197464
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,2,128,1,float16,float16,2047,0.04446400105953217
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,2,128,1,float16,fp8,1023,0.02117920070886612
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,2,128,1,float16,fp8,2047,0.032660800218582156
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,1,128,1,float16,fp8,15,0.011339200288057327
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,2,128,1,float16,fp8,4095,0.05116479992866516
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,2,128,1,float16,float16,8191,0.1330288052558899
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,2,128,1,float16,fp8,8191,0.09015359878540039
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,2,128,1,float16,float16,16383,0.2540047883987427
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,8,128,1,float16,float16,15,0.010260800272226334
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,4,128,1,float16,float16,1,0.014256000518798828
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,4,128,1,float16,fp8,1,0.01420000046491623
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,4,128,1,float16,float16,3,0.014271999895572662
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,2,128,1,float16,float16,32767,0.4931183815002441
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,2,128,1,float16,fp8,32767,0.3166815996170044
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,4,128,1,float16,float16,7,0.014254400134086609
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,4,128,1,float16,fp8,7,0.014076800644397735
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,4,128,1,float16,float16,15,0.014110399782657624
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,4,128,1,float16,float16,31,0.014529600739479065
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,4,128,1,float16,fp8,31,0.014120000600814819
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,4,128,1,float16,float16,63,0.014347200095653535
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,1,128,1,float16,float16,32767,0.25242559909820556
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,4,128,1,float16,fp8,63,0.014156800508499146
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,16,8,128,1,float16,fp8,2047,0.030551999807357788
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,4,128,1,float16,fp8,127,0.016022400557994844
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,4,128,1,float16,float16,127,0.01462399959564209
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,4,128,1,float16,fp8,255,0.017664000391960144
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,4,128,1,float16,float16,511,0.029532799124717714
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,4,128,1,float16,fp8,511,0.024247999489307403
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,4,128,1,float16,float16,1023,0.0460671991109848
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,4,128,1,float16,fp8,1023,0.034745600819587705
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,4,128,1,float16,float16,2047,0.07595040202140808
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,4,128,1,float16,float16,4095,0.13468799591064454
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,4,128,1,float16,fp8,4095,0.09331520199775696
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,2,128,1,float16,float16,4095,0.07426880002021789
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,4,128,1,float16,float16,8191,0.251531195640564
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,4,128,1,float16,fp8,8191,0.16588319540023805
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,2,128,1,float16,fp8,16383,0.16569279432296752
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,4,128,1,float16,fp8,3,0.014076800644397735
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,4,128,1,float16,fp8,16383,0.3159327983856201
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,4,128,1,float16,float16,16383,0.48874239921569823
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,1,128,1,float16,float16,1,0.00942239984869957
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,4,128,1,float16,fp8,15,0.013956800103187561
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,1,128,1,float16,float16,3,0.009619200229644775
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,1,128,1,float16,fp8,1,0.009510400146245957
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,1,128,1,float16,float16,7,0.009564799815416336
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,1,128,1,float16,fp8,3,0.009384000301361084
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,1,128,1,float16,float16,15,0.009719999879598618
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,1,128,1,float16,fp8,15,0.009399999678134919
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,1,128,1,float16,float16,31,0.009712000191211701
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,1,128,1,float16,fp8,31,0.009576000273227692
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,1,128,1,float16,float16,63,0.009670399874448777
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,4,128,1,float16,float16,255,0.017220799624919892
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,1,128,1,float16,float16,127,0.009486400336027146
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,1,128,1,float16,fp8,127,0.010201600193977357
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,4,128,1,float16,fp8,32767,0.6166863918304444
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,1,128,1,float16,fp8,255,0.01101119965314865
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,4,128,1,float16,float16,32767,0.9641936302185059
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,1,128,1,float16,fp8,511,0.011974400281906128
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,1,128,1,float16,float16,511,0.012848000228404998
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,1,128,1,float16,fp8,1023,0.012673600018024445
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,8,4,128,1,float16,fp8,2047,0.05486559867858887
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,1,128,1,float16,float16,2047,0.013387200236320496
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,1,128,1,float16,float16,4095,0.014468799531459808
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,1,128,1,float16,fp8,4095,0.014368000626564025
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,1,128,1,float16,float16,8191,0.015689599514007568
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,1,128,1,float16,fp8,8191,0.015872000157833098
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,1,128,1,float16,float16,16383,0.017742399871349335
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,1,128,1,float16,fp8,16383,0.018585599958896637
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,1,128,1,float16,float16,32767,0.0208079993724823
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,1,128,1,float16,fp8,32767,0.020308800041675568
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,2,128,1,float16,fp8,1,0.00963200032711029
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,2,128,1,float16,float16,3,0.00952640026807785
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,2,128,1,float16,fp8,3,0.009548799693584442
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,2,128,1,float16,float16,7,0.009470400214195252
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,2,128,1,float16,fp8,7,0.009539200365543366
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,2,128,1,float16,float16,15,0.009468799829483033
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,2,128,1,float16,fp8,15,0.009539200365543366
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,2,128,1,float16,float16,31,0.00963359996676445
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,2,128,1,float16,fp8,31,0.009491200000047684
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,2,128,1,float16,float16,63,0.009697599709033966
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,2,128,1,float16,fp8,63,0.009617599844932555
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,2,128,1,float16,float16,127,0.00968639999628067
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,2,128,1,float16,fp8,127,0.01034879982471466
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,2,128,1,float16,float16,255,0.010996799916028976
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,2,128,1,float16,fp8,255,0.011230400204658509
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,2,128,1,float16,float16,511,0.013100799918174744
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,2,128,1,float16,fp8,511,0.012120000272989272
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,2,128,1,float16,float16,1023,0.013223999738693237
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,2,128,1,float16,fp8,1023,0.012923200428485871
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,2,128,1,float16,float16,2047,0.013710400462150574
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,2,128,1,float16,fp8,2047,0.013212800025939941
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,2,128,1,float16,float16,4095,0.014203199744224548
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,2,128,1,float16,fp8,4095,0.014180800318717957
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,2,128,1,float16,float16,8191,0.015318399667739869
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,2,128,1,float16,fp8,8191,0.015624000132083893
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,2,128,1,float16,float16,16383,0.017531199753284453
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,2,128,1,float16,fp8,16383,0.01751679927110672
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,2,128,1,float16,float16,32767,0.020476800203323365
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,2,128,1,float16,fp8,32767,0.020275199413299562
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,1,128,1,float16,fp8,7,0.009428799897432328
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,4,128,1,float16,float16,1,0.009657599776983262
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,4,128,1,float16,fp8,1,0.009462399780750275
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,4,128,1,float16,float16,3,0.009419199824333192
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,4,128,1,float16,fp8,3,0.009452799707651139
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,4,128,1,float16,float16,7,0.009328000247478485
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,4,128,1,float16,fp8,7,0.009455999732017517
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,4,128,1,float16,float16,15,0.009391999989748
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,1,128,1,float16,float16,255,0.010673599690198899
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,4,128,1,float16,fp8,15,0.009303999692201614
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,4,128,1,float16,float16,31,0.009353599697351455
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,4,128,1,float16,fp8,31,0.009268800169229508
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,4,128,1,float16,float16,63,0.0094991996884346
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,4,128,1,float16,fp8,63,0.009251199662685394
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,4,128,1,float16,float16,127,0.009443199634552002
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,4,128,1,float16,fp8,127,0.01011039987206459
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,4,128,1,float16,float16,255,0.010742399841547012
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,4,128,1,float16,fp8,255,0.010967999696731567
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,4,128,1,float16,float16,511,0.012880000472068786
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,4,128,1,float16,float16,1023,0.013291199505329133
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,1,128,1,float16,fp8,2047,0.013240000605583191
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,4,128,1,float16,float16,2047,0.013675199449062347
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,4,128,1,float16,float16,4095,0.014480000734329224
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,4,128,1,float16,fp8,4095,0.014580799639225006
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,4,128,1,float16,float16,8191,0.01611199975013733
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,4,128,1,float16,fp8,8191,0.01578879952430725
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,2,128,1,float16,float16,1,0.009505599737167358
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,4,128,1,float16,float16,16383,0.018900799751281738
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,4,128,1,float16,fp8,16383,0.018564799427986146
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,4,128,1,float16,float16,32767,0.0303600013256073
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,4,128,1,float16,fp8,32767,0.023705600202083586
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,1,128,1,float16,float16,1,0.009363199770450591
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,1,128,1,float16,fp8,1,0.009436800330877303
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,1,128,1,float16,float16,3,0.009359999746084213
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,1,128,1,float16,fp8,3,0.009284800291061402
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,1,128,1,float16,float16,7,0.00942559987306595
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,1,128,1,float16,fp8,7,0.009310399740934372
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,1,128,1,float16,float16,15,0.009324800223112106
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,1,128,1,float16,fp8,15,0.009347199648618697
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,1,128,1,float16,float16,31,0.009470400214195252
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,1,128,1,float16,fp8,31,0.009307199716567993
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,1,128,1,float16,float16,63,0.009614399820566177
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,1,128,1,float16,fp8,63,0.00931520015001297
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,1,128,1,float16,float16,127,0.009668800234794616
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,1,128,1,float16,fp8,127,0.010097599774599075
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,1,128,1,float16,float16,255,0.010889600217342376
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,1,128,1,float16,fp8,255,0.01101600006222725
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,1,128,1,float16,float16,511,0.012996800243854523
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,1,128,1,float16,fp8,511,0.012088000029325485
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,1,128,1,float16,float16,1023,0.013123199343681335
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,1,128,1,float16,fp8,1023,0.012566399574279786
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,1,128,1,float16,float16,2047,0.013422399759292603
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,1,128,1,float16,fp8,2047,0.013148799538612366
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,1,128,1,float16,float16,4095,0.014159999787807465
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,1,128,1,float16,fp8,4095,0.014248000085353851
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,1,128,1,float16,float16,8191,0.015863999724388123
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,1,128,1,float16,fp8,8191,0.01578560024499893
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,1,128,1,float16,float16,16383,0.017636799812316896
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,1,128,1,float16,fp8,16383,0.01788959950208664
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,1,128,1,float16,float16,32767,0.020473599433898926
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,1,128,1,float16,fp8,32767,0.020603199303150178
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,2,128,1,float16,float16,1,0.009344000369310379
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,2,128,1,float16,fp8,1,0.009497600048780442
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,2,128,1,float16,float16,3,0.00931679978966713
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,1,128,1,float16,fp8,63,0.00955360010266304
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,2,128,1,float16,fp8,3,0.009487999975681305
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,2,128,1,float16,float16,7,0.009384000301361084
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,2,128,1,float16,fp8,7,0.009384000301361084
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,2,128,1,float16,fp8,15,0.009089600294828415
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,2,128,1,float16,float16,31,0.009507200121879578
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,2,128,1,float16,float16,63,0.009537599980831146
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,2,128,1,float16,fp8,63,0.009265600144863129
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,4,128,1,float16,fp8,511,0.012515200674533844
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,2,128,1,float16,float16,127,0.009393599629402161
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,4,128,1,float16,fp8,1023,0.012824000418186187
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,1,128,1,float16,float16,1023,0.013017599284648896
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,2,128,1,float16,fp8,255,0.011113599687814713
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,8,4,128,1,float16,fp8,2047,0.01311040073633194
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,2,128,1,float16,float16,511,0.012857599556446076
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,2,128,1,float16,fp8,511,0.012430399656295776
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,2,128,1,float16,float16,1023,0.01308320015668869
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,2,128,1,float16,fp8,2047,0.013035200536251068
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,2,128,1,float16,float16,4095,0.014747199416160584
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,2,128,1,float16,fp8,4095,0.014324800670146942
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,2,128,1,float16,float16,8191,0.016126400232315062
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,2,128,1,float16,fp8,8191,0.01595200002193451
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,2,128,1,float16,float16,16383,0.018782399594783783
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,2,128,1,float16,fp8,16383,0.018129600584506987
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,2,128,1,float16,float16,32767,0.029819199442863466
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,2,128,1,float16,fp8,32767,0.02346239984035492
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,4,128,1,float16,float16,1,0.009393599629402161
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,4,128,1,float16,fp8,1,0.009452799707651139
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,4,128,1,float16,float16,3,0.009612800180912017
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,4,128,1,float16,fp8,3,0.009443199634552002
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,4,128,1,float16,float16,7,0.009550400078296661
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,4,128,1,float16,fp8,7,0.009391999989748
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,4,128,1,float16,float16,15,0.009567999839782714
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,4,128,1,float16,fp8,15,0.009268800169229508
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,2,128,1,float16,float16,15,0.009587199985980987
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,2,128,1,float16,fp8,31,0.009220799803733826
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,4,128,1,float16,float16,63,0.009675200283527374
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,4,128,1,float16,fp8,63,0.009657599776983262
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,4,128,1,float16,float16,127,0.009785600006580353
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,4,128,1,float16,fp8,127,0.010384000092744827
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,4,128,1,float16,float16,255,0.010980799794197083
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,4,128,1,float16,fp8,255,0.01109279990196228
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,2,128,1,float16,fp8,1023,0.012788799405097962
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,4,128,1,float16,float16,511,0.012695999443531036
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,4,128,1,float16,fp8,511,0.01231039986014366
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,4,128,1,float16,fp8,1023,0.012899200618267059
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,4,128,1,float16,float16,2047,0.014256000518798828
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,4,128,1,float16,fp8,2047,0.01369439959526062
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,4,128,1,float16,float16,4095,0.015505599975585937
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,4,128,1,float16,fp8,4095,0.014694400131702423
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,4,128,1,float16,float16,8191,0.017720000445842744
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,4,128,1,float16,fp8,8191,0.017343999445438386
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,4,128,1,float16,float16,16383,0.029185599088668822
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,4,128,1,float16,fp8,16383,0.022702400386333466
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,4,128,1,float16,float16,32767,0.04472320079803467
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,4,128,1,float16,fp8,32767,0.033958399295806886
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,1,128,1,float16,float16,1,0.010577599704265594
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,1,128,1,float16,fp8,1,0.010286399722099304
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,1,128,1,float16,float16,3,0.01016639992594719
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,1,128,1,float16,fp8,3,0.010121600329875946
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,1,128,1,float16,float16,7,0.010355199873447418
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,1,128,1,float16,fp8,7,0.010328000038862228
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,1,128,1,float16,float16,15,0.010345599800348281
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,1,128,1,float16,fp8,15,0.010131199657917023
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,1,128,1,float16,float16,31,0.010232000052928925
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,4,128,1,float16,float16,31,0.0095551997423172
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,1,128,1,float16,fp8,31,0.010195200145244599
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,4,128,1,float16,fp8,31,0.009388799965381622
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,1,128,1,float16,float16,63,0.010441599786281586
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,1,128,1,float16,fp8,127,0.0110895998775959
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,1,128,1,float16,float16,127,0.01067200005054474
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,1,128,1,float16,float16,255,0.01196800023317337
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,1,128,1,float16,fp8,255,0.011822400242090225
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,1,128,1,float16,float16,511,0.014192000031471252
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,1,128,1,float16,fp8,511,0.013920000195503235
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,2,128,1,float16,fp8,127,0.01021919995546341
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,1,128,1,float16,float16,1023,0.015755200386047365
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,1,128,1,float16,fp8,1023,0.014441600441932679
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,1,128,1,float16,fp8,2047,0.01730719953775406
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,1,128,1,float16,float16,4095,0.029142400622367857
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,1,128,1,float16,fp8,4095,0.021473599970340727
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,2,128,1,float16,float16,2047,0.013726399838924408
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,1,128,1,float16,float16,8191,0.04427039921283722
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,1,128,1,float16,fp8,8191,0.03319199979305267
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,4,128,1,float16,float16,1023,0.013235199451446533
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,1,128,1,float16,float16,16383,0.07480159997940064
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,1,128,1,float16,fp8,16383,0.05224000215530396
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,2,128,1,float16,float16,1,0.010820800065994262
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,2,128,1,float16,fp8,1,0.010625600069761276
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,2,128,1,float16,float16,3,0.010523200035095215
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,2,128,1,float16,fp8,3,0.010606399923563003
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,2,128,1,float16,float16,7,0.010564800351858139
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,2,128,1,float16,fp8,7,0.010440000146627427
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,2,128,1,float16,float16,15,0.010382399708032609
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,2,128,1,float16,fp8,15,0.01043360009789467
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,2,128,1,float16,float16,31,0.010422399640083313
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,2,128,1,float16,fp8,31,0.01037440001964569
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,2,128,1,float16,float16,63,0.010507199913263321
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,2,128,1,float16,fp8,63,0.010360000282526016
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,2,128,1,float16,float16,127,0.010601600259542465
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,2,128,1,float16,fp8,127,0.0112527996301651
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,2,128,1,float16,float16,255,0.012030400335788727
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,2,128,1,float16,fp8,255,0.012030400335788727
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,2,128,1,float16,float16,511,0.014091199636459351
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,2,128,1,float16,fp8,511,0.015225599706172942
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,2,128,1,float16,float16,1023,0.016952000558376312
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,2,128,1,float16,fp8,1023,0.016883200407028197
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,2,128,1,float16,float16,2047,0.029016000032424927
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,2,128,1,float16,float16,4095,0.044356799125671385
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,2,128,1,float16,fp8,4095,0.03257279992103577
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,2,128,1,float16,float16,8191,0.074372798204422
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,2,128,1,float16,fp8,8191,0.051425600051879884
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,2,128,1,float16,float16,16383,0.13324960470199584
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,2,128,1,float16,fp8,16383,0.08980640172958373
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,2,128,1,float16,fp8,32767,0.165555202960968
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,2,128,1,float16,float16,32767,0.2541232109069824
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,1,128,1,float16,fp8,63,0.010401599854230881
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,4,128,1,float16,float16,1,0.01074720025062561
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,4,128,1,float16,fp8,1,0.010958400368690491
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,4,128,1,float16,fp8,3,0.010521599650382995
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,4,128,1,float16,float16,7,0.01067039966583252
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,4,128,1,float16,fp8,7,0.010465600341558457
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,4,128,1,float16,float16,15,0.010814400017261505
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,4,128,1,float16,float16,31,0.01053600013256073
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,4,128,1,float16,fp8,31,0.010583999752998351
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,4,128,1,float16,float16,63,0.010707200318574906
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,4,128,1,float16,fp8,63,0.010606399923563003
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,4,128,1,float16,float16,127,0.011007999628782272
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,1,128,1,float16,float16,32767,0.1354912042617798
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,4,128,1,float16,fp8,127,0.011689600348472596
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,4,128,1,float16,fp8,255,0.012332800030708312
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,4,128,1,float16,float16,255,0.01255359947681427
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,4,128,1,float16,float16,511,0.014548799395561219
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,4,128,1,float16,fp8,511,0.015801599621772765
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,4,128,1,float16,float16,1023,0.027625599503517152
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,4,128,1,float16,fp8,1023,0.020283199846744537
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,4,128,1,float16,float16,2047,0.04309920072555542
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,4,128,1,float16,fp8,2047,0.03133119940757752
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,2,128,1,float16,fp8,2047,0.02242559939622879
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,4,128,1,float16,float16,4095,0.07285280227661133
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,4,128,1,float16,fp8,4095,0.05037279725074768
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,4,128,1,float16,fp8,8191,0.08845760226249695
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,4,128,1,float16,float16,8191,0.13182719945907592
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,4,128,1,float16,fp8,16383,0.16367839574813842
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,4,128,1,float16,float16,16383,0.2503056049346924
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,4,128,1,float16,float16,3,0.010811199992895126
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,8,2,128,1,float16,float16,255,0.010676799714565277
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,1,128,1,float16,fp8,1,0.00928959995508194
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,4,128,1,float16,fp8,15,0.01074879989027977
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,1,128,1,float16,float16,3,0.009427200257778167
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,1,128,1,float16,float16,7,0.00931359976530075
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,4,128,1,float16,float16,32767,0.4879727840423584
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,1,128,1,float16,fp8,7,0.00939679965376854
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,1,128,1,float16,fp8,15,0.009446399658918381
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,1,128,1,float16,float16,31,0.009587199985980987
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,1,128,1,float16,fp8,31,0.00926399976015091
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,1,128,1,float16,fp8,32767,0.08995199799537659
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,1,128,1,float16,float16,63,0.009478399902582169
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,1,128,1,float16,fp8,63,0.00926240012049675
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,1,128,1,float16,fp8,127,0.010127999633550645
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,1,128,1,float16,float16,255,0.01080320030450821
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,1,128,1,float16,fp8,255,0.011035200208425522
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,1,128,1,float16,float16,511,0.01297599971294403
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,1,128,1,float16,fp8,511,0.012299200147390365
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,1,128,1,float16,float16,1023,0.013288000226020813
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,1,128,1,float16,fp8,1023,0.01290079951286316
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,1,128,1,float16,float16,2047,0.013691200315952301
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,1,128,1,float16,fp8,2047,0.013291199505329133
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,1,128,1,float16,float16,4095,0.014761599898338317
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,1,128,1,float16,fp8,4095,0.014614400267601014
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,1,128,1,float16,float16,8191,0.01648319959640503
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,1,128,1,float16,fp8,8191,0.01569119989871979
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,1,128,1,float16,float16,16383,0.01892160028219223
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,1,128,1,float16,fp8,16383,0.01823520064353943
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,1,128,1,float16,float16,32767,0.030847999453544616
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,1,128,1,float16,fp8,32767,0.023647999763488768
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,2,128,1,float16,float16,1,0.009769599884748459
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,2,128,1,float16,fp8,1,0.009636799991130828
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,2,128,1,float16,float16,3,0.009708800166845322
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,1,128,1,float16,float16,1,0.009539200365543366
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,2,128,1,float16,fp8,3,0.009494400024414063
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,1,128,1,float16,float16,2047,0.017207999527454377
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,2,128,1,float16,fp8,7,0.009518399834632874
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,2,128,1,float16,float16,7,0.009777600318193436
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,1,128,1,float16,fp8,3,0.009364800155162811
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,2,128,1,float16,float16,15,0.009734400361776353
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,2,128,1,float16,fp8,15,0.009513600170612336
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,2,128,1,float16,float16,63,0.009854400157928466
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,2,128,1,float16,fp8,31,0.009510400146245957
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,2,128,1,float16,fp8,63,0.009467200189828873
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,1,128,1,float16,float16,15,0.009492799639701843
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,2,128,1,float16,fp8,127,0.010040000081062317
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,2,128,1,float16,float16,255,0.010756800323724747
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,2,128,1,float16,fp8,255,0.010927999764680863
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,2,128,1,float16,fp8,511,0.01225920021533966
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,2,128,1,float16,float16,511,0.013166399300098419
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,2,128,1,float16,fp8,1023,0.012895999848842621
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,2,128,1,float16,fp8,2047,0.014076800644397735
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,1,128,1,float16,float16,127,0.009593600034713745
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,2,128,1,float16,float16,4095,0.015607999265193939
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,2,128,1,float16,fp8,4095,0.014705599844455719
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,2,128,1,float16,fp8,8191,0.017531199753284453
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,2,128,1,float16,float16,16383,0.029423999786376952
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,2,128,1,float16,fp8,16383,0.022915199398994446
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,2,128,1,float16,float16,32767,0.04563199877738953
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,2,128,1,float16,fp8,32767,0.03388319909572601
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,4,128,1,float16,float16,1,0.00968639999628067
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,4,128,1,float16,fp8,1,0.009375999867916106
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,4,128,1,float16,float16,3,0.009747199714183807
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,4,128,1,float16,fp8,3,0.009369599819183349
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,4,128,1,float16,float16,7,0.009759999811649323
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,4,128,1,float16,fp8,7,0.009401600062847137
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,4,128,1,float16,float16,15,0.00979039967060089
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,4,128,1,float16,fp8,15,0.009388799965381622
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,4,128,1,float16,float16,31,0.009457600116729737
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,4,128,1,float16,fp8,31,0.009388799965381622
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,4,128,1,float16,float16,63,0.009598399698734283
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,4,128,1,float16,fp8,63,0.009619200229644775
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,4,128,1,float16,float16,127,0.00968960002064705
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,4,128,1,float16,fp8,127,0.010209599882364273
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,4,128,1,float16,float16,255,0.011343999952077865
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,4,128,1,float16,fp8,255,0.01120000034570694
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,4,128,1,float16,float16,511,0.01311199963092804
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,4,128,1,float16,fp8,511,0.012219200283288956
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,4,128,1,float16,float16,1023,0.013903999328613281
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,4,128,1,float16,fp8,1023,0.0133775994181633
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,4,128,1,float16,float16,2047,0.01530880033969879
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,4,128,1,float16,fp8,2047,0.014475199580192565
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,4,128,1,float16,float16,4095,0.017479999363422392
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,4,128,1,float16,fp8,4095,0.017020800709724428
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,4,128,1,float16,float16,8191,0.028201600909233092
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,4,128,1,float16,float16,16383,0.04349119961261749
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,4,128,1,float16,fp8,16383,0.03258880078792572
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,4,128,1,float16,float16,32767,0.07345759868621826
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,8,4,128,1,float16,fp8,32767,0.3144448041915894
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,2,128,1,float16,float16,31,0.009649600088596343
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,1,128,1,float16,float16,1,0.012939199805259705
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,4,128,1,float16,fp8,32767,0.054983997344970705
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,2,128,1,float16,float16,127,0.009756799787282944
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,1,128,1,float16,fp8,1,0.01290079951286316
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,1,128,1,float16,fp8,3,0.013030399382114411
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,2,128,1,float16,float16,1023,0.013305599987506866
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,1,128,1,float16,fp8,7,0.012995199859142303
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,1,128,1,float16,float16,15,0.012484800070524216
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,1,128,1,float16,float16,31,0.012727999687194824
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,2,128,1,float16,float16,8191,0.017667199671268462
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,1,128,1,float16,fp8,31,0.012608000636100769
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,1,128,1,float16,float16,63,0.013195200264453888
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,1,128,1,float16,fp8,63,0.012729600071907043
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,1,128,1,float16,float16,127,0.01356319934129715
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,1,128,1,float16,fp8,127,0.01361439973115921
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,1,128,1,float16,float16,255,0.015435199439525604
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,1,128,1,float16,fp8,255,0.01446560025215149
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,1,128,1,float16,float16,511,0.017427200078964235
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,1,128,1,float16,fp8,511,0.01815840005874634
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,1,128,1,float16,float16,1023,0.03031519949436188
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,1,128,1,float16,fp8,1023,0.022782400250434875
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,1,128,1,float16,float16,2047,0.04513440132141113
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,1,128,1,float16,fp8,2047,0.033985599875450134
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,1,128,1,float16,float16,4095,0.07554559707641602
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,1,128,1,float16,fp8,4095,0.053406399488449094
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,1,128,1,float16,float16,8191,0.13494399785995484
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,1,128,1,float16,fp8,8191,0.0919264018535614
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,4,128,1,float16,fp8,8191,0.02144639939069748
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,2,128,1,float16,float16,1,0.015627199411392213
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,1,128,1,float16,fp8,16383,0.16868319511413574
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,2,128,1,float16,fp8,1,0.015382400155067444
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,2,128,1,float16,float16,3,0.015174399316310882
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,2,128,1,float16,float16,7,0.015542399883270264
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,2,128,1,float16,fp8,7,0.015408000349998474
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,2,128,1,float16,float16,15,0.015715199708938598
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,2,128,1,float16,fp8,15,0.015800000727176668
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,2,128,1,float16,float16,31,0.015643200278282164
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,2,128,1,float16,fp8,31,0.015449599921703338
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,2,128,1,float16,float16,63,0.01555040031671524
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,2,128,1,float16,fp8,63,0.01555040031671524
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,8,2,128,1,float16,float16,2047,0.014123199880123139
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,1,128,1,float16,fp8,15,0.012884800136089326
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,2,128,1,float16,fp8,127,0.01682559996843338
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,2,128,1,float16,fp8,255,0.018649600446224213
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,2,128,1,float16,float16,511,0.031249600648880004
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,2,128,1,float16,fp8,511,0.02552480101585388
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,2,128,1,float16,float16,1023,0.04757600128650665
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,2,128,1,float16,fp8,1023,0.036287999153137206
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,2,128,1,float16,float16,2047,0.07742879986763
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,2,128,1,float16,fp8,2047,0.055953598022460936
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,2,128,1,float16,float16,4095,0.13551520109176635
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,2,128,1,float16,fp8,4095,0.09362720251083374
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,1,128,1,float16,float16,16383,0.25540320873260497
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,2,128,1,float16,float16,8191,0.25271360874176024
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,2,128,1,float16,fp8,8191,0.17020319700241088
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,2,128,1,float16,fp8,3,0.015292799472808838
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,4,128,1,float16,float16,1,0.02131039947271347
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,4,128,1,float16,fp8,1,0.021040000021457672
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,4,128,1,float16,float16,3,0.021342399716377258
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,4,128,1,float16,float16,7,0.021240000426769257
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,2,128,1,float16,fp8,16383,0.3194240093231201
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,4,128,1,float16,fp8,7,0.021104000508785248
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,2,128,1,float16,float16,16383,0.48951520919799807
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,4,128,1,float16,fp8,15,0.021132799983024596
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,4,128,1,float16,float16,31,0.02117920070886612
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,4,128,1,float16,fp8,31,0.021139200031757354
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,1,128,1,float16,float16,7,0.012929600477218629
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,2,128,1,float16,float16,127,0.01563359946012497
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,4,128,1,float16,float16,63,0.021223999559879303
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,4,128,1,float16,float16,127,0.02158239930868149
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,4,128,1,float16,fp8,127,0.024454399943351746
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,4,128,1,float16,float16,255,0.03411200046539307
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,2,128,1,float16,float16,255,0.018544000387191773
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,4,128,1,float16,fp8,255,0.028364801406860353
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,4,128,1,float16,fp8,511,0.04216319918632507
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,4,128,1,float16,float16,1023,0.08087999820709228
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,4,128,1,float16,fp8,1023,0.05989919900894165
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,4,128,1,float16,float16,2047,0.1392367959022522
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,4,128,1,float16,fp8,2047,0.0982591986656189
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,4,128,1,float16,float16,4095,0.25510239601135254
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,4,128,1,float16,fp8,4095,0.17208319902420044
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,4,128,1,float16,float16,8191,0.48848638534545896
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,4,128,1,float16,fp8,8191,0.3257152080535889
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,1,128,1,float16,float16,1,0.017715199291706084
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,1,128,1,float16,fp8,1,0.018116800487041472
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,1,128,1,float16,float16,3,0.018296000361442567
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,1,128,1,float16,fp8,3,0.01807200014591217
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,4,128,1,float16,float16,16383,0.9590847969055176
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,1,128,1,float16,float16,7,0.018142400681972502
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,1,128,1,float16,float16,15,0.018297599256038667
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,1,128,1,float16,fp8,7,0.017654399573802947
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,4,128,1,float16,fp8,16383,0.6343647956848144
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,1,128,1,float16,fp8,15,0.01801439970731735
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,4,128,1,float16,fp8,3,0.02099200040102005
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,1,128,1,float16,float16,31,0.018139199912548067
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,1,128,1,float16,fp8,31,0.01803999990224838
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,1,128,1,float16,float16,127,0.01781439930200577
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,1,128,1,float16,fp8,127,0.019601599872112276
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,4,128,1,float16,fp8,63,0.021158400177955627
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,1,128,1,float16,fp8,255,0.02133760005235672
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,1,128,1,float16,float16,511,0.033843201398849485
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,4,128,1,float16,float16,511,0.04961279928684235
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,1,128,1,float16,fp8,511,0.02834399938583374
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,1,128,1,float16,float16,1023,0.050190401077270505
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,1,128,1,float16,fp8,1023,0.03924799859523773
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,1,128,1,float16,float16,2047,0.08037279844284058
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,1,128,1,float16,fp8,2047,0.05923839807510376
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,1,128,1,float16,float16,4095,0.13863999843597413
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,1,128,1,float16,fp8,4095,0.09686239957809448
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,2,128,1,float16,float16,1,0.023792000114917757
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,2,128,1,float16,fp8,1,0.02338079959154129
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,2,128,1,float16,float16,3,0.023600000143051147
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,2,128,1,float16,float16,7,0.02359360009431839
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,2,128,1,float16,fp8,7,0.023479999601840974
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,2,128,1,float16,float16,15,0.02389120012521744
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,2,128,1,float16,fp8,15,0.02356799989938736
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,2,128,1,float16,float16,31,0.023894399404525757
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,2,128,1,float16,fp8,31,0.023716799914836884
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,2,128,1,float16,float16,63,0.0239424005150795
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,2,128,1,float16,fp8,63,0.0234592005610466
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,1,128,1,float16,float16,63,0.018131199479103088
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,4,128,1,float16,float16,15,0.02112320065498352
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,2,128,1,float16,float16,127,0.023956799507141115
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,2,128,1,float16,fp8,127,0.02701759934425354
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,8,1,128,1,float16,float16,3,0.013153600692749023
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,2,128,1,float16,float16,255,0.03654879927635193
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,2,128,1,float16,float16,511,0.05213919878005981
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,2,128,1,float16,fp8,511,0.04527359902858734
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,2,128,1,float16,fp8,1023,0.06328480243682862
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,2,128,1,float16,float16,1023,0.08407040238380432
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,2,128,1,float16,float16,2047,0.14232159852981568
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,2,128,1,float16,fp8,2047,0.10209920406341552
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,2,128,1,float16,float16,4095,0.25809760093688966
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,2,128,1,float16,fp8,4095,0.17774399518966674
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,4,128,1,float16,fp8,1,0.034622400999069214
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,4,128,1,float16,float16,3,0.0348688006401062
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,4,128,1,float16,fp8,3,0.034729599952697754
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,4,128,1,float16,float16,7,0.035128000378608706
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,4,128,1,float16,float16,15,0.03516319990158081
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,4,128,1,float16,fp8,15,0.034651198983192445
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,4,128,1,float16,float16,31,0.03505919873714447
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,2,128,1,float16,fp8,3,0.023555199801921844
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,4,128,1,float16,fp8,31,0.03482080101966858
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,4,128,1,float16,float16,63,0.035545599460601804
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,4,128,1,float16,fp8,63,0.03467200100421906
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,4,128,1,float16,float16,127,0.042238399386405945
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,4,128,1,float16,fp8,127,0.04158720076084137
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,4,128,1,float16,float16,255,0.0570527970790863
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,4,128,1,float16,fp8,255,0.051288002729415895
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,4,128,1,float16,float16,511,0.08659520149230956
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,4,128,1,float16,fp8,511,0.07513920068740845
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,4,128,1,float16,float16,1023,0.14532639980316162
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,4,128,1,float16,fp8,1023,0.10963679552078247
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,1,128,1,float16,fp8,63,0.017476800084114074
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,4,128,1,float16,fp8,2047,0.18692959547042848
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,2,128,1,float16,fp8,255,0.03049759864807129
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,1,128,1,float16,float16,255,0.021611200273036958
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,1,128,1,float16,float16,1,0.0290367990732193
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,4,128,1,float16,float16,4095,0.4910031795501709
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,1,128,1,float16,fp8,1,0.029014399647712706
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,1,128,1,float16,fp8,3,0.028977599740028382
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,1,128,1,float16,float16,3,0.029198399186134337
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,1,128,1,float16,fp8,7,0.028932800889015196
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,1,128,1,float16,float16,7,0.02922559976577759
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,1,128,1,float16,float16,31,0.029212799668312073
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,1,128,1,float16,fp8,31,0.029110398888587952
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,4,128,1,float16,float16,1,0.03463200032711029
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,1,128,1,float16,fp8,63,0.029252800345420837
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,1,128,1,float16,fp8,127,0.03271040022373199
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,1,128,1,float16,float16,255,0.04221599996089935
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,1,128,1,float16,fp8,255,0.03628639876842499
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,4,128,1,float16,fp8,7,0.03462719917297363
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,1,128,1,float16,float16,511,0.05757279992103577
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,1,128,1,float16,fp8,511,0.05098400115966797
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,1,128,1,float16,float16,1023,0.09016000032424927
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,1,128,1,float16,fp8,1023,0.07036640048027039
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,1,128,1,float16,float16,2047,0.14808640480041504
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,1,128,1,float16,fp8,2047,0.1096992015838623
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,2,128,1,float16,float16,1,0.03997440040111542
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,2,128,1,float16,fp8,1,0.04046559929847717
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,2,128,1,float16,float16,3,0.04021120071411133
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,2,128,1,float16,fp8,3,0.04049279987812042
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,2,128,1,float16,float16,7,0.0401856005191803
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,2,128,1,float16,fp8,7,0.04055840075016022
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,2,128,1,float16,float16,15,0.040227198600769044
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,2,128,1,float16,fp8,15,0.04053280055522919
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,2,128,1,float16,float16,31,0.04024640023708344
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,2,128,1,float16,fp8,31,0.04043520092964172
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,2,128,1,float16,float16,63,0.04052639901638031
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,2,128,1,float16,fp8,63,0.040479999780654904
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,2,128,1,float16,float16,127,0.04808480143547058
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,2,128,1,float16,fp8,127,0.047697600722312924
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,2,128,1,float16,float16,255,0.06430879831314087
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,2,128,1,float16,fp8,255,0.05824000239372253
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,2,128,1,float16,float16,511,0.09234079718589783
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,2,128,1,float16,fp8,511,0.08215680122375488
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,2,128,1,float16,fp8,1023,0.11864800453186035
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,1,128,1,float16,float16,15,0.029323199391365053
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,1,128,1,float16,float16,63,0.029297599196434022
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,2,128,1,float16,float16,2047,0.2699951887130737
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,1,128,1,float16,float16,127,0.02972640097141266
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,2,128,1,float16,fp8,2047,0.19503999948501588
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,4,128,1,float16,float16,1,0.06163679957389832
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,4,128,1,float16,fp8,1,0.0618511974811554
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,4,128,1,float16,float16,3,0.061819201707839964
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,4,128,1,float16,fp8,3,0.061975997686386106
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,4,128,1,float16,float16,7,0.06164960265159607
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,4,128,1,float16,fp8,7,0.06204320192337036
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,4,128,1,float16,float16,15,0.062412798404693604
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,4,128,1,float16,fp8,15,0.061745601892471316
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,4,128,1,float16,float16,31,0.061921602487564086
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,4,128,1,float16,fp8,31,0.06251680254936218
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,4,128,1,float16,float16,63,0.06717759966850281
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,4,128,1,float16,fp8,63,0.06323360204696656
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,4,128,1,float16,float16,127,0.07221279740333557
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,4,128,1,float16,fp8,127,0.07882400155067444
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,4,128,1,float16,float16,255,0.10189919471740723
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,4,128,1,float16,fp8,255,0.09548159837722778
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,4,128,1,float16,float16,511,0.15782079696655274
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,4,128,1,float16,fp8,511,0.1394927978515625
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,4,128,1,float16,float16,1023,0.2737071990966797
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,4,128,1,float16,fp8,1023,0.2099008083343506
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,1,128,1,float16,float16,1,0.009881599992513656
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,1,128,1,float16,fp8,1,0.009644799679517747
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,4,128,1,float16,float16,2047,0.5020656108856201
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,4,128,1,float16,fp8,2047,0.36839520931243896
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,1,128,1,float16,fp8,3,0.009987200051546097
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,4,128,1,float16,float16,2047,0.26164960861206055
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,1,128,1,float16,float16,7,0.009947200119495393
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,1,128,1,float16,fp8,7,0.009724800288677216
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,1,128,1,float16,float16,15,0.009988799691200256
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,1,128,1,float16,fp8,15,0.009736000001430512
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,1,128,1,float16,float16,31,0.009950400143861771
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,1,128,1,float16,fp8,31,0.010023999959230423
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,1,128,1,float16,float16,63,0.009827200323343277
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,2,128,1,float16,float16,1023,0.1535040020942688
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,1,128,1,float16,float16,127,0.010175999999046326
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,1,128,1,float16,fp8,127,0.010264000296592713
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,1,128,1,float16,float16,255,0.011110399663448334
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,1,128,1,float16,float16,511,0.013232000172138214
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,1,128,1,float16,fp8,255,0.011449600011110306
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,1,128,1,float16,float16,1023,0.013827200233936309
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,1,128,1,float16,fp8,1023,0.013064000010490417
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,1,128,1,float16,fp8,2047,0.013942399621009826
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,1,128,1,float16,float16,4095,0.015328000485897064
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,1,128,1,float16,fp8,4095,0.014531199634075165
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,1,128,1,float16,float16,8191,0.017876799404621124
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,1,128,1,float16,fp8,8191,0.01744000017642975
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,1,128,1,float16,float16,16383,0.029718399047851562
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,1,128,1,float16,fp8,16383,0.022814400494098663
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,1,128,1,float16,float16,32767,0.04621599912643433
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,2,128,1,float16,float16,1,0.009646400064229965
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,2,128,1,float16,fp8,1,0.009481599926948548
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,2,128,1,float16,float16,3,0.009564799815416336
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,2,128,1,float16,fp8,3,0.009636799991130828
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,2,128,1,float16,float16,7,0.009779199957847595
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,2,128,1,float16,fp8,7,0.009641599655151368
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,2,128,1,float16,float16,15,0.00984319970011711
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,2,128,1,float16,fp8,15,0.009676799923181535
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,2,128,1,float16,float16,31,0.009673599898815156
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,1,128,1,float16,float16,3,0.009878399968147277
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,2,128,1,float16,fp8,31,0.009464000165462495
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,2,128,1,float16,float16,63,0.009620799869298934
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,2,128,1,float16,float16,127,0.009700799733400345
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,2,128,1,float16,fp8,127,0.010454399883747101
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,2,128,1,float16,float16,255,0.011135999858379365
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,2,128,1,float16,fp8,255,0.01149279996752739
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,2,128,1,float16,float16,511,0.013617600500583648
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,1,128,1,float16,fp8,63,0.009691199660301209
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,2,128,1,float16,fp8,511,0.012822400033473968
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,8,4,128,1,float16,fp8,4095,0.33497281074523927
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,2,128,1,float16,fp8,1023,0.013769599795341491
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,1,128,1,float16,fp8,511,0.012603199481964112
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,2,128,1,float16,float16,2047,0.015489600598812103
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,2,128,1,float16,float16,4095,0.01797440052032471
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,2,128,1,float16,fp8,4095,0.01679839938879013
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,2,128,1,float16,fp8,8191,0.02136639952659607
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,2,128,1,float16,float16,16383,0.043987199664115906
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,2,128,1,float16,fp8,16383,0.032579201459884646
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,2,128,1,float16,float16,32767,0.07397440075874329
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,2,128,1,float16,fp8,32767,0.05151519775390625
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,4,128,1,float16,float16,1,0.009796799719333648
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,4,128,1,float16,fp8,1,0.009825599938631057
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,4,128,1,float16,float16,3,0.009864000231027603
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,4,128,1,float16,fp8,3,0.009849599748849868
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,1,128,1,float16,fp8,32767,0.034030398726463316
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,4,128,1,float16,float16,7,0.009857600182294845
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,4,128,1,float16,fp8,7,0.009771200269460678
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,4,128,1,float16,float16,15,0.009932799637317658
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,4,128,1,float16,fp8,15,0.009673599898815156
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,4,128,1,float16,fp8,31,0.009668800234794616
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,4,128,1,float16,float16,63,0.00984480008482933
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,4,128,1,float16,fp8,63,0.009702400118112565
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,4,128,1,float16,float16,127,0.010102400183677673
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,4,128,1,float16,fp8,127,0.010699199885129929
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,4,128,1,float16,float16,255,0.011708799749612808
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,2,128,1,float16,fp8,63,0.009424000233411788
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,4,128,1,float16,fp8,255,0.011406400054693223
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,4,128,1,float16,float16,511,0.013544000685214996
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,4,128,1,float16,fp8,511,0.013348799943923951
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,4,128,1,float16,float16,1023,0.014852799475193024
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,4,128,1,float16,fp8,1023,0.01387840062379837
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,4,128,1,float16,float16,2047,0.0162992000579834
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,4,128,1,float16,fp8,2047,0.01642400026321411
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,4,128,1,float16,float16,4095,0.028123199939727783
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,2,128,1,float16,fp8,2047,0.014244799315929414
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,4,128,1,float16,fp8,4095,0.02075839936733246
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,1,128,1,float16,float16,2047,0.014345599710941315
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,4,128,1,float16,float16,8191,0.04332480132579804
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,2,128,1,float16,float16,8191,0.028993600606918336
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,4,128,1,float16,fp8,16383,0.050710397958755496
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,4,128,1,float16,float16,32767,0.1337440013885498
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,4,128,1,float16,fp8,32767,0.08877760171890259
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,8,1,128,1,float16,float16,1,0.04680800139904022
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,8,1,128,1,float16,fp8,1,0.04659200012683869
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,8,1,128,1,float16,float16,3,0.04688000082969666
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,8,1,128,1,float16,fp8,3,0.04670400023460388
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,8,1,128,1,float16,float16,7,0.04697119891643524
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,8,1,128,1,float16,fp8,7,0.046742400527000426
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,8,1,128,1,float16,float16,15,0.04711839854717255
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,8,1,128,1,float16,fp8,15,0.04693920016288757
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,8,1,128,1,float16,float16,31,0.04715839922428131
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,8,1,128,1,float16,fp8,31,0.04684160053730011
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,8,1,128,1,float16,float16,63,0.047651201486587524
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,8,1,128,1,float16,fp8,63,0.04684160053730011
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,8,1,128,1,float16,float16,127,0.05432959794998169
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,4,128,1,float16,float16,31,0.01005759984254837
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,8,1,128,1,float16,fp8,127,0.05380319952964783
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,8,1,128,1,float16,float16,255,0.06906560063362122
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,8,1,128,1,float16,fp8,255,0.06475840210914612
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,8,1,128,1,float16,float16,511,0.09904800057411194
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,8,1,128,1,float16,fp8,1023,0.12531360387802123
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,8,2,128,1,float16,float16,1,0.06791200041770935
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,8,2,128,1,float16,fp8,1,0.06858400106430054
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,8,2,128,1,float16,float16,3,0.06800159811973572
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,8,2,128,1,float16,fp8,3,0.06838719844818116
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,2,128,1,float16,float16,1023,0.013838399946689606
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,8,2,128,1,float16,float16,7,0.06793760061264038
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,8,2,128,1,float16,fp8,7,0.06803839802742004
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,8,2,128,1,float16,float16,15,0.06853920221328735
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,4,128,1,float16,fp8,8191,0.03215680122375488
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,8,2,128,1,float16,float16,31,0.06806719899177552
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,8,2,128,1,float16,fp8,31,0.06847040057182312
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,8,2,128,1,float16,float16,63,0.07539520263671876
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,8,2,128,1,float16,fp8,63,0.06930239796638489
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,8,2,128,1,float16,float16,127,0.08099359869956971
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,8,2,128,1,float16,fp8,127,0.0866320013999939
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,8,2,128,1,float16,float16,255,0.10804799795150757
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,8,2,128,1,float16,fp8,255,0.10362880229949951
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,8,2,128,1,float16,float16,511,0.1647279977798462
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,8,2,128,1,float16,fp8,511,0.14861119985580445
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,8,2,128,1,float16,float16,1023,0.2845312118530273
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,8,2,128,1,float16,fp8,1023,0.2195296049118042
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,8,1,128,1,float16,fp8,511,0.08930879831314087
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,8,4,128,1,float16,float16,1,0.11021920442581176
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,8,4,128,1,float16,fp8,1,0.10951520204544067
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,8,4,128,1,float16,float16,3,0.11019359827041626
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,8,4,128,1,float16,float16,7,0.10954400300979614
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,8,4,128,1,float16,fp8,7,0.11039680242538452
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,8,4,128,1,float16,float16,15,0.1097648024559021
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,8,4,128,1,float16,fp8,15,0.110316801071167
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,8,4,128,1,float16,float16,31,0.11267520189285278
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,8,4,128,1,float16,fp8,31,0.11016639471054077
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,8,4,128,1,float16,float16,63,0.12131359577178955
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,8,4,128,1,float16,fp8,63,0.11875519752502442
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,8,1,128,1,float16,fp8,15,0.029070401191711427
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,8,4,128,1,float16,float16,127,0.1298416018486023
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,8,4,128,1,float16,fp8,127,0.14483040571212769
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,8,4,128,1,float16,float16,255,0.18463200330734253
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,8,4,128,1,float16,fp8,255,0.17448799610137938
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,8,4,128,1,float16,float16,511,0.29907999038696287
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,8,4,128,1,float16,float16,16383,0.07361760139465331
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,8,4,128,1,float16,fp8,511,0.2638047933578491
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,8,1,128,1,float16,float16,1,0.0790336012840271
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,8,1,128,1,float16,fp8,1,0.07921760082244873
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,8,1,128,1,float16,fp8,3,0.07900159955024719
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,8,1,128,1,float16,float16,7,0.07835519909858704
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,8,4,128,1,float16,fp8,1023,0.40125279426574706
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,8,1,128,1,float16,fp8,7,0.07886559963226318
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,8,4,128,1,float16,float16,1023,0.5292287826538086
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,8,1,128,1,float16,float16,15,0.07853599786758422
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,8,1,128,1,float16,float16,31,0.07832319736480713
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,8,1,128,1,float16,fp8,15,0.07882879972457886
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,8,1,128,1,float16,float16,63,0.0869376003742218
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,8,1,128,1,float16,fp8,31,0.07951679825782776
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,8,1,128,1,float16,fp8,63,0.0802735984325409
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,8,1,128,1,float16,float16,127,0.09277920126914978
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,8,1,128,1,float16,float16,255,0.11793279647827148
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,8,1,128,1,float16,fp8,255,0.11559040546417236
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,8,1,128,1,float16,float16,511,0.17213759422302247
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,8,1,128,1,float16,fp8,511,0.16161279678344725
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,8,2,128,1,float16,float16,1,0.12015199661254883
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,8,2,128,1,float16,fp8,1,0.12021600008010865
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,8,2,128,1,float16,float16,3,0.11938879489898682
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,8,2,128,1,float16,fp8,3,0.12060480117797852
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,8,4,128,1,float16,fp8,3,0.10990560054779053
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,8,1,128,1,float16,float16,1023,0.1614896059036255
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,8,2,128,1,float16,float16,7,0.11940959692001343
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,8,2,128,1,float16,fp8,7,0.12047040462493896
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,8,2,128,1,float16,float16,15,0.11986399888992309
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,8,2,128,1,float16,fp8,15,0.1207919955253601
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,8,2,128,1,float16,float16,31,0.12472800016403199
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,8,2,128,1,float16,fp8,31,0.12072319984436035
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,8,2,128,1,float16,float16,63,0.13413439989089965
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,8,2,128,1,float16,fp8,63,0.13246560096740723
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,8,2,128,1,float16,fp8,127,0.1565999984741211
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,8,2,128,1,float16,fp8,15,0.0681984007358551
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,8,2,128,1,float16,float16,255,0.19157760143280028
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,8,2,128,1,float16,float16,511,0.30205440521240234
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,8,2,128,1,float16,fp8,511,0.27797119617462157
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,8,1,128,1,float16,float16,3,0.07837600111961365
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,8,4,128,1,float16,float16,3,0.20229439735412597
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,8,4,128,1,float16,fp8,3,0.2032304048538208
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,8,4,128,1,float16,float16,7,0.2025536060333252
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,8,4,128,1,float16,fp8,7,0.20315680503845215
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,8,4,128,1,float16,float16,15,0.20536320209503173
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,8,1,128,1,float16,fp8,127,0.09864159822463989
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,8,4,128,1,float16,fp8,15,0.20391359329223632
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,8,4,128,1,float16,float16,31,0.2155359983444214
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,8,4,128,1,float16,fp8,31,0.21259520053863526
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,8,4,128,1,float16,float16,63,0.22465600967407226
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,8,4,128,1,float16,fp8,63,0.22263679504394532
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,8,4,128,1,float16,float16,127,0.23750081062316894
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,8,4,128,1,float16,fp8,127,0.2717008113861084
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,8,4,128,1,float16,float16,255,0.3426032066345215
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,8,4,128,1,float16,fp8,255,0.3302848100662231
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,8,2,128,1,float16,float16,127,0.14288640022277832
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,1,128,1,float16,float16,1,0.009640000015497207
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,8,4,128,1,float16,float16,511,0.5767920017242432
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,1,128,1,float16,fp8,1,0.009892799705266953
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,8,4,128,1,float16,fp8,511,0.5006256103515625
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,1,128,1,float16,float16,7,0.0102463997900486
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,1,128,1,float16,fp8,3,0.01000640019774437
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,8,2,128,1,float16,fp8,255,0.18964799642562866
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,1,128,1,float16,fp8,7,0.009904000163078307
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,1,128,1,float16,fp8,15,0.009718400239944459
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,1,128,1,float16,float16,31,0.010115200281143188
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,1,128,1,float16,fp8,31,0.009558399766683578
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,1,128,1,float16,fp8,63,0.009926400333642959
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,8,4,128,1,float16,float16,1,0.2032383918762207
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,1,128,1,float16,fp8,127,0.010574399679899215
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,1,128,1,float16,float16,255,0.01088479980826378
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,1,128,1,float16,fp8,255,0.011584000289440155
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,1,128,1,float16,float16,511,0.013324800133705138
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,1,128,1,float16,fp8,511,0.012985600531101227
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,1,128,1,float16,float16,1023,0.014129599928855896
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,1,128,1,float16,fp8,1023,0.01371999979019165
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,1,128,1,float16,float16,2047,0.015571199357509613
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,1,128,1,float16,fp8,2047,0.01467519998550415
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,1,128,1,float16,float16,4095,0.017822399735450745
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,1,128,1,float16,fp8,4095,0.017108799517154695
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,1,128,1,float16,float16,8191,0.029371199011802674
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,1,128,1,float16,fp8,8191,0.02152799963951111
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,1,128,1,float16,float16,16383,0.04383040070533752
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,1,128,1,float16,fp8,16383,0.03278720080852508
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,1,128,1,float16,float16,32767,0.07462400197982788
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,1,128,1,float16,fp8,32767,0.052241599559783934
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,2,128,1,float16,float16,1,0.010113599896430969
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,2,128,1,float16,fp8,1,0.01019200012087822
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,2,128,1,float16,float16,3,0.009950400143861771
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,2,128,1,float16,fp8,3,0.010224000364542008
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,2,128,1,float16,fp8,7,0.010275200009346008
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,2,128,1,float16,float16,15,0.010363200306892395
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,2,128,1,float16,fp8,15,0.010318399965763092
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,2,128,1,float16,float16,31,0.010302399843931198
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,2,128,1,float16,fp8,31,0.010180799663066864
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,2,128,1,float16,float16,63,0.010195200145244599
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,2,128,1,float16,fp8,63,0.009985599666833878
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,2,128,1,float16,float16,127,0.010463999956846238
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,2,128,1,float16,fp8,127,0.010903999954462052
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,2,128,1,float16,float16,255,0.011812800168991089
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,2,128,1,float16,fp8,255,0.011724799871444702
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,2,128,1,float16,float16,511,0.013817599415779114
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,1,128,1,float16,float16,3,0.00955360010266304
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,2,128,1,float16,fp8,511,0.013728000223636627
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,2,128,1,float16,float16,1023,0.014727999269962311
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,2,128,1,float16,float16,2047,0.016412800550460814
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,1,128,1,float16,float16,15,0.010283199697732925
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,2,128,1,float16,fp8,2047,0.01656319946050644
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,2,128,1,float16,float16,4095,0.02879360020160675
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,1,128,1,float16,float16,63,0.009827200323343277
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,2,128,1,float16,fp8,4095,0.021140800416469575
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,2,128,1,float16,fp8,8191,0.03250080049037933
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,1,128,1,float16,float16,127,0.010251200199127198
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,2,128,1,float16,float16,16383,0.07360960245132446
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,8,4,128,1,float16,fp8,1,0.20180799961090087
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,2,128,1,float16,float16,32767,0.13440799713134766
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,4,128,1,float16,float16,1,0.010254400223493576
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,4,128,1,float16,fp8,3,0.009830400347709656
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,4,128,1,float16,fp8,1,0.010054399818181991
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,4,128,1,float16,float16,7,0.009894400089979171
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,4,128,1,float16,fp8,7,0.01003360003232956
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,4,128,1,float16,float16,15,0.009907200187444686
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,4,128,1,float16,fp8,15,0.010062400251626968
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,4,128,1,float16,fp8,31,0.010049600154161453
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,4,128,1,float16,float16,31,0.010020799934864044
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,4,128,1,float16,float16,63,0.010215999931097031
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,4,128,1,float16,fp8,63,0.009985599666833878
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,4,128,1,float16,float16,127,0.01029760017991066
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,4,128,1,float16,fp8,127,0.010763200372457505
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,4,128,1,float16,float16,255,0.01165279969573021
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,4,128,1,float16,fp8,255,0.011561600118875503
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,2,128,1,float16,float16,7,0.009910400211811065
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,4,128,1,float16,float16,511,0.01398719996213913
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,4,128,1,float16,float16,1023,0.016411200165748596
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,4,128,1,float16,fp8,1023,0.016519999504089354
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,4,128,1,float16,float16,2047,0.02829119861125946
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,4,128,1,float16,fp8,2047,0.021524800360202788
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,4,128,1,float16,float16,4095,0.043278399109840396
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,4,128,1,float16,fp8,4095,0.031886398792266846
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,4,128,1,float16,float16,8191,0.07356160283088684
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,4,128,1,float16,fp8,8191,0.0502128005027771
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,4,128,1,float16,float16,16383,0.13224480152130128
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,2,128,1,float16,fp8,1023,0.014448000490665436
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,4,128,1,float16,fp8,16383,0.08785759806632995
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,2,128,1,float16,float16,8191,0.04418880045413971
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,4,128,1,float16,float16,32767,0.24962239265441893
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,1,0.011339200288057327
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,4,128,1,float16,fp8,32767,0.16334880590438844
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,1,128,1,float16,fp8,1,0.01096159964799881
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,3,0.011308799684047698
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,2,128,1,float16,fp8,16383,0.05135520100593567
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,7,0.0110944002866745
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,2,128,1,float16,fp8,32767,0.08982080221176147
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,15,0.011131200194358825
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,1,128,1,float16,fp8,31,0.011240000277757645
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,31,0.011372800171375274
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,63,0.011377599835395814
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,1,128,1,float16,fp8,63,0.011159999668598175
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,127,0.011814399808645248
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,1,128,1,float16,fp8,127,0.012380799651145935
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,255,0.013129599392414093
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,1,128,1,float16,fp8,255,0.012916800379753113
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,511,0.01496639996767044
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,1,128,1,float16,fp8,511,0.019388799369335175
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,1023,0.01890240013599396
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,1,128,1,float16,fp8,1023,0.01932799965143204
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,2047,0.03089280128479004
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,1,128,1,float16,fp8,2047,0.026499199867248534
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,4095,0.04577920138835907
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,4,128,1,float16,fp8,511,0.01478399932384491
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,8191,0.07714400291442872
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,1,128,1,float16,fp8,8191,0.05789600014686584
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,16383,0.13911839723587036
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,1,128,1,float16,fp8,16383,0.09602720141410828
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,1,128,1,float16,fp8,32767,0.17210079431533815
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,32767,0.25724000930786134
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,1,128,1,float16,fp8,65535,0.32490880489349366
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,65535,0.4947728157043457
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,1,0.011104000359773636
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,2,128,1,float16,fp8,1,0.011163199692964554
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,3,0.011240000277757645
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,1,128,1,float16,fp8,3,0.011296000331640244
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,1,128,1,float16,fp8,7,0.01114400029182434
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,1,128,1,float16,fp8,15,0.011116799712181092
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,2,128,1,float16,fp8,7,0.011611200124025344
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,8,4,128,1,float16,float16,3,0.010360000282526016
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,2,128,1,float16,fp8,15,0.011320000141859054
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,31,0.011614400148391723
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,2,128,1,float16,fp8,31,0.011139199882745743
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,63,0.011308799684047698
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,127,0.06834400296211243
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,2,128,1,float16,fp8,127,0.012107200175523757
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,255,0.01284160017967224
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,2,128,1,float16,fp8,255,0.012868799269199371
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,511,0.015057599544525147
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,2,128,1,float16,fp8,511,0.016648000478744505
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,1023,0.028412801027297974
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,2,128,1,float16,fp8,1023,0.021483199298381807
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,2047,0.04413279891014099
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,2,128,1,float16,fp8,2047,0.0321696013212204
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,4095,0.07424479722976685
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,2,128,1,float16,fp8,4095,0.05106239914894104
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,2,128,1,float16,fp8,8191,0.08971840143203735
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,16383,0.2503999948501587
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,2,128,1,float16,fp8,16383,0.1649791955947876
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,32767,0.492249584197998
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,7,0.011262399703264236
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,2,128,1,float16,fp8,3,0.011108800023794174
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,2,128,1,float16,fp8,32767,0.31460320949554443
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,1,0.00944959968328476
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,15,0.011451199650764465
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,3,0.009363199770450591
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,1,128,1,float16,fp8,3,0.009377600252628326
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,7,0.009310399740934372
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,1,128,1,float16,fp8,7,0.009459199756383896
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,15,0.009345600008964538
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,1,128,1,float16,fp8,15,0.00952960029244423
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,1,128,1,float16,fp8,31,0.009441599994897843
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,63,0.00971359983086586
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,1,128,1,float16,fp8,63,0.009391999989748
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,1,128,1,float16,fp8,4095,0.03610239923000336
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,65535,0.9687888145446777
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,1,128,1,float16,fp8,127,0.010076799988746643
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,1,128,1,float16,fp8,255,0.011179199814796448
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,255,0.01088479980826378
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,511,0.013007999956607818
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,1,128,1,float16,fp8,511,0.01281599998474121
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,8191,0.1347823977470398
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,1,128,1,float16,fp8,1023,0.013441599905490875
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,2047,0.014350399374961853
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,1,128,1,float16,fp8,2047,0.013691200315952301
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,4095,0.014313599467277527
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,1,128,1,float16,fp8,4095,0.015227200090885162
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,8191,0.016344000399112702
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,1,128,1,float16,fp8,8191,0.01844799965620041
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,16383,0.01950400024652481
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,1,128,1,float16,fp8,16383,0.01934719979763031
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,32767,0.021059200167655945
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,1,128,1,float16,fp8,32767,0.020798400044441223
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,65535,0.02401919960975647
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,1,128,1,float16,fp8,65535,0.023609599471092223
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,1,0.009438399970531464
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,1,128,1,float16,fp8,1,0.009207999706268311
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,2,128,1,float16,fp8,63,0.01165279969573021
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,3,0.009622400254011154
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,7,0.009556800127029419
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,2,128,1,float16,fp8,3,0.009310399740934372
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,2,128,1,float16,fp8,7,0.009544000029563904
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,31,0.00923840031027794
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,2,128,1,float16,fp8,15,0.00947680026292801
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,31,0.009427200257778167
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,63,0.009497600048780442
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,2,128,1,float16,fp8,63,0.009137599915266036
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,127,0.009471999853849411
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,127,0.00963200032711029
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,2,128,1,float16,fp8,127,0.00997759997844696
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,255,0.01063840016722679
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,511,0.01287200003862381
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,2,128,1,float16,fp8,511,0.012828800082206725
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,1023,0.01406400054693222
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,1023,0.01422239989042282
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,2,128,1,float16,fp8,1023,0.0134320005774498
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,2,128,1,float16,fp8,2047,0.013700799643993377
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,4095,0.014486399292945863
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,2,128,1,float16,fp8,4095,0.01544319987297058
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,8191,0.016603200137615202
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,2,128,1,float16,fp8,8191,0.016073599457740784
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,16383,0.017532800137996674
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,2,128,1,float16,fp8,16383,0.0173007994890213
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,32767,0.020423999428749083
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,2,128,1,float16,fp8,32767,0.02016959935426712
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,65535,0.031974399089813234
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,2,128,1,float16,fp8,65535,0.02543199956417084
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,1,0.009382399916648864
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,1,128,1,float16,fp8,1,0.009344000369310379
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,3,0.009375999867916106
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,1,128,1,float16,fp8,3,0.00931359976530075
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,7,0.00942239984869957
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,1,128,1,float16,fp8,7,0.009497600048780442
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,15,0.00957759991288185
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,1,128,1,float16,fp8,15,0.009505599737167358
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,31,0.009694399684667588
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,15,0.009564799815416336
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,63,0.009636799991130828
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,2,128,1,float16,fp8,31,0.009540800005197525
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,127,0.009550400078296661
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,255,0.010755199939012527
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,4,2,128,1,float16,fp8,65535,0.6185328006744385
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,2,128,1,float16,fp8,255,0.010830400139093399
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,511,0.013155199587345123
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,1023,0.014209599792957306
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,1,128,1,float16,fp8,1023,0.013577599823474885
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,2047,0.014390400052070618
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,1,128,1,float16,fp8,2047,0.013798399269580841
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,4095,0.014545600116252898
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,1,128,1,float16,fp8,4095,0.015497599542140961
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,8191,0.01656640022993088
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,1,128,1,float16,fp8,8191,0.018649600446224213
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,16383,0.02048960030078888
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,1,128,1,float16,fp8,16383,0.019776000082492827
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,32767,0.02282239943742752
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,1,128,1,float16,fp8,32767,0.02227199971675873
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,65535,0.03430399894714355
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,1,128,1,float16,fp8,65535,0.027963200211524965
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,1,0.00957920029759407
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,2,128,1,float16,fp8,1,0.009563200175762177
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,4,2,128,1,float16,fp8,1,0.009323199838399887
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,3,0.009569600224494934
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,7,0.009393599629402161
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,2,128,1,float16,fp8,7,0.009361600130796432
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,15,0.009595199674367904
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,1,128,1,float16,fp8,31,0.009536000341176987
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,31,0.009406399726867676
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,1,128,1,float16,fp8,63,0.009292799979448318
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,2,128,1,float16,fp8,31,0.009204799681901932
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,63,0.009567999839782714
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,2,128,1,float16,fp8,63,0.00926240012049675
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,127,0.009457600116729737
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,2,128,1,float16,fp8,127,0.010091199725866317
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,1,128,1,float16,fp8,255,0.011177600175142289
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,255,0.01067200005054474
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,2,128,1,float16,fp8,255,0.010945600271224976
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,2,128,1,float16,fp8,511,0.013276800513267517
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,1023,0.014323200285434722
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,2,128,1,float16,fp8,1023,0.013947199285030364
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,1,128,1,float16,fp8,511,0.012950399518013
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,2,128,1,float16,fp8,2047,0.013974399864673614
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,4095,0.014924800395965577
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,2,128,1,float16,fp8,4095,0.015588800609111785
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,8191,0.017576000094413756
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,2047,0.0141184002161026
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,2,128,1,float16,fp8,8191,0.016740800440311433
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,16383,0.01980479955673218
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,2,128,1,float16,fp8,16383,0.01935359984636307
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,32767,0.0310479998588562
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,65535,0.047860801219940186
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,2,128,1,float16,fp8,65535,0.03580319881439209
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,1,0.010360000282526016
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,1,128,1,float16,fp8,1,0.010236799716949463
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,3,0.010179200023412705
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,1,128,1,float16,fp8,3,0.01021760031580925
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,7,0.010201600193977357
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,1,128,1,float16,fp8,7,0.010182400047779084
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,15,0.010393600165843963
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,1,128,1,float16,fp8,15,0.010384000092744827
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,31,0.010391999781131745
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,1,128,1,float16,fp8,31,0.010419200360774993
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,63,0.010427200049161912
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,2,128,1,float16,fp8,15,0.009419199824333192
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,1,128,1,float16,fp8,63,0.010252799838781357
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,1,128,1,float16,fp8,127,0.011158400028944016
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,255,0.011998400092124939
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,1,128,1,float16,fp8,127,0.010225600004196167
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,1,128,1,float16,fp8,255,0.011844799667596818
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,1,128,1,float16,fp8,511,0.014827199280261993
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,511,0.01420000046491623
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,1023,0.016631999611854555
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,1,128,1,float16,fp8,1023,0.015969599783420562
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,2047,0.014420799911022186
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,2047,0.018452799320220946
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,4095,0.03019840121269226
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,8191,0.04695520102977753
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,1,128,1,float16,fp8,8191,0.038140800595283506
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,16383,0.08013120293617249
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,2,128,1,float16,fp8,32767,0.024724799394607543
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,1,128,1,float16,fp8,16383,0.05699040293693543
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,32767,0.14021919965744017
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,1,128,1,float16,fp8,32767,0.09588159918785095
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,1,128,1,float16,fp8,65535,0.1709488034248352
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,1,0.010239999741315842
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,2,128,1,float16,fp8,1,0.010255999863147736
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,3,0.010230399668216705
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,2,128,1,float16,fp8,3,0.010208000242710114
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,7,0.010438399761915207
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,2,128,1,float16,fp8,7,0.010427200049161912
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,15,0.010555200278759003
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,127,0.010579200088977813
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,2,128,1,float16,fp8,15,0.010288000106811523
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,2,128,1,float16,fp8,31,0.010214400291442872
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,63,0.010291200131177902
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,2,128,1,float16,fp8,63,0.010526400059461594
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,511,0.013072000443935394
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,127,0.010867200046777725
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,255,0.012278400361537933
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,2,128,1,float16,fp8,127,0.011497599631547928
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,1,128,1,float16,fp8,2047,0.018243199586868285
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,2,128,1,float16,fp8,255,0.012196800112724305
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,511,0.014580799639225006
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,2,128,1,float16,fp8,511,0.01855839937925339
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,1023,0.018196800351142885
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,2,128,1,float16,fp8,1023,0.01804479956626892
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,2047,0.0300464004278183
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,2,128,1,float16,fp8,2047,0.025705599784851076
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,4095,0.0449535995721817
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,2,128,1,float16,fp8,4095,0.035104000568389894
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,8191,0.0771023988723755
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,2,128,1,float16,fp8,8191,0.05406079888343811
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,2,128,1,float16,fp8,16383,0.09226239919662475
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,16383,0.1356592059135437
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,2,128,1,float16,fp8,32767,0.16840319633483886
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,1,0.009623999893665313
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,65535,0.4899360179901123
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,1,128,1,float16,fp8,1,0.00942080020904541
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,2,128,1,float16,fp8,65535,0.3203743934631348
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,1,128,1,float16,fp8,3,0.009457600116729737
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,3,0.009497600048780442
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,7,0.009603200107812881
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,1,128,1,float16,fp8,7,0.009359999746084213
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,15,0.00955199971795082
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,31,0.009507200121879578
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,1,128,1,float16,fp8,31,0.009465599805116654
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,63,0.009355200082063675
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,1,128,1,float16,fp8,63,0.009433600306510925
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,127,0.009457600116729737
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,255,0.01077599972486496
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,1,128,1,float16,fp8,255,0.010948800295591355
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,511,0.012780800461769104
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,31,0.01079839989542961
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,1,128,1,float16,fp8,511,0.013247999548912048
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,1,128,1,float16,fp8,1023,0.01393440067768097
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,2047,0.014628799259662628
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,1,128,1,float16,fp8,2047,0.013900800049304963
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,4095,0.014903999865055084
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,1,128,1,float16,fp8,4095,0.015523199737071992
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,8191,0.01778080016374588
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,1,128,1,float16,fp8,8191,0.019515199959278105
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,16383,0.0225615993142128
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,32767,0.2560816049575806
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,1,128,1,float16,fp8,16383,0.02191839963197708
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,65535,0.26004319190979003
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,32767,0.034027200937271115
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,1,128,1,float16,fp8,32767,0.027331200242042542
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,65535,0.0542352020740509
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,1,128,1,float16,fp8,65535,0.038545599579811095
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,2,128,1,float16,fp8,1,0.009583999961614608
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,1,128,1,float16,fp8,15,0.00942239984869957
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,4,2,128,1,float16,fp8,3,0.00939679965376854
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,3,0.009641599655151368
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,2,128,1,float16,fp8,3,0.009678400307893752
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,7,0.009635200351476669
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,1,128,1,float16,fp8,127,0.010174400359392165
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,2,128,1,float16,fp8,15,0.00958240032196045
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,31,0.009491200000047684
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,2,128,1,float16,fp8,31,0.009588800370693207
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,63,0.009491200000047684
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,2,128,1,float16,fp8,63,0.009667199850082398
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,127,0.009483200311660767
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,2,128,1,float16,fp8,127,0.01043040007352829
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,2,128,1,float16,fp8,255,0.011286400258541107
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,1023,0.01448799967765808
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,2,128,1,float16,fp8,511,0.013324800133705138
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,1023,0.014449599385261535
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,2,128,1,float16,fp8,1023,0.013799999654293061
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,4,1,128,1,float16,fp8,4095,0.02396479994058609
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,2047,0.015105600655078887
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,2,128,1,float16,fp8,2047,0.014271999895572662
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,4095,0.01621920019388199
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,2,128,1,float16,fp8,4095,0.016388800740242005
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,8191,0.019849599897861482
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,2,128,1,float16,fp8,8191,0.019196799397468566
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,16383,0.030692800879478455
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,2,128,1,float16,fp8,16383,0.02444159984588623
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,32767,0.04697439968585968
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,2,128,1,float16,fp8,32767,0.03538239896297455
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,65535,0.07661120295524597
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,2,128,1,float16,fp8,65535,0.05562880039215088
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,1,128,1,float16,float16,1,0.012742400169372559
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,1,128,1,float16,fp8,1,0.01242400035262108
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,1,128,1,float16,float16,3,0.01313759982585907
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,1,128,1,float16,fp8,3,0.012345600128173827
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,1,128,1,float16,float16,7,0.013273599743843078
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,1,128,1,float16,fp8,7,0.012372799962759019
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,1,128,1,float16,float16,15,0.013171200454235078
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,1,128,1,float16,fp8,15,0.012303999811410903
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,1,128,1,float16,float16,31,0.013155199587345123
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,1,128,1,float16,fp8,31,0.012643200159072877
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,1,128,1,float16,float16,63,0.013187199831008911
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,1,128,1,float16,fp8,63,0.01265760064125061
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,1,128,1,float16,float16,127,0.01345279961824417
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,1,128,1,float16,fp8,127,0.013767999410629273
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,1,128,1,float16,fp8,255,0.014750400185585022
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,1,128,1,float16,float16,511,0.016808000206947327
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,1,0.009571199864149093
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,1,128,1,float16,float16,1023,0.03028959929943085
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,2,128,1,float16,fp8,7,0.009515199810266495
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,1,128,1,float16,fp8,1023,0.022916799783706664
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,15,0.009908799827098847
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,1,128,1,float16,float16,2047,0.04505119919776916
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,1,128,1,float16,fp8,2047,0.0345984011888504
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,1,128,1,float16,float16,4095,0.0745024025440216
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,1,128,1,float16,fp8,4095,0.052920001745224
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,1,128,1,float16,float16,8191,0.1339136004447937
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,1,128,1,float16,fp8,8191,0.09180319905281067
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,255,0.010679999738931656
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,511,0.012878400087356568
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,2,128,1,float16,float16,1,0.01507679969072342
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,1,128,1,float16,fp8,16383,0.16747519969940186
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,2,128,1,float16,fp8,1,0.015406399965286255
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,2,128,1,float16,float16,3,0.01528480052947998
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,2,128,1,float16,float16,7,0.015535999834537507
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,2,128,1,float16,fp8,3,0.01520799994468689
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,2,128,1,float16,float16,15,0.015526400506496429
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,2,128,1,float16,fp8,15,0.015539200603961944
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,2,128,1,float16,float16,31,0.01568319946527481
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,1,128,1,float16,fp8,32767,0.31887359619140626
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,2,128,1,float16,fp8,31,0.015662400424480437
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,2,128,1,float16,float16,63,0.015707199275493623
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,2,128,1,float16,float16,127,0.015932799875736238
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,2,128,1,float16,fp8,127,0.017099200189113616
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,2,128,1,float16,fp8,63,0.015246400237083435
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,2,128,1,float16,float16,255,0.01849440038204193
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,2,128,1,float16,fp8,255,0.018961599469184874
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,2,128,1,float16,float16,511,0.030694401264190672
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,2,128,1,float16,fp8,511,0.025624001026153566
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,2,128,1,float16,float16,1023,0.047672000527381894
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,2,128,1,float16,fp8,1023,0.03592639863491058
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,2,128,1,float16,float16,2047,0.07721760272979736
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,2,128,1,float16,fp8,2047,0.05617280006408691
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,2,128,1,float16,float16,4095,0.13567039966583253
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,2,128,1,float16,fp8,4095,0.09368640184402466
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,2,128,1,float16,float16,8191,0.2534127950668335
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,2,128,1,float16,fp8,8191,0.17020800113677978
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,2,128,1,float16,float16,16383,0.48644318580627444
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,1,128,1,float16,float16,255,0.014596800506114959
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,4,1,128,1,float16,float16,1,0.017766399681568144
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,1,128,1,float16,fp8,511,0.018484799563884734
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,4,1,128,1,float16,fp8,1,0.01775519996881485
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,4,1,128,1,float16,float16,3,0.018174399435520173
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,4,1,128,1,float16,fp8,3,0.018199999630451203
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,4,1,128,1,float16,float16,7,0.01823839992284775
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,4,1,128,1,float16,fp8,7,0.01807519942522049
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,4,1,128,1,float16,float16,15,0.018134400248527527
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,4,1,128,1,float16,fp8,15,0.018059200048446654
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,2,128,1,float16,fp8,32767,0.6423967838287353
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,4,1,128,1,float16,float16,31,0.01815200001001358
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,4,1,128,1,float16,float16,63,0.018212799727916718
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,4,1,128,1,float16,fp8,63,0.018137599527835845
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,4,1,128,1,float16,float16,127,0.01841920018196106
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,4,1,128,1,float16,fp8,127,0.019888000190258028
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,4,1,128,1,float16,float16,255,0.021529600024223328
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,1,128,1,float16,float16,32767,0.48920159339904784
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,4,1,128,1,float16,fp8,255,0.021884800493717195
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,4,1,128,1,float16,fp8,511,0.028415998816490172
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,4,1,128,1,float16,float16,511,0.03440000116825104
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,4,1,128,1,float16,float16,1023,0.04969600141048432
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,4,1,128,1,float16,fp8,1023,0.03892639875411987
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,4,1,128,1,float16,fp8,2047,0.05894399881362915
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,4,1,128,1,float16,float16,2047,0.08017280101776122
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,4,1,128,1,float16,float16,4095,0.13874560594558716
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,4,1,128,1,float16,fp8,4095,0.09644320011138915
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,4,1,128,1,float16,float16,8191,0.2552783966064453
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,4,1,128,1,float16,fp8,8191,0.17294080257415773
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,4,2,128,1,float16,float16,1,0.023238399624824525
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,4,2,128,1,float16,fp8,1,0.02370399981737137
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,2,128,1,float16,fp8,16383,0.31963679790496824
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,4,2,128,1,float16,fp8,3,0.023800000548362732
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,4,2,128,1,float16,fp8,7,0.0237296000123024
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,4,2,128,1,float16,float16,7,0.023446400463581086
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,4,2,128,1,float16,float16,15,0.023980799317359924
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,4,2,128,1,float16,fp8,15,0.023313599824905395
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,4,2,128,1,float16,float16,31,0.02393600046634674
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,4,2,128,1,float16,fp8,31,0.023761600255966187
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,4,2,128,1,float16,float16,63,0.023897600173950196
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,4,2,128,1,float16,fp8,63,0.023822399973869323
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,4,2,128,1,float16,float16,127,0.024512000381946564
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,4,2,128,1,float16,fp8,127,0.0270687997341156
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,2,128,1,float16,float16,32767,0.9614447593688965
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,4,2,128,1,float16,float16,255,0.037257599830627444
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,4,2,128,1,float16,fp8,511,0.04503040015697479
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,1,128,1,float16,float16,16383,0.2519855976104736
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,4,2,128,1,float16,float16,511,0.05460960268974304
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,4,1,128,1,float16,fp8,31,0.018142400681972502
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,4,2,128,1,float16,float16,1023,0.08352000117301941
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,4,2,128,1,float16,fp8,7,0.01525920033454895
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,4,2,128,1,float16,fp8,1023,0.06628159880638122
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,4,2,128,1,float16,float16,4095,0.25714879035949706
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,4,2,128,1,float16,fp8,4095,0.17632479667663575
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,4,1,128,1,float16,float16,1,0.029235199093818665
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,4,1,128,1,float16,fp8,1,0.029161599278450013
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,4,1,128,1,float16,float16,3,0.029236799478530882
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,4,2,128,1,float16,float16,8191,0.4892896175384521
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,4,1,128,1,float16,fp8,3,0.028921601176261903
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,4,1,128,1,float16,fp8,7,0.029025599360466003
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,4,1,128,1,float16,float16,15,0.029209598898887634
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,4,1,128,1,float16,fp8,15,0.02893120050430298
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,4,1,128,1,float16,float16,31,0.029283198714256286
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,4,1,128,1,float16,fp8,31,0.02913439869880676
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,4,1,128,1,float16,float16,63,0.029478400945663452
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,4,1,128,1,float16,fp8,63,0.02909280061721802
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,4,2,128,1,float16,float16,3,0.023444800078868865
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,4,1,128,1,float16,float16,127,0.03052479922771454
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,4,1,128,1,float16,fp8,127,0.03265919983386993
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,4,1,128,1,float16,float16,255,0.041371199488639834
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,4,1,128,1,float16,fp8,255,0.036294400691986084
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,4,1,128,1,float16,float16,511,0.056734400987625125
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,4,1,128,1,float16,fp8,511,0.05088480114936829
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,4,1,128,1,float16,float16,1023,0.08971359729766845
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,4,2,128,1,float16,fp8,255,0.030697599053382874
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,4,1,128,1,float16,float16,2047,0.14758399724960328
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,4,1,128,1,float16,fp8,2047,0.10877759456634521
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,4,2,128,1,float16,float16,1,0.04014720022678375
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,4,1,128,1,float16,float16,4095,0.2630223989486694
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,4,2,128,1,float16,fp8,1,0.04018239974975586
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,4,1,128,1,float16,fp8,4095,0.18280320167541503
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,4,2,128,1,float16,fp8,3,0.04030239880084992
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,4,2,128,1,float16,float16,3,0.0398719996213913
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,4,2,128,1,float16,float16,7,0.04039359986782074
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,4,2,128,1,float16,fp8,7,0.040191999077796935
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,4,2,128,1,float16,float16,15,0.04030559957027435
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,4,2,128,1,float16,fp8,15,0.04046080112457275
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,4,2,128,1,float16,float16,31,0.04020319879055023
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,4,2,128,1,float16,fp8,31,0.04049600064754486
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,4,2,128,1,float16,float16,63,0.040670400857925414
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,4,2,128,1,float16,fp8,63,0.04051199853420258
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,4,2,128,1,float16,float16,127,0.0482448011636734
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,4,2,128,1,float16,fp8,127,0.047409600019454955
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,4,2,128,1,float16,float16,255,0.06385120153427123
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,4,2,128,1,float16,fp8,255,0.05748639702796936
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,4,2,128,1,float16,float16,511,0.09080320000648498
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,4,2,128,1,float16,fp8,511,0.08150720000267028
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,4,2,128,1,float16,float16,1023,0.15184320211410524
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,4,2,128,1,float16,fp8,1023,0.1166159987449646
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,4,1,128,1,float16,float16,7,0.0290367990732193
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,4,2,128,1,float16,float16,2047,0.2685120105743408
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,4,2,128,1,float16,fp8,2047,0.19494080543518066
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,1,0.009721600264310837
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,1,128,1,float16,fp8,1,0.009583999961614608
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,3,0.009782399982213974
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,1,128,1,float16,fp8,3,0.009616000205278396
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,4,2,128,1,float16,fp8,4095,0.34383840560913087
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,7,0.009719999879598618
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,15,0.009808000177145004
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,1,128,1,float16,fp8,7,0.009598399698734283
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,1,128,1,float16,fp8,15,0.009708800166845322
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,31,0.009780800342559815
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,63,0.009736000001430512
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,1,128,1,float16,fp8,63,0.009784000366926194
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,127,0.00995519980788231
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,1,128,1,float16,fp8,127,0.010351999849081039
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,255,0.011222399771213531
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,1,128,1,float16,fp8,255,0.011166399717330933
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,511,0.01324480026960373
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,1,128,1,float16,fp8,511,0.013230399787425995
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,1023,0.014499199390411378
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,1,128,1,float16,fp8,1023,0.01395999938249588
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,2047,0.015031999349594117
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,1,128,1,float16,fp8,2047,0.014716799557209014
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,4,2,128,1,float16,fp8,2047,0.10241440534591675
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,4095,0.015969599783420562
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,8191,0.019652800261974336
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,1,128,1,float16,fp8,4095,0.016484799981117248
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,1,128,1,float16,fp8,8191,0.021848000586032867
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,16383,0.03401280045509338
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,1,128,1,float16,fp8,16383,0.027315199375152588
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,32767,0.049262401461601255
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,1,128,1,float16,fp8,32767,0.038545599579811095
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,4,2,128,1,float16,fp8,8191,0.3316447973251343
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,1,128,1,float16,fp8,65535,0.05923839807510376
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,2,128,1,float16,fp8,1,0.009470400214195252
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,3,0.009576000273227692
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,2,128,1,float16,fp8,3,0.009412799775600434
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,7,0.009516800194978714
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,2,128,1,float16,fp8,7,0.009404800087213516
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,15,0.009625600278377533
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,2,128,1,float16,fp8,15,0.009548799693584442
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,31,0.009724800288677216
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,2,128,1,float16,fp8,31,0.009614399820566177
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,63,0.009875199943780898
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,1,128,1,float16,fp8,31,0.009771200269460678
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,2,128,1,float16,fp8,63,0.00960479974746704
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,2,128,1,float16,fp8,127,0.010239999741315842
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,255,0.010964799672365189
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,2,128,1,float16,fp8,255,0.011169599741697312
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,4,1,128,1,float16,fp8,1023,0.06938719749450684
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,511,0.01319040060043335
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,1023,0.015014399588108063
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,2,128,1,float16,fp8,1023,0.014588800072669984
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,2047,0.016120000183582305
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,2,128,1,float16,fp8,2047,0.01505119949579239
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,4095,0.01836320012807846
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,2,128,1,float16,fp8,4095,0.01902720034122467
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,8191,0.030836799740791322
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,4,2,128,1,float16,float16,2047,0.14145439863204956
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,2,128,1,float16,fp8,8191,0.02359360009431839
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,2,128,1,float16,fp8,16383,0.03484959900379181
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,16383,0.046193599700927734
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,32767,0.07636319994926452
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,65535,0.13700640201568604
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,65535,0.08035680055618286
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,2,128,1,float16,fp8,65535,0.09170560240745544
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,1,0.009918399900197983
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,4,1,128,1,float16,float16,1,0.04663200080394745
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,4,1,128,1,float16,fp8,1,0.046561598777770996
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,4,1,128,1,float16,float16,3,0.04684320092201233
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,4,1,128,1,float16,float16,7,0.04641439914703369
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,4,1,128,1,float16,fp8,7,0.04687840044498444
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,4,1,128,1,float16,float16,15,0.04658240079879761
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,4,2,128,1,float16,float16,4095,0.49668002128601074
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,4,1,128,1,float16,fp8,15,0.04680800139904022
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,4,1,128,1,float16,float16,63,0.04723680019378662
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,4,1,128,1,float16,fp8,63,0.047091200947761536
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,4,1,128,1,float16,float16,127,0.054451197385787964
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,4,1,128,1,float16,fp8,127,0.053724801540374754
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,4,1,128,1,float16,float16,255,0.06945919990539551
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,4,1,128,1,float16,float16,511,0.1016975998878479
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,2,128,1,float16,fp8,511,0.01356479972600937
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,4,1,128,1,float16,fp8,511,0.08863679766654968
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,4,1,128,1,float16,float16,1023,0.16123039722442628
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,4,1,128,1,float16,fp8,1023,0.12514079809188844
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,4,1,128,1,float16,float16,2047,0.2765568017959595
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,4,1,128,1,float16,fp8,2047,0.2037584066390991
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,4,2,128,1,float16,float16,1,0.06742720007896423
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,2,128,1,float16,fp8,32767,0.053518402576446536
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,4,2,128,1,float16,fp8,1,0.06850559711456299
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,4,2,128,1,float16,float16,3,0.0676479995250702
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,4,2,128,1,float16,fp8,3,0.06869120001792908
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,4,2,128,1,float16,float16,7,0.06810560226440429
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,4,2,128,1,float16,fp8,7,0.06792479753494263
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,4,2,128,1,float16,float16,15,0.0683456003665924
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,4,2,128,1,float16,fp8,15,0.06851999759674073
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,4,2,128,1,float16,float16,31,0.06773920059204101
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,4,2,128,1,float16,fp8,31,0.06841599941253662
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,4,2,128,1,float16,float16,63,0.07425280213356018
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,4,2,128,1,float16,float16,127,0.08061439990997314
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,4,2,128,1,float16,fp8,127,0.08680959939956664
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,4,1,128,1,float16,fp8,3,0.046639999747276305
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,4,2,128,1,float16,float16,255,0.10809439420700073
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,4,2,128,1,float16,fp8,255,0.10357919931411744
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,4,1,128,1,float16,float16,31,0.04650880098342895
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,4,2,128,1,float16,float16,511,0.1637231945991516
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,4,1,128,1,float16,fp8,31,0.04679999947547912
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,127,0.00990239977836609
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,4,2,128,1,float16,fp8,1023,0.22010719776153564
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,4,1,128,1,float16,fp8,255,0.06416959762573242
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,4,1,128,1,float16,float16,1,0.0788703978061676
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,4,1,128,1,float16,fp8,1,0.07878559827804565
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,4,2,128,1,float16,float16,2047,0.5079023838043213
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,4,1,128,1,float16,float16,3,0.07915359735488892
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,4,1,128,1,float16,fp8,3,0.07921119928359985
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,4,1,128,1,float16,fp8,7,0.0793615996837616
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,4,1,128,1,float16,float16,15,0.0782912015914917
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,4,1,128,1,float16,fp8,15,0.07947679758071899
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,4,1,128,1,float16,fp8,31,0.07943199872970581
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,4,1,128,1,float16,float16,63,0.08750079870223999
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,4,1,128,1,float16,fp8,63,0.07959039807319641
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,4,1,128,1,float16,float16,127,0.09195839762687683
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,4,1,128,1,float16,fp8,127,0.09854400157928467
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,4,1,128,1,float16,float16,255,0.11480000019073486
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,4,2,128,1,float16,fp8,63,0.06969280242919922
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,4,1,128,1,float16,fp8,255,0.1155135989189148
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,4,1,128,1,float16,float16,511,0.17154879570007325
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,4,1,128,1,float16,fp8,511,0.16243040561676025
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,4,2,128,1,float16,fp8,511,0.14900000095367433
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,4,1,128,1,float16,fp8,1023,0.2337712049484253
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,4,1,128,1,float16,float16,1023,0.29926719665527346
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,4,2,128,1,float16,float16,1,0.12067680358886719
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,4,2,128,1,float16,float16,1023,0.28467679023742676
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,4,2,128,1,float16,fp8,1,0.1212831974029541
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,4,2,128,1,float16,fp8,3,0.1210096001625061
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,4,2,128,1,float16,float16,7,0.11967040300369262
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,4,2,128,1,float16,fp8,7,0.12056479454040528
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,4,2,128,1,float16,float16,15,0.1204848051071167
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,4,2,128,1,float16,fp8,15,0.12090079784393311
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,4,1,128,1,float16,float16,7,0.07870240211486816
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,4,2,128,1,float16,fp8,31,0.1214400053024292
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,4,2,128,1,float16,float16,63,0.13313440084457398
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,4,1,128,1,float16,float16,31,0.07827360033988953
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,4,2,128,1,float16,fp8,63,0.13131999969482422
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,4,2,128,1,float16,float16,127,0.142467200756073
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,4,2,128,1,float16,fp8,127,0.15733120441436768
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,4,2,128,1,float16,float16,255,0.19046560525894166
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,4,2,128,1,float16,fp8,255,0.1879215955734253
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,4,2,128,1,float16,float16,511,0.3031264066696167
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,4,2,128,1,float16,fp8,511,0.2773744106292725
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,1,128,1,float16,fp8,1,0.009480000287294389
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,1,0.009683199971914292
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,3,0.009616000205278396
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,1,128,1,float16,fp8,3,0.009583999961614608
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,4,2,128,1,float16,fp8,1023,0.4157695770263672
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,1,128,1,float16,fp8,7,0.009614399820566177
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,1,128,1,float16,fp8,15,0.009607999771833419
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,15,0.009910400211811065
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,31,0.009771200269460678
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,1,128,1,float16,fp8,31,0.010126399993896484
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,1,128,1,float16,fp8,63,0.010179200023412705
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,127,0.010091199725866317
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,1,128,1,float16,fp8,127,0.010896000266075134
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,255,0.011078400164842605
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,1,128,1,float16,fp8,255,0.011515200138092041
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,511,0.013529600203037262
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,1023,0.015331199765205384
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,1,128,1,float16,fp8,511,0.013531200587749481
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,1,128,1,float16,fp8,1023,0.014827199280261993
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,2047,0.01634880006313324
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,1,128,1,float16,fp8,2047,0.015334400534629821
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,4095,0.018518400192260743
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,1,128,1,float16,fp8,4095,0.019337600469589232
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,4,2,128,1,float16,float16,3,0.11915040016174316
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,8191,0.031240001320838928
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,1,128,1,float16,fp8,8191,0.026732799410820008
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,16383,0.0487199991941452
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,32767,0.079312002658844
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,1,128,1,float16,fp8,32767,0.05682399868965149
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,65535,0.13991999626159668
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,1,0.010260800272226334
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,2,128,1,float16,fp8,1,0.010075200349092484
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,3,0.00997759997844696
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,2,128,1,float16,fp8,3,0.009761600196361542
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,7,0.009812799841165542
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,2,128,1,float16,fp8,7,0.009822399914264679
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,15,0.009969600290060044
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,2,128,1,float16,fp8,15,0.009958399832248688
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,31,0.009939199686050415
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,2,128,1,float16,fp8,31,0.009921599924564362
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,7,0.009566400200128555
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,63,0.010190399736166
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,2,128,1,float16,fp8,63,0.009803199768066406
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,127,0.010566399991512298
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,2,128,1,float16,fp8,127,0.011070399731397628
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,63,0.00987199991941452
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,511,0.013665600121021271
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,2,128,1,float16,fp8,511,0.014683200418949128
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,1023,0.016380800306797026
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,2,128,1,float16,fp8,1023,0.015123200416564942
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,2047,0.017385600507259368
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,2,128,1,float16,fp8,2047,0.01780800074338913
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,4095,0.02961759865283966
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,1,128,1,float16,fp8,16383,0.03750720024108887
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,2,128,1,float16,fp8,4095,0.02402399927377701
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,4,2,128,1,float16,fp8,2047,0.37477281093597414
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,8191,0.046367999911308286
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,1,128,1,float16,fp8,65535,0.09493119716644287
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,2,128,1,float16,fp8,8191,0.0362527996301651
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,16383,0.07588000297546386
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,2,128,1,float16,fp8,16383,0.05380799770355225
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,32767,0.13691359758377075
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,2,128,1,float16,fp8,32767,0.09142559766769409
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,1,0.011316800117492675
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,2,1,128,1,float16,fp8,1,0.01096000000834465
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,3,0.0110384002327919
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,2,1,128,1,float16,fp8,3,0.010985600203275681
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,7,0.011209599673748016
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,4,2,128,1,float16,float16,1023,0.5411359786987304
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,2,1,128,1,float16,fp8,7,0.011169599741697312
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,2,1,128,1,float16,fp8,15,0.011105599999427795
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,15,0.011099199950695037
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,31,0.011169599741697312
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,255,0.011657600104808808
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,63,0.011166399717330933
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,2,128,1,float16,fp8,255,0.011403200030326844
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,2,1,128,1,float16,fp8,63,0.01117599979043007
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,127,0.011316800117492675
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,2,1,128,1,float16,fp8,127,0.01239679977297783
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,511,0.014820800721645355
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,2,1,128,1,float16,fp8,511,0.02106879949569702
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,1023,0.019198399782180787
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,2,1,128,1,float16,fp8,1023,0.020137600600719452
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,2047,0.030905601382255555
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,2,1,128,1,float16,fp8,2047,0.02815999984741211
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,4095,0.0463919997215271
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,2,1,128,1,float16,fp8,4095,0.03726080060005188
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,4,2,128,1,float16,float16,31,0.12368639707565307
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,8191,0.0773855984210968
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,2,1,128,1,float16,fp8,8191,0.060203200578689574
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,16383,0.139900803565979
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,2,1,128,1,float16,fp8,16383,0.09747999906539917
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,32767,0.2582047939300537
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,65535,0.25556640625
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,4,2,128,1,float16,fp8,65535,0.1672320008277893
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,2,1,128,1,float16,fp8,32767,0.1739456057548523
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,2,1,128,1,float16,fp8,31,0.011134400218725204
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,2,1,128,1,float16,fp8,65535,0.3233520030975342
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,1,0.009436800330877303
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,65535,0.49497280120849607
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,255,0.012958399951457977
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,3,0.009433600306510925
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,2,1,128,1,float16,fp8,1,0.009064000099897385
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,7,0.009644799679517747
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,2,1,128,1,float16,fp8,7,0.0094480000436306
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,15,0.009480000287294389
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,2,1,128,1,float16,fp8,15,0.00947519987821579
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,31,0.009415999799966813
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,2,1,128,1,float16,fp8,31,0.009427200257778167
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,2,1,128,1,float16,fp8,131071,0.6270976066589355
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,2,1,128,1,float16,fp8,63,0.009174399822950364
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,127,0.009515199810266495
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,255,0.010608000308275222
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,131071,0.9661135673522949
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,2,1,128,1,float16,fp8,255,0.01091040000319481
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,511,0.012775999307632447
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,2,1,128,1,float16,fp8,127,0.00997920036315918
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,1023,0.01401440054178238
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,2,1,128,1,float16,fp8,1023,0.013488000631332398
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,2047,0.014351999759674073
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,2,1,128,1,float16,fp8,2047,0.013838399946689606
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,4095,0.014393599331378936
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,2,1,128,1,float16,fp8,4095,0.015587200224399567
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,8191,0.0162992000579834
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,2,1,128,1,float16,fp8,8191,0.018911999464035035
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,2,1,128,1,float16,fp8,16383,0.01918880045413971
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,32767,0.020865599811077117
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,2,1,128,1,float16,fp8,32767,0.020996800065040587
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,65535,0.02381120026111603
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,2,1,128,1,float16,fp8,65535,0.023710399866104126
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,131071,0.036078399419784545
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,2,1,128,1,float16,fp8,131071,0.029688000679016113
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,1,0.009436800330877303
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,2,1,128,1,float16,fp8,1,0.009286399930715561
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,3,0.009537599980831146
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,2,1,128,1,float16,fp8,3,0.009283199906349182
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,7,0.009547200053930283
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,2,1,128,1,float16,fp8,7,0.009272000193595887
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,15,0.009707199782133103
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,31,0.009455999732017517
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,2,1,128,1,float16,fp8,31,0.009460800141096116
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,2,1,128,1,float16,fp8,63,0.009540800005197525
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,127,0.009679999947547913
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,2,1,128,1,float16,fp8,127,0.010196799784898758
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,255,0.010847999900579452
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,2,1,128,1,float16,fp8,255,0.011121600121259689
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,511,0.013131199777126313
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,2,1,128,1,float16,fp8,511,0.013147200644016265
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,1023,0.01449120044708252
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,2,1,128,1,float16,fp8,1023,0.013806399703025819
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,2047,0.014324800670146942
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,2,1,128,1,float16,fp8,2047,0.0137472003698349
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,4095,0.014737600088119506
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,2,1,128,1,float16,fp8,3,0.009470400214195252
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,2,1,128,1,float16,fp8,4095,0.01589920073747635
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,8191,0.016803200542926788
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,2,1,128,1,float16,fp8,8191,0.01915999948978424
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,16383,0.020975999534130096
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,2,1,128,1,float16,fp8,16383,0.020076799392700195
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,2,1,128,1,float16,fp8,511,0.012904000282287598
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,32767,0.023526400327682495
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,16383,0.020027199387550355
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,2,1,128,1,float16,fp8,32767,0.022752000391483305
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,2,1,128,1,float16,fp8,65535,0.028439998626708984
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,131071,0.0506816029548645
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,1,0.010487999767065048
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,2,1,128,1,float16,fp8,131071,0.03927519917488098
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,2,1,128,1,float16,fp8,1,0.010476800054311753
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,3,0.010780800133943558
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,2,1,128,1,float16,fp8,3,0.010595200210809707
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,7,0.01069760024547577
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,2,1,128,1,float16,fp8,7,0.010659199953079224
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,15,0.010667199641466141
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,2,1,128,1,float16,fp8,15,0.0106175996363163
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,31,0.010583999752998351
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,2,1,128,1,float16,fp8,31,0.010654400289058685
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,63,0.010335999727249145
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,2,1,128,1,float16,fp8,63,0.01058719977736473
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,127,0.010862399637699128
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,2,1,128,1,float16,fp8,127,0.011486399918794632
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,255,0.01250080019235611
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,2,1,128,1,float16,fp8,255,0.012163200229406358
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,511,0.014424000680446625
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,2,1,128,1,float16,fp8,511,0.01533920019865036
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,1023,0.017140799760818483
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,2,1,128,1,float16,fp8,1023,0.01643519997596741
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,2047,0.01847040057182312
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,2,1,128,1,float16,fp8,2047,0.01905920058488846
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,4095,0.03044799864292145
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,2,1,128,1,float16,fp8,4095,0.024432000517845155
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,8191,0.046830400824546814
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,2,1,128,1,float16,fp8,8191,0.038924801349639895
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,16383,0.08080480098724366
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,64,1,2,1,128,1,float16,fp8,255,0.012912000715732574
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,32767,0.14036480188369752
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,63,0.009569600224494934
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,65535,0.2575632095336914
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,2,1,128,1,float16,fp8,65535,0.1728368043899536
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,65535,0.03505600094795227
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,1,0.009521599858999252
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,2,1,128,1,float16,fp8,1,0.00947680026292801
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,3,0.009593600034713745
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,2,1,128,1,float16,fp8,3,0.009374400228261947
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,7,0.009414400160312652
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,2,1,128,1,float16,fp8,15,0.009305600076913834
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,15,0.009571199864149093
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,31,0.009564799815416336
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,2,1,128,1,float16,fp8,15,0.009281600266695023
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,2,1,128,1,float16,fp8,31,0.009335999935865402
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,63,0.009576000273227692
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,2,1,128,1,float16,fp8,63,0.009299200028181076
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,127,0.00963359996676445
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,2,1,128,1,float16,fp8,127,0.010195200145244599
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,255,0.010795199871063232
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,2,1,128,1,float16,fp8,255,0.011116799712181092
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,2,1,128,1,float16,fp8,511,0.013462400436401368
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,1023,0.014638400077819825
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,2,1,128,1,float16,fp8,1023,0.014134399592876434
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,2047,0.014801600575447082
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,2,1,128,1,float16,fp8,2047,0.014262400567531586
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,4095,0.015211200714111328
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,2,1,128,1,float16,fp8,16383,0.057822400331497194
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,2,1,128,1,float16,fp8,4095,0.016190400719642638
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,2,1,128,1,float16,fp8,8191,0.020102399587631225
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,16383,0.023255999386310577
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,2,1,128,1,float16,fp8,32767,0.09667040109634399
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,2,1,128,1,float16,fp8,16383,0.022784000635147093
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,2,1,128,1,float16,fp8,32767,0.028249600529670717
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,65535,0.05093280076980591
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,2,1,128,1,float16,fp8,65535,0.038955199718475345
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,131071,0.08291360139846801
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,2,1,128,1,float16,fp8,131071,0.05909759998321533
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,1,0.012908799946308136
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,2,1,128,1,float16,fp8,131071,0.3239487886428833
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,2,1,128,1,float16,fp8,1,0.012886400520801543
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,131071,0.49918560981750487
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,2,1,128,1,float16,fp8,3,0.012838399410247803
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,3,0.012531200051307678
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,2,1,128,1,float16,fp8,7,0.01284479945898056
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,15,0.012617599964141846
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,7,0.012577599287033081
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,2,1,128,1,float16,fp8,15,0.01313599944114685
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,31,0.012742400169372559
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,2,1,128,1,float16,fp8,7,0.009516800194978714
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,2,1,128,1,float16,fp8,31,0.01255040019750595
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,63,0.013067199289798737
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,2,1,128,1,float16,fp8,63,0.012604799866676331
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,127,0.013006399571895599
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,255,0.015243199467658997
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,2,1,128,1,float16,fp8,127,0.01388320028781891
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,2,1,128,1,float16,fp8,255,0.014407999813556671
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,511,0.01666239947080612
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,1023,0.03052319884300232
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,511,0.012809599936008453
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,2,1,128,1,float16,fp8,1023,0.02285439968109131
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,2047,0.04505600035190582
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,2,1,128,1,float16,fp8,2047,0.03397440016269684
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,4095,0.07517439723014832
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,8191,0.13540799617767335
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,2,1,128,1,float16,fp8,8191,0.09179679751396179
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,8191,0.01777600049972534
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,16383,0.25130720138549806
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,32767,0.03483040034770966
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,32767,0.4890944004058838
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,2,1,128,1,float16,fp8,32767,0.3176800012588501
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,2,1,128,1,float16,float16,1,0.017718400061130523
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,2,1,128,1,float16,fp8,1,0.01770080029964447
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,2,1,128,1,float16,float16,3,0.017745600640773775
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,65535,0.9714303970336914
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,2,1,128,1,float16,fp8,3,0.017662400007247926
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,63,0.009603200107812881
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,2,1,128,1,float16,fp8,65535,0.6184624195098877
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,2,1,128,1,float16,float16,7,0.017668800055980684
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,2,1,128,1,float16,float16,15,0.017854399979114532
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,2,1,128,1,float16,fp8,31,0.017712000012397765
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,2,1,128,1,float16,fp8,511,0.018505600094795228
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,2,1,128,1,float16,float16,63,0.017859199643135072
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,2,1,128,1,float16,fp8,63,0.017983999848365784
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,2,1,128,1,float16,float16,127,0.018137599527835845
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,2,1,128,1,float16,fp8,127,0.019551999866962433
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,2,1,128,1,float16,float16,255,0.02051199972629547
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,2,1,128,1,float16,fp8,255,0.021331200003623964
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,2,1,128,1,float16,float16,511,0.03290719985961914
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,2,1,128,1,float16,fp8,4095,0.05337439775466919
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,2,1,128,1,float16,fp8,511,0.028182399272918702
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,2,1,128,1,float16,float16,1023,0.05055040121078491
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,2,1,128,1,float16,fp8,1023,0.03901599943637848
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,2,1,128,1,float16,float16,2047,0.07941600084304809
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,2,1,128,1,float16,fp8,2047,0.05907999873161316
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,2,1,128,1,float16,float16,4095,0.1378991961479187
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,2,1,128,1,float16,fp8,4095,0.09580320119857788
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,128,1,2,1,128,1,float16,fp8,16383,0.1671679973602295
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,2,1,128,1,float16,float16,8191,0.2583071947097778
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,2,1,128,1,float16,fp8,8191,0.1738752007484436
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,2,1,128,1,float16,float16,1,0.02930560111999512
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,2,1,128,1,float16,fp8,1,0.02922239899635315
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,2,1,128,1,float16,float16,3,0.029271999001502992
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,2,1,128,1,float16,float16,16383,0.49317121505737305
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,2,1,128,1,float16,float16,7,0.029252800345420837
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,2,1,128,1,float16,fp8,16383,0.3227152109146118
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,2,1,128,1,float16,float16,15,0.02884320020675659
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,2,1,128,1,float16,fp8,7,0.02911520004272461
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,2,1,128,1,float16,float16,31,0.028934401273727418
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,2,1,128,1,float16,fp8,15,0.02879520058631897
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,2,1,128,1,float16,fp8,31,0.02895359992980957
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,2,1,128,1,float16,fp8,63,0.02919999957084656
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,2,1,128,1,float16,float16,127,0.02940639853477478
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,2,1,128,1,float16,fp8,127,0.03267039954662323
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,2,1,128,1,float16,float16,255,0.042161598801612854
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,2,1,128,1,float16,fp8,255,0.036929601430892946
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,2,1,128,1,float16,float16,511,0.05652160048484802
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,2,1,128,1,float16,fp8,7,0.017635199427604675
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,2,1,128,1,float16,float16,1023,0.08907520174980163
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,2,1,128,1,float16,fp8,1023,0.06882240176200867
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,2,1,128,1,float16,fp8,15,0.017705599963665008
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,256,1,2,1,128,1,float16,float16,31,0.017979200184345245
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,2,1,128,1,float16,fp8,2047,0.10837279558181763
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,2,1,128,1,float16,float16,2047,0.14758399724960328
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,2,1,128,1,float16,fp8,4095,0.18196640014648438
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,1,0.009998399764299393
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,3,0.009812799841165542
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,2,1,128,1,float16,fp8,1,0.009591999650001525
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,2,1,128,1,float16,fp8,3,0.009753599762916565
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,2,1,128,1,float16,fp8,8191,0.333353590965271
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,2,1,128,1,float16,float16,8191,0.49416799545288087
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,2,1,128,1,float16,fp8,7,0.009729599952697754
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,31,0.00989760011434555
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,15,0.009747199714183807
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,2,1,128,1,float16,fp8,15,0.009814400225877762
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,2,1,128,1,float16,fp8,31,0.009779199957847595
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,63,0.009889599680900574
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,127,0.009976000338792802
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,2,1,128,1,float16,fp8,63,0.009811200201511383
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,2,1,128,1,float16,fp8,255,0.011292800307273865
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,2,1,128,1,float16,fp8,3,0.029287999868392943
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,511,0.013359999656677246
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,2,1,128,1,float16,fp8,511,0.013961599767208099
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,1023,0.014788800477981567
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,2047,0.015248000621795654
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,2,1,128,1,float16,float16,63,0.029044800996780397
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,2,1,128,1,float16,fp8,2047,0.014950400590896607
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,4095,0.01642719954252243
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,8191,0.02032800018787384
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,2,1,128,1,float16,fp8,8191,0.022859199345111846
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,2,1,128,1,float16,fp8,511,0.05108159780502319
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,16383,0.034436801075935365
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,2,1,128,1,float16,fp8,16383,0.028360000252723692
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,32767,0.0508575975894928
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,2,1,128,1,float16,fp8,32767,0.03933599889278412
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,65535,0.08244799971580505
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,2,1,128,1,float16,fp8,65535,0.0596127986907959
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,2,1,128,1,float16,fp8,131071,0.10024479627609253
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,512,1,2,1,128,1,float16,float16,4095,0.263155198097229
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,2,1,128,1,float16,float16,1,0.046035200357437134
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,2,1,128,1,float16,fp8,1,0.04613919854164124
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,2,1,128,1,float16,float16,3,0.046096000075340274
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,2,1,128,1,float16,fp8,3,0.04628159999847412
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,2,1,128,1,float16,float16,7,0.04658240079879761
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,2,1,128,1,float16,fp8,7,0.0465472012758255
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,2,1,128,1,float16,float16,15,0.046768000721931456
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,2,1,128,1,float16,fp8,15,0.04635039865970612
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,2,1,128,1,float16,float16,31,0.04652479887008667
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,2,1,128,1,float16,fp8,127,0.010463999956846238
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,2,1,128,1,float16,fp8,31,0.046348801255226134
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,255,0.011323200166225433
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,2,1,128,1,float16,float16,63,0.04773600101470947
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,2,1,128,1,float16,fp8,63,0.04659999907016754
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,2,1,128,1,float16,float16,127,0.05400320291519165
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,2,1,128,1,float16,fp8,127,0.053969597816467284
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,2,1,128,1,float16,float16,255,0.06954879760742187
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,2,1,128,1,float16,fp8,1023,0.014406399428844452
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,2,1,128,1,float16,float16,511,0.09611200094223023
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,2,1,128,1,float16,fp8,511,0.08833600282669067
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,2,1,128,1,float16,fp8,4095,0.01688160002231598
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,2,1,128,1,float16,fp8,1023,0.12447999715805054
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,2,1,128,1,float16,float16,1023,0.16079519987106322
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,2,1,128,1,float16,float16,2047,0.27503039836883547
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,2,1,128,1,float16,fp8,2047,0.2035759925842285
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,2,1,128,1,float16,float16,1,0.07787839770317077
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,2,1,128,1,float16,fp8,1,0.07901120185852051
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,2,1,128,1,float16,float16,3,0.07867519855499268
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,2,1,128,1,float16,float16,4095,0.5026703834533691
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,2,1,128,1,float16,float16,7,0.07872480154037476
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,2,1,128,1,float16,fp8,3,0.07885280251502991
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,2,1,128,1,float16,fp8,7,0.07853279709815979
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,2,1,128,1,float16,fp8,15,0.07957119941711426
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,2,1,128,1,float16,float16,31,0.07904639840126038
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,2,1,128,1,float16,fp8,31,0.07959840297698975
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,2,1,128,1,float16,fp8,63,0.07967519760131836
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,7,0.009848000109195709
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,2,1,128,1,float16,float16,127,0.09211199879646301
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,2,1,128,1,float16,fp8,127,0.09824640154838563
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,2,1,128,1,float16,fp8,255,0.11509120464324951
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,2,1,128,1,float16,float16,511,0.1741039991378784
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,2,1,128,1,float16,fp8,511,0.16147680282592775
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,2,1,128,1,float16,fp8,255,0.06442720293998719
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,2,1,128,1,float16,float16,1023,0.2991983890533447
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,1024,1,2,1,128,1,float16,fp8,4095,0.3508944034576416
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,2,1,128,1,float16,float16,2047,0.5211071968078613
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,1,0.009507200121879578
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,131071,0.1411919951438904
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,2,1,128,1,float16,fp8,1,0.009849599748849868
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,3,0.00958240032196045
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,2,1,128,1,float16,fp8,3,0.009612800180912017
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,7,0.009808000177145004
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,2,1,128,1,float16,fp8,7,0.009708800166845322
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,15,0.009799999743700027
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,2,1,128,1,float16,fp8,2047,0.39209280014038084
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,2,1,128,1,float16,fp8,15,0.00957920029759407
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,2,1,128,1,float16,float16,63,0.08647040128707886
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,31,0.009646400064229965
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,63,0.009785600006580353
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,127,0.00963039994239807
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,2,1,128,1,float16,fp8,63,0.009886399656534196
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,255,0.010828799754381179
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,2,1,128,1,float16,fp8,255,0.011716800183057785
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,511,0.013120000064373017
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,2,1,128,1,float16,float16,255,0.11570719480514527
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,1023,0.015300799906253815
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,2047,0.016251200437545778
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,2,1,128,1,float16,fp8,1023,0.014843200147151948
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,2,1,128,1,float16,fp8,2047,0.01523679941892624
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,4095,0.018452799320220946
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,2,1,128,1,float16,fp8,4095,0.019444799423217772
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,8191,0.03127999901771546
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,16383,0.05004799962043762
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,2,1,128,1,float16,fp8,8191,0.02728480100631714
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,32767,0.07942879796028138
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,2,1,128,1,float16,fp8,32767,0.057392001152038574
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,2,1,128,1,float16,fp8,65535,0.09542080163955688
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,65535,0.13987200260162352
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,2,1,128,1,float16,fp8,1023,0.23222720623016357
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,2,1,128,1,float16,fp8,131071,0.1723471999168396
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,131071,0.2571376085281372
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,2048,1,2,1,128,1,float16,float16,15,0.07880480289459228
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,2,1,128,1,float16,fp8,127,0.010655999928712846
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,2,1,128,1,float16,fp8,31,0.009617599844932555
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,2,1,128,1,float16,fp8,511,0.013851200044155122
SGLang,0.5.10,NVIDIA H200,generation_attention,flash_attention,16,1,2,1,128,1,float16,fp8,16383,0.03878560066223145
