framework,version,device,op_name,kernel_source,batch_size,isl,num_heads,num_key_value_heads,head_dim,beam_width,attn_dtype,kv_cache_dtype,step,latency
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,64,128,1,float16,float16,1,0.12829600572586058
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,64,128,1,float16,fp8,1,0.1561951994895935
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,64,128,1,float16,float16,3,0.12855520248413085
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,64,128,1,float16,fp8,3,0.1562831997871399
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,64,128,1,float16,float16,7,0.12900639772415162
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,64,128,1,float16,fp8,7,0.15638240575790405
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,64,128,1,float16,float16,15,0.12902560234069824
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,64,128,1,float16,fp8,15,0.15651040077209472
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,64,128,1,float16,float16,31,0.13515360355377198
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,64,128,1,float16,fp8,31,0.1571071982383728
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,64,128,1,float16,float16,63,0.13843200206756592
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,64,128,1,float16,fp8,63,0.1659216046333313
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,64,128,1,float16,float16,127,0.14246879816055297
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,64,128,1,float16,fp8,127,0.17052160501480101
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,64,128,1,float16,float16,255,0.19781440496444702
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,64,128,1,float16,fp8,255,0.22271039485931396
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,64,128,1,float16,float16,511,0.30339200496673585
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,64,128,1,float16,fp8,511,0.3391551971435547
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,64,128,1,float16,float16,1023,0.5426447868347168
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,64,128,1,float16,fp8,1023,0.5254432201385498
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,64,128,1,float16,float16,2047,1.0335344314575194
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,64,128,1,float16,fp8,2047,0.979032039642334
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,64,128,1,float16,float16,1,0.013993600010871887
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,64,128,1,float16,fp8,1,0.014051200449466705
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,64,128,1,float16,float16,3,0.01387840062379837
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,64,128,1,float16,fp8,3,0.014073599874973298
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,64,128,1,float16,float16,7,0.013870400190353394
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,64,128,1,float16,fp8,7,0.014056000113487243
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,64,128,1,float16,float16,15,0.014097599685192109
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,64,128,1,float16,fp8,15,0.014112000167369843
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,64,128,1,float16,float16,31,0.014056000113487243
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,64,128,1,float16,fp8,31,0.014153599739074707
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,64,128,1,float16,float16,63,0.013991999626159667
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,64,128,1,float16,fp8,63,0.01398559957742691
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,64,128,1,float16,float16,127,0.013967999815940857
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,64,128,1,float16,fp8,127,0.013532799482345582
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,64,128,1,float16,float16,255,0.015270400047302245
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,64,128,1,float16,fp8,255,0.015348799526691437
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,64,128,1,float16,float16,511,0.01841759979724884
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,64,128,1,float16,fp8,511,0.01923840045928955
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,64,128,1,float16,float16,1023,0.021321600675582884
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,64,128,1,float16,fp8,1023,0.021425600349903106
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,64,128,1,float16,float16,2047,0.033406400680541994
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,64,128,1,float16,fp8,2047,0.027809599041938783
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,64,128,1,float16,float16,1,0.013583999872207642
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,64,128,1,float16,fp8,1,0.01361439973115921
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,64,128,1,float16,float16,3,0.013575999438762665
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,64,128,1,float16,fp8,3,0.01371999979019165
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,64,128,1,float16,float16,7,0.013436800241470337
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,64,128,1,float16,fp8,7,0.013507199287414551
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,64,128,1,float16,float16,15,0.013382400572299957
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,64,128,1,float16,fp8,15,0.013577599823474885
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,64,128,1,float16,float16,31,0.013513599336147309
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,64,128,1,float16,fp8,31,0.013665600121021271
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,64,128,1,float16,float16,63,0.013708800077438354
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,64,128,1,float16,fp8,63,0.013784000277519226
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,64,128,1,float16,float16,127,0.013777600228786468
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,64,128,1,float16,fp8,127,0.013780799508094788
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,64,128,1,float16,float16,255,0.015491199493408204
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,64,128,1,float16,fp8,255,0.01555040031671524
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,64,128,1,float16,float16,511,0.019223999977111817
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,64,128,1,float16,fp8,511,0.019251200556755065
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,64,128,1,float16,float16,1023,0.03236640095710754
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,64,128,1,float16,fp8,1023,0.026662400364875792
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,64,128,1,float16,float16,2047,0.047295999526977536
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,64,128,1,float16,fp8,2047,0.04224160015583038
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,64,128,1,float16,float16,1,0.07103999853134155
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,64,128,1,float16,fp8,1,0.08287360072135926
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,64,128,1,float16,float16,3,0.07106239795684814
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,64,128,1,float16,float16,7,0.07123519778251648
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,64,128,1,float16,fp8,7,0.08320479989051818
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,64,128,1,float16,fp8,15,0.0832751989364624
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,64,128,1,float16,float16,63,0.0777296006679535
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,64,128,1,float16,float16,255,0.10871039628982544
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,64,128,1,float16,fp8,255,0.11947040557861328
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,64,128,1,float16,float16,511,0.1637071967124939
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,64,128,1,float16,fp8,511,0.17843199968338014
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,64,128,1,float16,float16,1023,0.2857680082321167
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,64,128,1,float16,fp8,1023,0.27506399154663086
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,64,128,1,float16,fp8,3,0.08240479826927186
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,64,128,1,float16,float16,2047,0.5286896228790283
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,64,128,1,float16,float16,15,0.07052000164985657
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,64,128,1,float16,float16,31,0.07128639817237854
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,64,128,1,float16,fp8,2047,0.4544112205505371
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,64,128,1,float16,float16,127,0.07917919754981995
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,64,128,1,float16,float16,3,0.01725600063800812
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,64,128,1,float16,fp8,3,0.017833599448204042
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,64,128,1,float16,fp8,1,0.0176816001534462
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,64,128,1,float16,fp8,15,0.0178399994969368
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,64,128,1,float16,fp8,7,0.0178384006023407
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,64,128,1,float16,float16,31,0.01735360026359558
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,64,128,1,float16,float16,15,0.01738400012254715
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,64,128,1,float16,float16,63,0.01743520051240921
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,64,128,1,float16,fp8,63,0.01800000071525574
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,64,128,1,float16,fp8,127,0.018049600720405578
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,64,128,1,float16,float16,255,0.021320000290870667
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,64,128,1,float16,fp8,255,0.02205280065536499
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,64,128,1,float16,float16,511,0.03311040103435516
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,64,128,1,float16,fp8,511,0.02940160036087036
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,64,128,1,float16,float16,1023,0.04969919919967651
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,64,128,1,float16,float16,1,0.017308799922466277
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,64,128,1,float16,fp8,127,0.09038079977035522
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,64,128,1,float16,fp8,63,0.08273599743843078
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,64,128,1,float16,fp8,1023,0.046166399121284486
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,64,128,1,float16,float16,7,0.017156800627708434
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,64,128,1,float16,float16,1,0.24174399375915528
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,64,128,1,float16,fp8,2047,0.06899039745330811
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,64,128,1,float16,float16,2047,0.08034560084342957
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,64,128,1,float16,float16,127,0.0176256000995636
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,64,128,1,float16,float16,3,0.24185121059417725
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,64,128,1,float16,fp8,1,0.2981856107711792
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,64,128,1,float16,fp8,3,0.2966288089752197
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,64,128,1,float16,float16,7,0.24145278930664063
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,64,128,1,float16,float16,15,0.243887996673584
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,64,128,1,float16,fp8,7,0.2984224081039429
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,64,128,1,float16,fp8,15,0.2981247901916504
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,64,128,1,float16,fp8,31,0.3044831991195679
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,64,128,1,float16,float16,63,0.25857279300689695
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,64,128,1,float16,float16,31,0.2547008037567139
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,64,128,1,float16,fp8,63,0.3142607927322388
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,64,128,1,float16,float16,127,0.26337759494781493
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,64,128,1,float16,fp8,127,0.3199968099594116
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,64,128,1,float16,float16,255,0.3762207984924316
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,64,128,1,float16,fp8,255,0.42225117683410646
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,64,128,1,float16,float16,511,0.5900112152099609
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,64,128,1,float16,fp8,511,0.6482895851135254
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,64,128,1,float16,float16,1,0.46277918815612795
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,64,128,1,float16,fp8,1,0.582151985168457
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,64,128,1,float16,float16,3,0.4626607894897461
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,64,128,1,float16,fp8,3,0.5805935859680176
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,64,128,1,float16,fp8,1023,1.0301775932312012
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,64,128,1,float16,float16,7,0.46623520851135253
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,64,128,1,float16,fp8,7,0.5813983917236328
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,64,128,1,float16,float16,15,0.48880481719970703
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,64,128,1,float16,fp8,15,0.5762991905212402
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,64,128,1,float16,float16,31,0.4889791965484619
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,64,128,1,float16,float16,63,0.4949967861175537
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,64,128,1,float16,fp8,31,0.6097487926483154
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,64,128,1,float16,fp8,63,0.6122943878173828
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,64,128,1,float16,float16,127,0.5083471775054932
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,64,128,1,float16,fp8,127,0.6214623928070069
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,64,128,1,float16,float16,255,0.7261663913726807
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,64,128,1,float16,fp8,255,0.8317232131958008
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,64,64,128,1,float16,fp8,1,1.1373935699462892
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,64,64,128,1,float16,float16,3,0.9546832084655762
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,64,64,128,1,float16,fp8,3,1.1305184364318848
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,64,128,1,float16,fp8,31,0.08296319842338562
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,64,64,128,1,float16,float16,7,0.9593359947204589
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,64,128,1,float16,fp8,31,0.01785759925842285
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,64,64,128,1,float16,float16,15,0.9597104072570801
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,64,64,128,1,float16,fp8,7,1.1285167694091798
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,64,64,128,1,float16,fp8,15,1.1956928253173829
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,64,64,128,1,float16,float16,31,0.9616671562194824
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,64,128,1,float16,float16,1,0.025385600328445435
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,64,64,128,1,float16,fp8,31,1.2044256210327149
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,64,128,1,float16,fp8,1,0.02729920148849487
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,64,64,128,1,float16,float16,63,0.9736016273498536
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,64,128,1,float16,float16,3,0.024942399561405183
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,64,128,1,float16,fp8,7,0.027116799354553224
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,64,128,1,float16,float16,15,0.025006398558616638
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,64,64,128,1,float16,float16,127,1.000990390777588
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,64,64,128,1,float16,fp8,63,1.2039664268493653
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,64,128,1,float16,fp8,15,0.027118399739265442
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,64,128,1,float16,float16,63,0.025249600410461426
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,64,128,1,float16,fp8,31,0.02731199860572815
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,64,128,1,float16,fp8,63,0.027188798785209654
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,64,128,1,float16,float16,127,0.02560639977455139
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,64,128,1,float16,fp8,127,0.0273360013961792
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,64,64,128,1,float16,fp8,127,1.2287936210632324
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,64,128,1,float16,float16,255,0.038867199420928956
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,64,128,1,float16,fp8,255,0.035867199301719666
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,64,128,1,float16,fp8,511,0.05303199887275696
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,64,128,1,float16,float16,511,0.055615997314453124
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,64,128,1,float16,float16,1023,0.0858560025691986
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,64,128,1,float16,float16,2047,0.14592959880828857
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,64,128,1,float16,fp8,2047,0.1250815987586975
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,64,128,1,float16,fp8,1023,0.0785968005657196
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,64,128,1,float16,float16,1023,1.107316780090332
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,64,64,128,1,float16,float16,1,1.900604820251465
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,64,128,1,float16,fp8,3,0.027297601103782654
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,64,128,1,float16,float16,7,0.02545279860496521
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,64,64,128,1,float16,fp8,1,2.37542724609375
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,64,128,1,float16,float16,31,0.025470399856567384
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,64,64,128,1,float16,float16,1,0.9511024475097656
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,64,64,128,1,float16,float16,7,1.8986928939819336
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,64,64,128,1,float16,float16,15,1.900187110900879
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,64,64,128,1,float16,fp8,7,2.378232002258301
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,64,64,128,1,float16,float16,31,1.9045215606689454
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,64,64,128,1,float16,fp8,15,2.375819206237793
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,64,64,128,1,float16,fp8,31,2.3760976791381836
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,64,64,128,1,float16,float16,63,1.9194768905639648
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,64,64,128,1,float16,fp8,3,2.4010223388671874
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,64,64,128,1,float16,fp8,63,2.3797183990478517
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,64,64,128,1,float16,float16,1,3.7830577850341798
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,64,64,128,1,float16,float16,3,3.7769840240478514
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,64,64,128,1,float16,float16,3,1.8984495162963868
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,64,64,128,1,float16,fp8,1,4.735190582275391
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,64,128,1,float16,float16,1,0.040007999539375304
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,64,128,1,float16,fp8,1,0.04535039961338043
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,64,64,128,1,float16,float16,7,3.776030349731445
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,64,128,1,float16,float16,3,0.03988800048828125
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,64,128,1,float16,fp8,3,0.04529440104961395
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,64,64,128,1,float16,fp8,3,4.729868698120117
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,64,128,1,float16,float16,7,0.04004479944705963
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,64,128,1,float16,fp8,7,0.045500800013542175
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,64,128,1,float16,fp8,15,0.0457504004240036
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,64,64,128,1,float16,float16,15,3.7836734771728517
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,64,128,1,float16,float16,31,0.0405023992061615
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,64,128,1,float16,float16,63,0.041310399770736694
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,64,64,128,1,float16,fp8,7,4.740643310546875
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,64,128,1,float16,fp8,63,0.04564639925956726
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,64,128,1,float16,float16,127,0.046916800737380984
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,64,128,1,float16,fp8,127,0.045817598700523376
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,64,128,1,float16,float16,255,0.06241120100021362
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,64,128,1,float16,fp8,255,0.06489120125770569
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,64,128,1,float16,float16,511,0.09090719819068908
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,64,128,1,float16,fp8,511,0.09419999718666076
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,64,128,1,float16,float16,1023,0.1556831955909729
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,64,128,1,float16,fp8,1023,0.14300160408020018
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,64,128,1,float16,float16,2047,0.2787919998168945
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,64,64,128,1,float16,float16,31,3.790542221069336
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,64,128,1,float16,fp8,2047,0.2346208095550537
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,48,48,128,1,float16,float16,1,0.09905760288238526
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,64,64,128,1,float16,fp8,15,4.707932662963867
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,48,48,128,1,float16,float16,3,0.0993936002254486
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,48,48,128,1,float16,fp8,3,0.11809600591659546
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,48,48,128,1,float16,float16,7,0.09941759705543518
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,48,48,128,1,float16,float16,15,0.09938880205154418
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,48,48,128,1,float16,fp8,7,0.11896159648895263
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,48,48,128,1,float16,float16,31,0.10035840272903443
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,48,48,128,1,float16,fp8,31,0.11968159675598145
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,48,48,128,1,float16,float16,63,0.10653760433197021
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,64,64,128,1,float16,fp8,31,4.692111968994141
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,48,48,128,1,float16,fp8,63,0.12671200037002564
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,48,48,128,1,float16,float16,127,0.10909119844436646
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,48,48,128,1,float16,float16,255,0.15151840448379517
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,48,48,128,1,float16,fp8,255,0.16928960084915162
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,48,48,128,1,float16,fp8,127,0.1318112015724182
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,48,48,128,1,float16,float16,511,0.23252320289611816
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,48,48,128,1,float16,fp8,511,0.25679841041564944
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,48,48,128,1,float16,float16,1023,0.4176815986633301
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,48,48,128,1,float16,fp8,1023,0.4010623931884766
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,48,48,128,1,float16,float16,1,0.012838399410247803
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,48,48,128,1,float16,float16,3,0.012822400033473968
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,48,48,128,1,float16,fp8,1,0.01329279989004135
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,48,48,128,1,float16,float16,7,0.012908799946308136
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,48,48,128,1,float16,fp8,7,0.013097600638866424
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,48,48,128,1,float16,fp8,1,0.11926560401916504
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,48,48,128,1,float16,fp8,2047,0.6710703849792481
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,48,48,128,1,float16,float16,15,0.012857599556446076
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,48,48,128,1,float16,fp8,15,0.012966400384902954
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,48,48,128,1,float16,float16,31,0.012675200402736665
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,48,48,128,1,float16,fp8,63,0.012990400195121765
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,48,48,128,1,float16,float16,63,0.012878400087356568
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,48,48,128,1,float16,float16,127,0.01271200031042099
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,48,48,128,1,float16,fp8,127,0.013201600313186646
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,48,48,128,1,float16,float16,255,0.014420799911022186
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,48,48,128,1,float16,fp8,255,0.015014399588108063
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,48,48,128,1,float16,fp8,15,0.11935679912567139
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,48,48,128,1,float16,float16,511,0.017601600289344786
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,48,48,128,1,float16,fp8,511,0.01871040016412735
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,48,48,128,1,float16,fp8,1023,0.024007999897003175
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,64,128,1,float16,float16,15,0.04020319879055023
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,48,48,128,1,float16,fp8,2047,0.03126879930496216
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,48,48,128,1,float16,float16,1023,0.020051200687885285
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,48,48,128,1,float16,fp8,1,0.013284799456596375
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,48,48,128,1,float16,float16,3,0.013016000390052795
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,48,48,128,1,float16,fp8,3,0.01329759955406189
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,48,48,128,1,float16,float16,1,0.012904000282287598
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,48,48,128,1,float16,float16,7,0.013172799348831176
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,48,48,128,1,float16,fp8,7,0.013208000361919403
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,48,48,128,1,float16,float16,15,0.012779200077056884
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,48,48,128,1,float16,fp8,15,0.013467200100421906
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,48,48,128,1,float16,float16,31,0.012948800623416901
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,48,48,128,1,float16,fp8,63,0.01319040060043335
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,48,48,128,1,float16,fp8,31,0.013155199587345123
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,48,48,128,1,float16,fp8,127,0.013334399461746216
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,48,48,128,1,float16,float16,255,0.01494240015745163
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,48,48,128,1,float16,fp8,255,0.01525920033454895
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,48,48,128,1,float16,float16,511,0.017923200130462648
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,48,48,128,1,float16,fp8,511,0.018966400623321535
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,48,48,128,1,float16,float16,1023,0.03243519961833954
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,48,48,128,1,float16,fp8,1023,0.029867199063301087
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,48,48,128,1,float16,float16,2047,0.7992688179016113
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,48,48,128,1,float16,float16,2047,0.04737119972705841
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,48,48,128,1,float16,float16,1,0.055212801694869994
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,48,48,128,1,float16,fp8,2047,0.043859198689460754
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,48,48,128,1,float16,fp8,1,0.06383680105209351
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,48,48,128,1,float16,float16,3,0.0551472008228302
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,48,48,128,1,float16,fp8,3,0.06393280029296874
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,48,48,128,1,float16,float16,7,0.05545600056648255
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,48,48,128,1,float16,fp8,7,0.0639840006828308
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,48,48,128,1,float16,float16,15,0.05553600192070007
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,48,48,128,1,float16,fp8,15,0.06399840116500854
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,48,48,128,1,float16,float16,31,0.055825597047805785
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,48,48,128,1,float16,fp8,31,0.06414880156517029
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,48,48,128,1,float16,float16,63,0.06049280166625977
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,48,48,128,1,float16,float16,127,0.06291840076446534
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,48,48,128,1,float16,fp8,127,0.07025920152664185
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,48,48,128,1,float16,float16,255,0.08539360165596008
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,64,128,1,float16,fp8,31,0.04556640088558197
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,48,48,128,1,float16,float16,63,0.01313440054655075
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,48,48,128,1,float16,float16,127,0.012931199371814727
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,48,48,128,1,float16,fp8,255,0.0913424015045166
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,48,48,128,1,float16,fp8,511,0.13506879806518554
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,48,48,128,1,float16,fp8,1023,0.2070319890975952
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,48,48,128,1,float16,float16,2047,0.40157761573791506
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,48,48,128,1,float16,fp8,31,0.012940800189971924
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,48,48,128,1,float16,float16,1,0.016771200299263
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,48,48,128,1,float16,fp8,1,0.017422400414943695
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,48,48,128,1,float16,float16,3,0.01685599982738495
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,48,48,128,1,float16,fp8,2047,0.34259040355682374
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,48,48,128,1,float16,fp8,3,0.017526400089263917
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,48,48,128,1,float16,fp8,63,0.06406400203704835
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,48,48,128,1,float16,float16,7,0.0165120005607605
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,48,48,128,1,float16,float16,15,0.016715200245380403
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,48,48,128,1,float16,fp8,15,0.017377600073814392
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,48,48,128,1,float16,float16,31,0.0166703999042511
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,48,48,128,1,float16,fp8,31,0.01757279932498932
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,48,48,128,1,float16,float16,63,0.01674239933490753
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,48,48,128,1,float16,fp8,63,0.017585599422454835
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,48,48,128,1,float16,float16,127,0.01703999936580658
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,48,48,128,1,float16,fp8,127,0.01764000058174133
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,48,48,128,1,float16,float16,255,0.020623999834060668
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,48,48,128,1,float16,fp8,255,0.021657599508762358
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,48,48,128,1,float16,float16,511,0.030244800448417663
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,48,48,128,1,float16,float16,1023,0.046374401450157164
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,48,48,128,1,float16,float16,511,0.12704639434814452
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,48,48,128,1,float16,fp8,1023,0.04473280012607574
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,48,48,128,1,float16,float16,2047,0.07540640234947205
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,48,48,128,1,float16,fp8,2047,0.0688048005104065
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,48,48,128,1,float16,float16,1,0.1841264009475708
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,48,48,128,1,float16,fp8,1,0.224783992767334
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,48,48,128,1,float16,float16,3,0.18437600135803223
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,48,48,128,1,float16,float16,1023,0.21905601024627686
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,48,48,128,1,float16,fp8,3,0.22472960948944093
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,48,48,128,1,float16,float16,7,0.1846544027328491
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,48,48,128,1,float16,float16,15,0.18466720581054688
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,48,48,128,1,float16,fp8,7,0.2250591993331909
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,48,48,128,1,float16,float16,31,0.1945024013519287
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,48,48,128,1,float16,fp8,31,0.2269984006881714
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,48,48,128,1,float16,float16,63,0.19674079418182372
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,48,48,128,1,float16,fp8,63,0.23864800930023194
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,48,48,128,1,float16,float16,127,0.2009968042373657
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,48,48,128,1,float16,fp8,7,0.017547200620174407
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,48,48,128,1,float16,fp8,127,0.2443631887435913
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,48,48,128,1,float16,float16,2047,0.034534400701522826
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,48,48,128,1,float16,float16,255,0.2873136043548584
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,48,48,128,1,float16,fp8,255,0.3217008113861084
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,48,48,128,1,float16,float16,511,0.4461215972900391
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,48,48,128,1,float16,fp8,511,0.49069762229919434
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,48,48,128,1,float16,float16,1,0.3513472080230713
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,48,48,128,1,float16,float16,1023,0.7996304035186768
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,48,48,128,1,float16,fp8,1,0.4395967960357666
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,48,48,128,1,float16,fp8,3,0.43892641067504884
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,48,48,128,1,float16,fp8,1023,0.7664912223815918
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,48,48,128,1,float16,float16,7,0.3522223949432373
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,48,48,128,1,float16,float16,15,0.3655760049819946
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,48,48,128,1,float16,fp8,3,0.013156799972057343
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,48,48,128,1,float16,fp8,15,0.4313807964324951
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,48,48,128,1,float16,float16,31,0.3714063882827759
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,48,48,128,1,float16,fp8,31,0.4605152130126953
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,48,48,128,1,float16,float16,63,0.3748032093048096
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,48,48,128,1,float16,fp8,63,0.45932321548461913
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,48,48,128,1,float16,float16,127,0.38545761108398435
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,48,48,128,1,float16,fp8,127,0.4725359916687012
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,48,48,128,1,float16,float16,255,0.549232006072998
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,48,48,128,1,float16,float16,1,0.6919087886810302
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,48,48,128,1,float16,fp8,255,0.6228640079498291
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,48,48,128,1,float16,fp8,1,0.8665776252746582
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,48,48,128,1,float16,float16,3,0.7000432014465332
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,48,48,128,1,float16,fp8,511,0.029051199555397034
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,48,48,128,1,float16,fp8,3,0.8681759834289551
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,48,48,128,1,float16,float16,3,0.3523967981338501
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,48,48,128,1,float16,float16,15,0.7242303848266601
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,48,48,128,1,float16,fp8,7,0.8468640327453614
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,48,48,128,1,float16,fp8,15,0.8767328262329102
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,48,48,128,1,float16,fp8,7,0.4403984069824219
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,48,48,128,1,float16,float16,63,0.7323935985565185
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,48,48,128,1,float16,fp8,31,0.9083951950073242
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,48,48,128,1,float16,float16,1,0.02080480009317398
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,48,48,128,1,float16,fp8,1,0.022273600101470947
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,48,48,128,1,float16,float16,3,0.02080000042915344
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,48,48,128,1,float16,fp8,3,0.022414399683475493
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,48,48,128,1,float16,float16,127,0.751148796081543
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,48,48,128,1,float16,fp8,7,0.02242880016565323
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,48,48,128,1,float16,fp8,15,0.02235199958086014
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,48,48,128,1,float16,fp8,127,0.926211166381836
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,48,48,128,1,float16,float16,31,0.020926399528980254
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,48,48,128,1,float16,float16,63,0.02096160054206848
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,48,48,128,1,float16,fp8,31,0.022465600073337554
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,48,48,128,1,float16,fp8,63,0.022332799434661866
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,48,48,128,1,float16,float16,127,0.021320000290870667
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,48,48,128,1,float16,fp8,127,0.022353599965572356
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,48,48,128,1,float16,float16,255,0.030300799012184142
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,48,48,128,1,float16,fp8,255,0.02831520140171051
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,48,48,128,1,float16,float16,511,0.041736000776290895
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,48,48,128,1,float16,fp8,511,0.04178400039672851
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,48,48,128,1,float16,float16,1023,0.06646080017089843
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,48,48,128,1,float16,fp8,1023,0.061217600107192995
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,48,48,128,1,float16,float16,2047,0.11275839805603027
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,48,48,128,1,float16,fp8,2047,0.09711199998855591
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,48,48,128,1,float16,float16,7,0.7135024070739746
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,48,48,128,1,float16,float16,31,0.725380802154541
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,48,48,128,1,float16,float16,1,1.4306991577148438
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,48,48,128,1,float16,fp8,15,0.2266000032424927
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,48,48,128,1,float16,fp8,63,0.9074879646301269
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,48,48,128,1,float16,fp8,1,1.6897392272949219
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,48,48,128,1,float16,float16,7,0.020849600434303284
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,48,48,128,1,float16,float16,15,0.02099200040102005
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,48,48,128,1,float16,float16,3,1.4255184173583983
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,48,48,128,1,float16,float16,7,1.4297648429870606
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,48,48,128,1,float16,fp8,3,1.715407943725586
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,48,48,128,1,float16,float16,15,1.4257696151733399
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,48,48,128,1,float16,fp8,7,1.7753423690795898
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,48,48,128,1,float16,float16,31,1.43221435546875
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,48,48,128,1,float16,fp8,15,1.7954608917236328
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,48,48,128,1,float16,fp8,31,1.7879487991333007
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,48,48,128,1,float16,float16,63,1.4452351570129394
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,48,48,128,1,float16,fp8,63,1.7875104904174806
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,48,48,128,1,float16,float16,1,2.830188751220703
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,48,48,128,1,float16,float16,3,2.831145668029785
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,48,48,128,1,float16,fp8,1,3.5733615875244142
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,48,48,128,1,float16,float16,1,0.03201920092105866
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,48,48,128,1,float16,fp8,1,0.035897600650787356
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,48,48,128,1,float16,float16,3,0.03214240074157715
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,48,48,128,1,float16,fp8,3,0.0361519992351532
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,48,48,128,1,float16,float16,7,0.03214240074157715
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,48,48,128,1,float16,fp8,7,0.03611519932746887
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,48,48,128,1,float16,float16,15,2.8364816665649415
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,48,48,128,1,float16,float16,15,0.03223359882831574
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,48,48,128,1,float16,float16,31,0.032183998823165895
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,48,48,128,1,float16,fp8,7,3.540758514404297
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,48,48,128,1,float16,fp8,31,0.036108800768852235
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,48,48,128,1,float16,fp8,63,0.03614560067653656
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,48,48,128,1,float16,float16,127,0.03747040033340454
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,48,48,128,1,float16,fp8,127,0.03620319962501526
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,48,48,128,1,float16,float16,255,0.05033119916915894
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,48,48,128,1,float16,float16,31,2.8435760498046876
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,48,48,128,1,float16,fp8,255,0.05063040256500244
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,48,48,128,1,float16,float16,511,0.0751200020313263
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,48,48,128,1,float16,fp8,15,3.561715316772461
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,48,48,128,1,float16,float16,1023,0.11872320175170899
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,48,48,128,1,float16,fp8,1023,0.11029119491577148
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,48,48,128,1,float16,float16,2047,0.20885920524597168
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,48,48,128,1,float16,fp8,31,3.5360671997070314
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,48,48,128,1,float16,fp8,2047,0.17979040145874023
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,40,40,128,1,float16,float16,1,0.08463039994239807
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,40,40,128,1,float16,float16,3,0.08456000089645385
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,48,48,128,1,float16,fp8,3,3.565377426147461
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,40,40,128,1,float16,float16,7,0.0844319999217987
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,40,40,128,1,float16,fp8,3,0.10141760110855103
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,40,40,128,1,float16,float16,15,0.0845200002193451
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,40,40,128,1,float16,fp8,7,0.1014143943786621
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,40,40,128,1,float16,float16,31,0.08544800281524659
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,40,40,128,1,float16,fp8,31,0.10147039890289307
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,40,40,128,1,float16,float16,63,0.09102720022201538
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,40,40,128,1,float16,fp8,63,0.1073248028755188
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,40,40,128,1,float16,float16,127,0.0933247983455658
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,48,48,128,1,float16,fp8,15,0.036160001158714296
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,40,40,128,1,float16,float16,255,0.12864480018615723
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,48,48,128,1,float16,float16,63,0.032867199182510375
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,40,40,128,1,float16,fp8,255,0.14509119987487792
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,40,40,128,1,float16,fp8,511,0.21728799343109131
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,48,48,128,1,float16,fp8,511,0.07319840192794799
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,40,40,128,1,float16,float16,1023,0.35087358951568604
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,40,40,128,1,float16,fp8,1023,0.33688480854034425
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,40,40,128,1,float16,float16,1,0.012740799784660339
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,40,40,128,1,float16,fp8,1,0.01348000019788742
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,40,40,128,1,float16,float16,2047,0.6523727893829345
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,40,40,128,1,float16,float16,3,0.01279360055923462
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,48,48,128,1,float16,float16,7,2.835683250427246
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,40,40,128,1,float16,float16,7,0.012785600125789642
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,40,40,128,1,float16,float16,15,0.012772800028324127
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,40,40,128,1,float16,fp8,3,0.013652800023555756
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,40,40,128,1,float16,fp8,15,0.013447999954223633
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,40,40,128,1,float16,fp8,7,0.013275200128555298
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,40,40,128,1,float16,fp8,31,0.013476799428462981
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,40,40,128,1,float16,fp8,15,0.10151679515838623
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,40,40,128,1,float16,float16,63,0.012585599720478059
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,40,40,128,1,float16,float16,127,0.01279519945383072
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,40,40,128,1,float16,fp8,127,0.01345600038766861
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,40,40,128,1,float16,float16,255,0.014579200744628906
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,40,40,128,1,float16,fp8,255,0.01536639928817749
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,40,40,128,1,float16,fp8,511,0.01897439956665039
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,40,40,128,1,float16,float16,1023,0.01851679980754852
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,40,40,128,1,float16,fp8,127,0.11157280206680298
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,40,40,128,1,float16,fp8,1023,0.019859200716018675
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,40,40,128,1,float16,fp8,2047,0.023494400084018707
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,40,40,128,1,float16,float16,1,0.012668800354003907
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,40,40,128,1,float16,float16,2047,0.025254398584365845
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,40,40,128,1,float16,fp8,1,0.01334560066461563
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,40,40,128,1,float16,float16,511,0.1987887978553772
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,40,40,128,1,float16,fp8,3,0.013497599959373474
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,40,40,128,1,float16,fp8,7,0.0134320005774498
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,40,40,128,1,float16,float16,15,0.012729600071907043
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,40,40,128,1,float16,fp8,15,0.013579200208187103
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,40,40,128,1,float16,float16,31,0.012806400656700134
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,40,40,128,1,float16,fp8,31,0.0134320005774498
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,40,40,128,1,float16,float16,63,0.012824000418186187
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,40,40,128,1,float16,fp8,2047,0.5628928184509278
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,40,40,128,1,float16,fp8,1,0.1015504002571106
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,40,40,128,1,float16,fp8,63,0.013531200587749481
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,40,40,128,1,float16,fp8,127,0.013553600013256072
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,40,40,128,1,float16,float16,127,0.012948800623416901
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,40,40,128,1,float16,fp8,255,0.01578560024499893
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,40,40,128,1,float16,fp8,63,0.013459199666976928
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,40,40,128,1,float16,fp8,511,0.019176000356674196
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,40,40,128,1,float16,float16,1023,0.030177599191665648
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,40,40,128,1,float16,float16,511,0.017679999768733978
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,40,40,128,1,float16,fp8,1023,0.0303600013256073
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,40,40,128,1,float16,float16,2047,0.04601759910583496
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,40,40,128,1,float16,fp8,2047,0.04324800074100495
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,40,40,128,1,float16,float16,1,0.04757120013237
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,40,40,128,1,float16,fp8,1,0.05546879768371582
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,40,40,128,1,float16,float16,3,0.047644799947738646
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,40,40,128,1,float16,fp8,3,0.05540800094604492
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,40,40,128,1,float16,float16,7,0.047758400440216064
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,40,40,128,1,float16,fp8,7,0.05552639961242676
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,40,40,128,1,float16,float16,15,0.04778240025043488
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,40,40,128,1,float16,fp8,15,0.05564799904823303
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,40,40,128,1,float16,float16,31,0.047830399870872495
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,40,40,128,1,float16,fp8,31,0.05565599799156189
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,40,40,128,1,float16,fp8,63,0.05559200048446655
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,40,40,128,1,float16,float16,127,0.0543615996837616
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,40,40,128,1,float16,fp8,127,0.05890880227088928
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,40,40,128,1,float16,float16,255,0.0727183997631073
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,40,40,128,1,float16,fp8,255,0.07901600003242493
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,40,40,128,1,float16,float16,511,0.10784319639205933
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,40,40,128,1,float16,fp8,511,0.1154528021812439
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,40,40,128,1,float16,float16,1023,0.184443199634552
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,40,40,128,1,float16,float16,31,0.012726399302482604
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,40,40,128,1,float16,float16,511,0.017950400710105896
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,40,40,128,1,float16,fp8,1023,0.1769744038581848
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,40,40,128,1,float16,float16,1,0.016630400717258454
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,40,40,128,1,float16,fp8,1,0.017910400032997133
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,40,40,128,1,float16,float16,2047,0.33277919292449953
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,40,40,128,1,float16,float16,3,0.01663679927587509
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,40,40,128,1,float16,fp8,2047,0.28984320163726807
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,40,40,128,1,float16,float16,7,0.016572800278663636
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,40,40,128,1,float16,fp8,3,0.017688000202178956
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,40,40,128,1,float16,float16,15,0.016527999937534333
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,40,40,128,1,float16,fp8,7,0.01765599995851517
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,40,40,128,1,float16,fp8,15,0.01772959977388382
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,40,40,128,1,float16,float16,31,0.016575999557971954
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,40,40,128,1,float16,fp8,31,0.01780160069465637
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,40,40,128,1,float16,fp8,63,0.017936000227928163
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,40,40,128,1,float16,float16,127,0.01675360053777695
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,40,40,128,1,float16,float16,3,0.012678399682044983
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,40,40,128,1,float16,float16,255,0.020360000431537628
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,40,40,128,1,float16,fp8,255,0.021878400444984437
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,40,40,128,1,float16,float16,63,0.0499343991279602
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,40,40,128,1,float16,float16,1023,0.04537599980831146
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,40,40,128,1,float16,fp8,1023,0.04435519874095917
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,40,40,128,1,float16,float16,2047,0.07471839785575866
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,40,40,128,1,float16,fp8,2047,0.06967359781265259
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,40,40,128,1,float16,float16,1,0.1555616021156311
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,40,40,128,1,float16,fp8,1,0.189300799369812
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,40,40,128,1,float16,float16,3,0.15549440383911134
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,40,40,128,1,float16,fp8,3,0.18940479755401612
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,40,40,128,1,float16,float16,7,0.1555567979812622
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,40,40,128,1,float16,float16,15,0.15640480518341066
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,40,40,128,1,float16,float16,255,0.014582400023937226
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,40,40,128,1,float16,float16,31,0.16213760375976563
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,40,40,128,1,float16,fp8,31,0.19077919721603392
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,40,40,128,1,float16,float16,63,0.16599999666213988
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,40,40,128,1,float16,float16,63,0.016451199352741242
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,40,40,128,1,float16,fp8,127,0.018113599717617036
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,40,40,128,1,float16,float16,127,0.17068320512771606
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,40,40,128,1,float16,float16,7,0.012703999876976013
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,40,40,128,1,float16,fp8,127,0.20695199966430664
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,40,40,128,1,float16,float16,255,0.23914880752563478
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,40,40,128,1,float16,fp8,255,0.2711424112319946
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,40,40,128,1,float16,float16,511,0.37192320823669434
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,40,40,128,1,float16,fp8,511,0.4116655826568604
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,40,40,128,1,float16,float16,1,0.29535679817199706
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,40,40,128,1,float16,float16,1023,0.6934432029724121
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,40,40,128,1,float16,fp8,1023,0.641051197052002
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,40,40,128,1,float16,fp8,15,0.18946399688720703
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,40,40,128,1,float16,fp8,1,0.3689248085021973
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,40,40,128,1,float16,float16,3,0.2946079969406128
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,40,40,128,1,float16,float16,7,0.2961440086364746
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,40,40,128,1,float16,fp8,7,0.3692336082458496
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,40,40,128,1,float16,float16,15,0.30240321159362793
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,40,40,128,1,float16,fp8,15,0.3698848009109497
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,40,40,128,1,float16,float16,31,0.31285600662231444
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,40,40,128,1,float16,fp8,31,0.38543999195098877
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,40,40,128,1,float16,fp8,63,0.2006671905517578
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,40,40,128,1,float16,float16,63,0.3151295900344849
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,40,40,128,1,float16,fp8,63,0.3887840032577515
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,40,40,128,1,float16,float16,511,0.028115200996398925
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,40,40,128,1,float16,float16,127,0.3240880012512207
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,40,40,128,1,float16,fp8,511,0.029390400648117064
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,40,40,128,1,float16,fp8,127,0.3975087881088257
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,40,40,128,1,float16,float16,255,0.46011199951171877
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,40,40,128,1,float16,fp8,255,0.5250912189483643
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,40,40,128,1,float16,float16,3,0.5777775764465332
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,40,40,128,1,float16,fp8,1,0.721608018875122
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,40,40,128,1,float16,fp8,3,0.7282320022583008
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,40,40,128,1,float16,float16,7,0.5891600131988526
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,40,40,128,1,float16,float16,15,0.6049967765808105
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,40,40,128,1,float16,fp8,7,0.7249472141265869
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,40,40,128,1,float16,fp8,7,0.18952800035476686
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,40,40,128,1,float16,fp8,15,0.7221327781677246
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,40,40,128,1,float16,float16,31,0.6100431919097901
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,40,40,128,1,float16,float16,63,0.6135903835296631
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,40,40,128,1,float16,fp8,31,0.7611616134643555
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,40,40,128,1,float16,fp8,1,0.022603200376033784
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,40,40,128,1,float16,fp8,63,0.7581600189208985
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,40,40,128,1,float16,float16,3,0.020556800067424774
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,40,40,128,1,float16,float16,7,0.020535999536514284
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,40,40,128,1,float16,fp8,3,0.02261119931936264
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,40,40,128,1,float16,fp8,7,0.022580799460411072
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,40,40,128,1,float16,fp8,15,0.022625599801540375
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,40,40,128,1,float16,float16,31,0.020759999752044678
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,40,40,128,1,float16,fp8,31,0.022784000635147093
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,40,40,128,1,float16,float16,63,0.020665599405765532
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,40,40,128,1,float16,fp8,63,0.02258400022983551
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,40,40,128,1,float16,float16,127,0.020796799659729005
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,40,40,128,1,float16,fp8,127,0.0226160004734993
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,40,40,128,1,float16,float16,255,0.027684798836708067
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,40,40,128,1,float16,fp8,255,0.02871200144290924
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,40,40,128,1,float16,float16,1,0.5754159927368164
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,40,40,128,1,float16,fp8,511,0.04159359931945801
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,40,40,128,1,float16,float16,1023,0.06272000074386597
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,40,40,128,1,float16,fp8,1023,0.06219840049743652
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,40,40,128,1,float16,float16,2047,0.10856000185012818
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,40,40,128,1,float16,fp8,2047,0.09811679720878601
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,40,40,128,1,float16,fp8,3,0.36952478885650636
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,40,40,128,1,float16,float16,127,0.629633617401123
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,40,40,128,1,float16,fp8,127,0.7761568069458008
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,40,40,128,1,float16,float16,15,0.020580799877643587
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,40,40,128,1,float16,float16,1,1.1904208183288574
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,40,40,128,1,float16,fp8,1,1.393387222290039
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,40,40,128,1,float16,float16,3,1.1910191535949708
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,40,40,128,1,float16,float16,7,1.19236478805542
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,40,40,128,1,float16,fp8,3,1.386467170715332
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,40,40,128,1,float16,fp8,7,1.438606357574463
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,40,40,128,1,float16,float16,15,1.196116828918457
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,40,40,128,1,float16,fp8,15,1.493387222290039
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,40,40,128,1,float16,float16,31,1.1942048072814941
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,40,40,128,1,float16,float16,63,1.2037903785705566
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,40,40,128,1,float16,fp8,31,1.4931615829467773
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,40,40,128,1,float16,float16,1,0.020577600598335265
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,40,40,128,1,float16,fp8,63,1.4885616302490234
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,40,40,128,1,float16,float16,1,2.3697040557861326
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,40,40,128,1,float16,float16,511,0.04024479985237121
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,40,40,128,1,float16,float16,3,2.3585248947143556
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,40,40,128,1,float16,float16,7,2.3608976364135743
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,40,40,128,1,float16,float16,1,0.028527998924255372
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,40,40,128,1,float16,fp8,1,0.031911998987197876
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,40,40,128,1,float16,fp8,3,2.96658878326416
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,40,40,128,1,float16,fp8,3,0.03177759945392609
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,40,40,128,1,float16,float16,7,0.02794240117073059
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,40,40,128,1,float16,fp8,7,0.03192479908466339
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,40,40,128,1,float16,float16,15,0.028137600421905516
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,40,40,128,1,float16,fp8,15,0.03197920024394989
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,40,40,128,1,float16,float16,31,0.028313601016998292
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,40,40,128,1,float16,fp8,31,0.03205280005931854
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,40,40,128,1,float16,float16,63,0.028307199478149414
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,40,40,128,1,float16,fp8,63,0.03182719945907593
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,40,40,128,1,float16,fp8,15,2.9567167282104494
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,40,40,128,1,float16,float16,127,0.029937601089477538
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,40,40,128,1,float16,fp8,127,0.03239679932594299
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,40,40,128,1,float16,float16,255,0.043614399433135984
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,40,40,128,1,float16,fp8,255,0.043808001279830935
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,40,40,128,1,float16,float16,511,0.0605135977268219
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,40,40,128,1,float16,fp8,31,2.9388959884643553
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,40,40,128,1,float16,fp8,511,0.06369760036468505
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,40,40,128,1,float16,float16,1023,0.10084160566329955
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,40,40,128,1,float16,fp8,1023,0.09488319754600524
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,40,40,128,1,float16,float16,2047,0.17455680370330812
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,40,40,128,1,float16,fp8,2047,0.15286240577697754
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,32,128,1,float16,float16,3,0.06989759802818299
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,32,128,1,float16,fp8,1,0.0826960027217865
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,40,40,128,1,float16,fp8,1,2.9704975128173827
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,32,128,1,float16,float16,7,0.06988639831542968
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,32,128,1,float16,float16,15,0.07015200257301331
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,40,40,128,1,float16,float16,15,2.3672016143798826
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,32,128,1,float16,fp8,15,0.08276159763336181
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,32,128,1,float16,float16,31,0.07075679898262024
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,32,128,1,float16,fp8,31,0.08280640244483947
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,40,40,128,1,float16,fp8,7,2.979960060119629
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,32,128,1,float16,float16,63,0.07586560249328614
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,32,128,1,float16,fp8,63,0.08403999805450439
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,32,128,1,float16,float16,127,0.07785440087318421
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,32,128,1,float16,float16,255,0.10631519556045532
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,32,128,1,float16,fp8,127,0.09028319716453552
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,32,128,1,float16,float16,511,0.16105600595474243
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,32,128,1,float16,fp8,511,0.17548320293426514
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,40,40,128,1,float16,float16,31,2.3710048675537108
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,32,128,1,float16,fp8,1023,0.27138400077819824
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,32,128,1,float16,fp8,2047,0.45462560653686523
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,32,128,1,float16,float16,1,0.07012159824371338
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,32,128,1,float16,fp8,3,0.08246719837188721
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,32,128,1,float16,float16,1,0.012449599802494049
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,32,128,1,float16,fp8,1,0.01286720037460327
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,32,128,1,float16,float16,3,0.01225920021533966
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,32,128,1,float16,fp8,3,0.012856000661849975
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,32,128,1,float16,float16,7,0.012273599952459335
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,32,128,1,float16,float16,4095,1.0226079940795898
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,32,128,1,float16,fp8,7,0.012894399464130402
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,32,128,1,float16,fp8,255,0.11764800548553467
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,32,128,1,float16,fp8,15,0.012907199561595917
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,32,128,1,float16,float16,31,0.012651200592517852
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,32,128,1,float16,fp8,31,0.013121600449085235
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,32,128,1,float16,float16,63,0.012668800354003907
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,32,128,1,float16,fp8,63,0.013023999333381654
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,32,128,1,float16,float16,127,0.01271200031042099
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,32,128,1,float16,fp8,127,0.013014400005340576
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,32,128,1,float16,float16,255,0.01422239989042282
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,32,128,1,float16,fp8,255,0.014723199605941772
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,32,128,1,float16,float16,511,0.017214399576187134
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,32,128,1,float16,fp8,511,0.018489600718021394
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,32,128,1,float16,float16,2047,0.5279007911682129
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,32,128,1,float16,float16,1023,0.01818079948425293
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,32,128,1,float16,float16,2047,0.02019200026988983
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,32,128,1,float16,fp8,2047,0.021264000236988066
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,32,128,1,float16,fp8,1023,0.017529599368572235
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,32,128,1,float16,float16,4095,0.03127839863300323
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,32,128,1,float16,fp8,4095,0.02712480127811432
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,32,128,1,float16,float16,1,0.012643200159072877
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,32,128,1,float16,fp8,1,0.01308639943599701
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,32,128,1,float16,float16,3,0.01247360035777092
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,32,128,1,float16,fp8,3,0.013043199479579926
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,32,128,1,float16,float16,7,0.012441600114107132
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,32,128,1,float16,fp8,7,0.012987199425697326
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,32,128,1,float16,fp8,7,0.08182880282402039
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,32,128,1,float16,fp8,15,0.013121600449085235
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,40,40,128,1,float16,float16,3,0.028601598739624024
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,32,128,1,float16,float16,31,0.012539200484752655
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,32,128,1,float16,fp8,31,0.01300320029258728
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,32,128,1,float16,float16,63,0.012654399871826172
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,32,128,1,float16,fp8,63,0.012958399951457977
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,32,128,1,float16,float16,255,0.014575999975204468
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,32,128,1,float16,fp8,4095,0.8466896057128906
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,32,128,1,float16,fp8,255,0.015072000026702882
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,32,128,1,float16,fp8,511,0.018648000061511995
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,32,128,1,float16,float16,511,0.017446400225162507
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,32,128,1,float16,float16,1023,0.0200080007314682
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,32,128,1,float16,fp8,1023,0.021027199923992157
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,32,128,1,float16,float16,15,0.012619200348854064
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,32,128,1,float16,fp8,2047,0.026982399821281432
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,32,128,1,float16,float16,4095,0.04591200053691864
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,32,128,1,float16,fp8,4095,0.04135519862174988
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,32,128,1,float16,float16,1,0.03983519971370697
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,32,128,1,float16,fp8,1,0.0455839991569519
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,32,128,1,float16,float16,1023,0.2837791919708252
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,32,128,1,float16,fp8,3,0.04556959867477417
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,32,128,1,float16,fp8,7,0.045745599269866946
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,32,128,1,float16,float16,15,0.04019840061664581
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,32,128,1,float16,fp8,15,0.04573279917240143
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,32,128,1,float16,float16,31,0.04025599956512451
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,32,128,1,float16,fp8,31,0.04559679925441742
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,32,128,1,float16,float16,63,0.04009760022163391
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,32,128,1,float16,fp8,63,0.045561599731445315
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,32,128,1,float16,float16,127,0.04543839991092682
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,32,128,1,float16,fp8,127,0.04578559994697571
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,32,128,1,float16,float16,255,0.06062399744987488
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,32,128,1,float16,fp8,255,0.06380000114440917
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,32,128,1,float16,float16,15,0.012489599734544754
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,32,128,1,float16,float16,511,0.08906880021095276
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,32,128,1,float16,fp8,511,0.09287679791450501
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,32,128,1,float16,float16,127,0.012745599448680877
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,32,128,1,float16,float16,1023,0.15082080364227296
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,32,128,1,float16,fp8,1023,0.14258719682693483
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,32,128,1,float16,fp8,127,0.013067199289798737
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,32,128,1,float16,float16,2047,0.2700831890106201
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,32,128,1,float16,float16,1,0.01268479973077774
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,32,128,1,float16,float16,3,0.012664000689983367
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,32,128,1,float16,fp8,1,0.013206399977207184
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,32,128,1,float16,float16,2047,0.030985599756240843
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,32,128,1,float16,float16,7,0.012614400684833526
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,32,128,1,float16,fp8,7,0.012948800623416901
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,32,128,1,float16,float16,15,0.012580800056457519
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,32,128,1,float16,fp8,4095,0.43326239585876464
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,32,128,1,float16,fp8,15,0.012972800433635712
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,32,128,1,float16,float16,31,0.012601600587368011
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,32,128,1,float16,float16,63,0.012547199428081513
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,32,128,1,float16,fp8,31,0.013105599582195282
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,32,128,1,float16,float16,3,0.0402319997549057
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,32,128,1,float16,fp8,127,0.01295360028743744
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,32,128,1,float16,float16,255,0.014638400077819825
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,32,128,1,float16,fp8,255,0.01515199989080429
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,32,128,1,float16,float16,511,0.017932799458503724
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,32,128,1,float16,float16,7,0.03983039855957031
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,32,128,1,float16,fp8,511,0.018887999653816222
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,32,128,1,float16,float16,1023,0.0305184006690979
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,32,128,1,float16,fp8,1023,0.026263999938964843
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,32,128,1,float16,float16,2047,0.04518879950046539
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,32,128,1,float16,fp8,2047,0.041576001048088077
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,32,128,1,float16,float16,4095,0.07552480101585388
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,32,128,1,float16,fp8,4095,0.06605280041694642
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,32,128,1,float16,float16,1,0.12773280143737792
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,32,128,1,float16,fp8,1,0.15599520206451417
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,32,128,1,float16,fp8,3,0.1559999942779541
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,32,128,1,float16,float16,7,0.12715519666671754
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,32,128,1,float16,fp8,7,0.1561792016029358
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,32,128,1,float16,float16,15,0.1277791976928711
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,32,128,1,float16,fp8,15,0.15607520341873168
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,32,128,1,float16,float16,31,0.13140159845352173
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,32,128,1,float16,fp8,31,0.15589439868927002
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,32,128,1,float16,float16,63,0.13615520000457765
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,32,128,1,float16,fp8,63,0.1644063949584961
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,32,128,1,float16,fp8,2047,0.23346879482269287
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,32,128,1,float16,fp8,3,0.013155199587345123
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,32,128,1,float16,fp8,127,0.16710879802703857
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,32,128,1,float16,float16,4095,0.5089888095855712
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,32,128,1,float16,float16,255,0.1952623963356018
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,32,128,1,float16,fp8,63,0.013225600123405457
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,32,128,1,float16,fp8,255,0.2193984031677246
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,32,128,1,float16,float16,127,0.0127920001745224
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,32,128,1,float16,float16,511,0.3062671899795532
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,32,128,1,float16,fp8,511,0.33564159870147703
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,32,128,1,float16,float16,1023,0.5492656230926514
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,32,128,1,float16,float16,1,0.2391983985900879
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,32,128,1,float16,float16,3,0.12704639434814452
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,32,128,1,float16,float16,3,0.23925440311431884
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,32,128,1,float16,fp8,1,0.29667201042175295
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,32,128,1,float16,fp8,3,0.2965935945510864
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,32,128,1,float16,float16,7,0.23951520919799804
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,32,128,1,float16,fp8,2047,0.8916223526000977
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,32,128,1,float16,float16,15,0.2413487911224365
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,32,128,1,float16,fp8,15,0.29696478843688967
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,32,128,1,float16,float16,31,0.2537247896194458
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,32,128,1,float16,fp8,31,0.30197279453277587
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,32,128,1,float16,float16,63,0.25514559745788573
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,32,128,1,float16,fp8,63,0.3115567922592163
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,32,128,1,float16,float16,127,0.26243839263916013
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,32,128,1,float16,fp8,127,0.32096960544586184
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,32,128,1,float16,float16,255,0.3718991994857788
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,32,128,1,float16,fp8,255,0.4212639808654785
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,32,128,1,float16,float16,511,0.5899968147277832
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,32,128,1,float16,float16,1,0.4641439914703369
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,32,128,1,float16,fp8,511,0.6511568069458008
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,32,128,1,float16,fp8,1023,0.5261072158813477
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,32,128,1,float16,fp8,1,0.581663990020752
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,32,128,1,float16,float16,3,0.4643727779388428
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,32,128,1,float16,fp8,3,0.5825583934783936
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,32,128,1,float16,float16,2047,1.0533679962158202
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,32,128,1,float16,float16,15,0.48800320625305177
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,32,128,1,float16,fp8,7,0.5800415992736816
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,32,128,1,float16,fp8,7,0.2975343942642212
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,32,128,1,float16,fp8,15,0.5797008037567138
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,32,128,1,float16,float16,63,0.495468807220459
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,32,128,1,float16,fp8,31,0.6110928058624268
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,32,128,1,float16,fp8,63,0.6082223892211914
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,32,128,1,float16,float16,1,0.016624000668525696
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,32,128,1,float16,fp8,1,0.017518399655818938
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,32,128,1,float16,float16,127,0.14018880128860473
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,32,128,1,float16,float16,3,0.016603200137615202
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,32,128,1,float16,float16,7,0.016622400283813475
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,32,128,1,float16,fp8,7,0.017560000717639922
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,32,128,1,float16,float16,15,0.016788800060749055
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,32,128,1,float16,float16,255,0.727894401550293
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,32,128,1,float16,fp8,15,0.017769600450992584
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,32,128,1,float16,fp8,255,0.8255696296691895
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,32,128,1,float16,fp8,31,0.017759999632835387
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,32,128,1,float16,float16,31,0.016828800737857818
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,32,128,1,float16,fp8,63,0.01780800074338913
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,32,128,1,float16,float16,63,0.016769599914550782
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,32,128,1,float16,fp8,127,0.017852799594402315
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,32,128,1,float16,float16,255,0.02046400010585785
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,32,128,1,float16,fp8,255,0.021819199621677398
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,32,128,1,float16,float16,511,0.03092319965362549
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,32,128,1,float16,fp8,511,0.029076799750328064
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,32,128,1,float16,float16,1023,0.04762240052223206
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,32,128,1,float16,float16,2047,0.07785760164260865
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,32,128,1,float16,fp8,2047,0.06881600022315978
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,32,128,1,float16,float16,4095,0.1372607946395874
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,32,128,1,float16,float16,7,0.46756319999694823
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,32,128,1,float16,fp8,4095,0.11820000410079956
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,32,128,1,float16,float16,31,0.4986464023590088
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,32,128,1,float16,float16,127,0.5079472064971924
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,32,32,128,1,float16,float16,1,0.9485199928283692
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,32,128,1,float16,fp8,127,0.6193808078765869
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,32,32,128,1,float16,fp8,1,1.12533597946167
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,32,128,1,float16,fp8,3,0.017604799568653108
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,32,32,128,1,float16,float16,3,0.9557071685791015
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,32,32,128,1,float16,float16,7,0.9572272300720215
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,32,32,128,1,float16,fp8,3,1.1182080268859864
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,32,128,1,float16,float16,127,0.017059199512004852
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,32,32,128,1,float16,float16,15,0.9578288078308106
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,32,32,128,1,float16,fp8,7,1.1305343627929687
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,32,128,1,float16,fp8,1023,0.045307201147079465
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,32,32,128,1,float16,float16,31,0.9686944007873535
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,32,32,128,1,float16,float16,63,0.9717871665954589
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,32,32,128,1,float16,fp8,63,1.1868927955627442
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,32,32,128,1,float16,float16,127,0.9956656455993652
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,32,32,128,1,float16,fp8,127,1.2175279617309571
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,32,32,128,1,float16,float16,1,1.8902864456176758
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,32,32,128,1,float16,fp8,1,2.383799934387207
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,32,32,128,1,float16,float16,3,1.8944704055786132
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,32,32,128,1,float16,float16,7,1.8925344467163085
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,32,32,128,1,float16,fp8,3,2.3842960357666017
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,32,32,128,1,float16,fp8,15,1.2006367683410644
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,32,32,128,1,float16,fp8,7,2.3867584228515626
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,32,128,1,float16,float16,1,0.024107199907302857
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,32,32,128,1,float16,fp8,31,1.1957455635070802
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,32,128,1,float16,float16,3,0.024673600494861603
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,32,32,128,1,float16,float16,31,1.9029327392578126
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,32,128,1,float16,fp8,3,0.026870399713516235
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,32,32,128,1,float16,fp8,15,2.375275230407715
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,32,128,1,float16,float16,7,0.024163199961185454
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,32,128,1,float16,fp8,7,0.02645759880542755
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,32,128,1,float16,float16,15,0.024707199633121492
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,32,128,1,float16,float16,31,0.02470400035381317
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,32,32,128,1,float16,float16,63,1.9217519760131836
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,32,128,1,float16,fp8,31,0.026875200867652892
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,32,128,1,float16,fp8,63,0.026892799139022826
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,32,128,1,float16,float16,63,0.024534399807453155
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,32,128,1,float16,float16,127,0.025054401159286498
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,32,128,1,float16,fp8,127,0.026836800575256347
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,32,128,1,float16,float16,255,0.03667680025100708
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,32,128,1,float16,fp8,255,0.0350847989320755
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,32,128,1,float16,float16,511,0.05055519938468933
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,32,128,1,float16,fp8,511,0.051553601026535036
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,32,32,128,1,float16,fp8,63,2.3431535720825196
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,32,128,1,float16,float16,1023,0.08303040266036987
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,32,128,1,float16,fp8,1023,0.07714239954948425
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,32,128,1,float16,fp8,2047,0.12535200119018555
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,32,128,1,float16,float16,4095,0.26216959953308105
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,32,128,1,float16,fp8,4095,0.2245136022567749
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,32,32,128,1,float16,float16,15,1.8966224670410157
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,24,24,128,1,float16,fp8,1,0.06454560160636902
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,24,24,128,1,float16,float16,3,0.05464640259742737
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,24,24,128,1,float16,fp8,3,0.06454240083694458
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,32,128,1,float16,fp8,1,0.02677760124206543
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,24,24,128,1,float16,float16,7,0.05489919781684875
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,24,24,128,1,float16,fp8,7,0.06474720239639283
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,24,24,128,1,float16,float16,15,0.05475040078163147
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,24,24,128,1,float16,float16,31,0.05535039901733398
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,24,24,128,1,float16,fp8,15,0.06443200111389161
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,24,24,128,1,float16,float16,63,0.05900480151176453
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,24,24,128,1,float16,fp8,31,0.0645359992980957
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,24,24,128,1,float16,fp8,63,0.06445760130882264
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,24,24,128,1,float16,float16,127,0.06277440190315246
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,24,24,128,1,float16,fp8,127,0.07035840153694153
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,24,24,128,1,float16,float16,255,0.08327839970588684
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,24,24,128,1,float16,fp8,255,0.09228000044822693
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,32,128,1,float16,fp8,15,0.026532799005508423
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,32,32,128,1,float16,fp8,31,2.354884719848633
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,24,24,128,1,float16,float16,1023,0.21848320960998535
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,24,24,128,1,float16,fp8,1023,0.20841119289398194
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,24,24,128,1,float16,float16,2047,0.3949023962020874
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,24,24,128,1,float16,fp8,2047,0.3461375951766968
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,24,24,128,1,float16,float16,1,0.012147200107574464
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,32,128,1,float16,float16,2047,0.14171680212020873
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,24,24,128,1,float16,float16,3,0.012305600196123123
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,24,24,128,1,float16,float16,4095,0.7767680168151856
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,24,24,128,1,float16,float16,1,0.054657602310180665
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,24,24,128,1,float16,fp8,3,0.012960000336170197
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,24,24,128,1,float16,float16,15,0.012254399806261062
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,24,24,128,1,float16,fp8,15,0.013124799728393555
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,24,24,128,1,float16,fp8,7,0.013011200726032257
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,24,24,128,1,float16,fp8,31,0.013116799294948578
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,24,24,128,1,float16,float16,63,0.012332800030708312
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,24,24,128,1,float16,fp8,63,0.013143999874591828
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,24,24,128,1,float16,float16,127,0.012256000190973282
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,24,24,128,1,float16,fp8,127,0.013171200454235078
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,24,24,128,1,float16,float16,255,0.014044800400733947
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,24,24,128,1,float16,fp8,255,0.015003199875354766
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,24,24,128,1,float16,float16,511,0.01703200042247772
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,24,24,128,1,float16,fp8,511,0.018691200017929076
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,24,24,128,1,float16,float16,1023,0.016607999801635742
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,24,24,128,1,float16,fp8,1023,0.017662400007247926
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,24,24,128,1,float16,fp8,2047,0.019870400428771973
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,24,24,128,1,float16,float16,4095,0.027105599641799927
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,24,24,128,1,float16,fp8,4095,0.025811201333999632
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,24,24,128,1,float16,float16,1,0.012436799705028534
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,24,24,128,1,float16,float16,511,0.12613760232925414
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,24,24,128,1,float16,float16,3,0.012412799894809723
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,24,24,128,1,float16,fp8,3,0.013116799294948578
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,24,24,128,1,float16,float16,7,0.0124208003282547
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,24,24,128,1,float16,fp8,7,0.013334399461746216
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,24,24,128,1,float16,float16,15,0.012430399656295776
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,24,24,128,1,float16,fp8,15,0.013316799700260163
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,24,24,128,1,float16,float16,31,0.01228799968957901
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,24,24,128,1,float16,fp8,31,0.013255999982357025
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,24,24,128,1,float16,float16,63,0.01220960021018982
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,24,24,128,1,float16,fp8,1,0.01290079951286316
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,24,24,128,1,float16,fp8,63,0.013081599771976472
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,24,24,128,1,float16,float16,127,0.01242239996790886
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,24,24,128,1,float16,fp8,127,0.013035200536251068
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,24,24,128,1,float16,float16,255,0.014260800182819366
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,24,24,128,1,float16,fp8,255,0.015110400319099427
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,24,24,128,1,float16,float16,511,0.017291200160980225
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,24,24,128,1,float16,fp8,511,0.018724800646305086
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,24,24,128,1,float16,float16,1023,0.019734400510787963
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,24,24,128,1,float16,float16,7,0.01223360002040863
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,24,24,128,1,float16,float16,31,0.012270399928092956
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,24,24,128,1,float16,float16,4095,0.053395199775695804
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,24,24,128,1,float16,fp8,4095,0.05150880217552185
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,24,24,128,1,float16,float16,1,0.03222880065441132
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,24,24,128,1,float16,float16,2047,0.019870400428771973
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,24,24,128,1,float16,fp8,1,0.036520001292228696
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,24,24,128,1,float16,float16,3,0.032094401121139524
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,24,24,128,1,float16,float16,7,0.03214559853076935
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,24,24,128,1,float16,fp8,7,0.03666079938411713
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,24,24,128,1,float16,float16,15,0.03203040063381195
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,24,24,128,1,float16,fp8,511,0.13499679565429687
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,24,24,128,1,float16,fp8,15,0.03660799860954285
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,24,24,128,1,float16,float16,31,0.03207040131092072
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,24,24,128,1,float16,fp8,31,0.03665120005607605
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,24,24,128,1,float16,float16,63,0.03227199912071228
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,24,24,128,1,float16,fp8,63,0.03652159869670868
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,24,24,128,1,float16,float16,127,0.036259201169013974
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,24,24,128,1,float16,fp8,127,0.03656319975852966
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,24,24,128,1,float16,float16,255,0.048871999979019164
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,24,24,128,1,float16,fp8,255,0.05087199807167053
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,24,24,128,1,float16,float16,511,0.06883040070533752
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,24,24,128,1,float16,fp8,511,0.07326239943504334
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,24,24,128,1,float16,fp8,4095,0.6429215908050537
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,24,24,128,1,float16,float16,1023,0.11715840101242066
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,24,24,128,1,float16,fp8,2047,0.031376001238822934
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,24,24,128,1,float16,fp8,1023,0.11128640174865723
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,24,24,128,1,float16,float16,2047,0.20567359924316406
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,24,24,128,1,float16,fp8,2047,0.1810096025466919
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,24,24,128,1,float16,float16,1,0.012211199849843979
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,24,24,128,1,float16,float16,4095,0.38297600746154786
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,24,24,128,1,float16,fp8,1,0.013055999577045441
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,24,24,128,1,float16,fp8,4095,0.3278336048126221
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,24,24,128,1,float16,fp8,3,0.03656800091266632
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,24,24,128,1,float16,float16,7,0.0124719999730587
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,24,24,128,1,float16,float16,15,0.01255200058221817
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,24,24,128,1,float16,fp8,15,0.01326719969511032
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,24,24,128,1,float16,fp8,1,0.013288000226020813
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,24,24,128,1,float16,fp8,31,0.013526399433612824
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,24,24,128,1,float16,float16,63,0.012665599584579468
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,24,24,128,1,float16,fp8,63,0.01350879967212677
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,24,24,128,1,float16,float16,127,0.012556800246238708
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,24,24,128,1,float16,fp8,127,0.013299199938774108
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,24,24,128,1,float16,float16,255,0.014252799749374389
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,24,24,128,1,float16,fp8,255,0.015559999644756317
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,24,24,128,1,float16,float16,511,0.017534400522708892
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,24,24,128,1,float16,fp8,511,0.01903039962053299
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,24,24,128,1,float16,float16,1023,0.030932798981666565
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,24,24,128,1,float16,fp8,1023,0.030140799283981324
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,24,24,128,1,float16,float16,2047,0.04580639898777008
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,24,24,128,1,float16,fp8,2047,0.045126399397850035
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,24,24,128,1,float16,float16,2047,0.03344480097293854
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,24,24,128,1,float16,fp8,4095,0.07076320052146912
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,24,24,128,1,float16,float16,1,0.09867680072784424
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,24,24,128,1,float16,fp8,1,0.11981920003890992
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,24,24,128,1,float16,float16,3,0.09859679937362671
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,24,24,128,1,float16,fp8,3,0.013084800541400909
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,24,24,128,1,float16,float16,3,0.01255359947681427
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,24,24,128,1,float16,fp8,3,0.11960480213165284
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,24,24,128,1,float16,fp8,7,0.013326400518417358
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,24,24,128,1,float16,float16,7,0.09804480075836182
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,24,24,128,1,float16,fp8,7,0.11990079879760743
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,24,24,128,1,float16,float16,15,0.09891039729118348
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,24,24,128,1,float16,fp8,15,0.11989120244979859
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,24,24,128,1,float16,float16,31,0.09946399927139282
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,24,24,128,1,float16,float16,31,0.012454400211572647
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,24,24,128,1,float16,fp8,31,0.12005599737167358
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,24,24,128,1,float16,float16,63,0.10600800514221191
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,24,24,128,1,float16,fp8,63,0.1267135977745056
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,24,24,128,1,float16,float16,127,0.10877439975738526
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,24,24,128,1,float16,fp8,127,0.13120319843292236
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,24,24,128,1,float16,float16,255,0.15048160552978515
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,24,24,128,1,float16,fp8,255,0.17030880451202393
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,24,24,128,1,float16,float16,511,0.23194239139556885
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,24,24,128,1,float16,fp8,511,0.2568016052246094
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,24,24,128,1,float16,float16,1023,0.4177584171295166
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,24,24,128,1,float16,fp8,1023,0.40180320739746095
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,24,24,128,1,float16,fp8,1,0.22314720153808593
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,24,24,128,1,float16,float16,3,0.18286720514297486
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,24,24,128,1,float16,float16,7,0.1824944019317627
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,24,24,128,1,float16,fp8,3,0.2257296085357666
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,24,24,128,1,float16,float16,2047,0.7748640060424805
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,24,24,128,1,float16,float16,15,0.18399200439453126
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,24,24,128,1,float16,fp8,1023,0.023923200368881226
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,24,24,128,1,float16,float16,4095,0.07449439764022828
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,24,24,128,1,float16,float16,31,0.19573119878768921
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,24,24,128,1,float16,fp8,31,0.22661120891571046
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,24,24,128,1,float16,float16,63,0.19664000272750853
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,24,24,128,1,float16,fp8,63,0.2395632028579712
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,24,24,128,1,float16,fp8,127,0.24391520023345947
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,24,24,128,1,float16,float16,255,0.28325281143188474
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,24,24,128,1,float16,fp8,255,0.32065279483795167
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,24,24,128,1,float16,float16,511,0.4524655818939209
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,24,24,128,1,float16,float16,1,0.3531951904296875
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,24,24,128,1,float16,fp8,511,0.4936336040496826
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,24,24,128,1,float16,fp8,1,0.4323567867279053
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,24,24,128,1,float16,fp8,3,0.43534078598022463
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,24,24,128,1,float16,float16,7,0.3539760112762451
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,24,24,128,1,float16,float16,1,0.18324480056762696
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,24,24,128,1,float16,fp8,7,0.43489761352539064
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,24,24,128,1,float16,float16,15,0.36605761051177976
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,24,24,128,1,float16,fp8,15,0.4356959819793701
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,24,24,128,1,float16,float16,31,0.3740511894226074
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,24,24,128,1,float16,fp8,2047,0.6797887802124023
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,24,24,128,1,float16,fp8,7,0.22528960704803466
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,24,24,128,1,float16,fp8,15,0.2255039930343628
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,24,24,128,1,float16,float16,63,0.3776256084442139
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,24,24,128,1,float16,fp8,63,0.46215038299560546
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,24,24,128,1,float16,float16,127,0.3872384071350098
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,24,24,128,1,float16,float16,1,0.016260799765586854
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,24,24,128,1,float16,float16,127,0.2006256103515625
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,24,24,128,1,float16,float16,3,0.0162432000041008
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,24,24,128,1,float16,fp8,1,0.017617599666118623
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,24,24,128,1,float16,float16,255,0.548199987411499
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,24,24,128,1,float16,float16,7,0.016392000019550323
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,24,24,128,1,float16,float16,15,0.016364799439907075
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,24,24,128,1,float16,fp8,7,0.017505599558353423
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,24,24,128,1,float16,fp8,255,0.6239791870117187
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,24,24,128,1,float16,float16,31,0.016315199434757233
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,24,24,128,1,float16,fp8,31,0.017739200592041017
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,24,24,128,1,float16,fp8,63,0.017768000066280366
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,24,24,128,1,float16,float16,127,0.016624000668525696
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,24,24,128,1,float16,fp8,127,0.017785599827766417
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,24,24,128,1,float16,float16,255,0.02011200040578842
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,24,24,128,1,float16,fp8,255,0.021670399606227873
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,24,24,128,1,float16,float16,3,0.353054404258728
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,24,24,128,1,float16,fp8,511,0.02914080023765564
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,24,24,128,1,float16,float16,1023,0.045105600357055665
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,24,24,128,1,float16,fp8,1023,0.04596480131149292
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,24,24,128,1,float16,float16,2047,0.07402240037918091
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,24,24,128,1,float16,fp8,2047,0.06982399821281433
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,24,24,128,1,float16,float16,4095,0.13186880350112914
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,24,24,128,1,float16,fp8,4095,0.12259360551834106
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,24,24,128,1,float16,fp8,31,0.45928478240966797
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,24,24,128,1,float16,fp8,127,0.4721327781677246
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,24,24,128,1,float16,fp8,3,0.017715199291706084
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,24,24,128,1,float16,fp8,1,0.8370960235595704
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,24,24,128,1,float16,fp8,15,0.01761920005083084
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,24,24,128,1,float16,float16,63,0.016433599591255187
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,24,24,128,1,float16,float16,3,0.698956823348999
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,24,24,128,1,float16,fp8,3,0.8385503768920899
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,24,24,128,1,float16,fp8,7,0.8413791656494141
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,24,24,128,1,float16,float16,15,0.7233312129974365
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,24,24,128,1,float16,fp8,15,0.8832752227783203
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,24,24,128,1,float16,float16,31,0.7264912128448486
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,24,24,128,1,float16,float16,63,0.7334720134735108
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,24,24,128,1,float16,fp8,63,0.8983440399169922
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,24,24,128,1,float16,float16,127,0.752953577041626
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,24,24,128,1,float16,float16,1,0.6918111801147461
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,24,24,128,1,float16,fp8,127,0.9132384300231934
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,24,24,128,1,float16,float16,1,1.4214719772338866
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,24,24,128,1,float16,float16,7,0.7131264209747314
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,24,24,128,1,float16,fp8,1,1.6872304916381835
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,24,24,128,1,float16,float16,511,0.028406399488449096
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,24,24,128,1,float16,float16,7,1.4223584175109862
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,24,24,128,1,float16,fp8,3,1.7117376327514648
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,24,24,128,1,float16,fp8,31,0.9006416320800781
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,24,24,128,1,float16,fp8,7,1.7785823822021485
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,24,24,128,1,float16,float16,15,1.4263968467712402
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,24,24,128,1,float16,float16,1,0.020204800367355346
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,24,24,128,1,float16,fp8,1,0.022392000257968902
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,24,24,128,1,float16,float16,3,0.020659199357032774
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,24,24,128,1,float16,float16,31,1.4324751853942872
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,24,24,128,1,float16,float16,7,0.020635199546813966
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,24,24,128,1,float16,fp8,7,0.022291199862957002
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,24,24,128,1,float16,float16,63,1.4440976142883302
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,24,24,128,1,float16,fp8,31,1.7634960174560548
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,24,24,128,1,float16,float16,31,0.020868800580501556
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,24,24,128,1,float16,fp8,31,0.022444799542427063
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,24,24,128,1,float16,float16,63,0.02066880017518997
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,24,24,128,1,float16,fp8,63,0.022259199619293214
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,24,24,128,1,float16,float16,127,0.020796799659729005
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,24,24,128,1,float16,fp8,127,0.022484800219535826
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,24,24,128,1,float16,float16,255,0.028465598821640015
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,24,24,128,1,float16,fp8,255,0.028491199016571045
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,24,24,128,1,float16,float16,511,0.039894399046897885
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,24,24,128,1,float16,fp8,511,0.04153760075569153
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,24,24,128,1,float16,float16,1023,0.06420639753341675
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,24,24,128,1,float16,fp8,1023,0.062009602785110474
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,24,24,128,1,float16,float16,2047,0.10968799591064453
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,24,24,128,1,float16,float16,3,1.4199184417724608
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,24,24,128,1,float16,float16,4095,0.19838080406188965
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,16,128,1,float16,float16,1,0.04012799859046936
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,24,24,128,1,float16,fp8,4095,0.1718816041946411
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,16,128,1,float16,float16,3,0.03978239893913269
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,16,128,1,float16,fp8,1,0.04596160054206848
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,16,128,1,float16,float16,7,0.03967519998550415
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,16,128,1,float16,fp8,3,0.045556798577308655
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,16,128,1,float16,fp8,7,0.04599039852619171
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,16,128,1,float16,float16,15,0.039766401052474976
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,16,128,1,float16,fp8,15,0.04602240025997162
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,16,128,1,float16,float16,31,0.0398144006729126
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,16,128,1,float16,fp8,31,0.04595359861850738
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,24,24,128,1,float16,fp8,15,1.7850543975830078
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,16,128,1,float16,fp8,63,0.04593279957771301
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,16,128,1,float16,float16,127,0.04569920003414154
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,16,128,1,float16,fp8,127,0.04655680060386658
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,24,24,128,1,float16,fp8,15,0.022489599883556366
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,16,128,1,float16,fp8,255,0.06393120288848878
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,16,128,1,float16,float16,511,0.08860639929771423
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,16,128,1,float16,fp8,511,0.09342079758644103
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,16,128,1,float16,float16,1023,0.15102239847183227
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,16,128,1,float16,fp8,1023,0.14246079921722413
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,16,128,1,float16,float16,2047,0.26937921047210694
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,16,128,1,float16,fp8,2047,0.23524160385131837
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,24,24,128,1,float16,fp8,2047,0.09695519804954529
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,16,128,1,float16,fp8,4095,0.433516788482666
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,16,128,1,float16,float16,8191,0.9973055839538574
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,16,128,1,float16,float16,1,0.01183520033955574
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,16,128,1,float16,float16,63,0.04012799859046936
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,16,128,1,float16,fp8,8191,0.8253376007080078
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,24,24,128,1,float16,float16,15,0.0206496000289917
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,16,128,1,float16,fp8,1,0.012276799976825714
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,16,128,1,float16,float16,255,0.06120319962501526
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,16,128,1,float16,float16,7,0.011788800358772278
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,16,128,1,float16,fp8,3,0.012510399520397186
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,16,128,1,float16,float16,15,0.01165120005607605
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,16,128,1,float16,fp8,15,0.012308800220489502
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,16,128,1,float16,float16,31,0.011790399998426437
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,16,128,1,float16,fp8,31,0.012329600006341934
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,16,128,1,float16,fp8,63,0.012334399670362473
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,16,128,1,float16,float16,127,0.011908800154924393
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,16,128,1,float16,fp8,127,0.012439999729394913
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,16,128,1,float16,float16,255,0.013827200233936309
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,16,128,1,float16,fp8,255,0.014424000680446625
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,24,24,128,1,float16,fp8,63,1.7620527267456054
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,16,128,1,float16,float16,511,0.016865600645542145
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,16,128,1,float16,float16,1023,0.016300800442695617
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,16,128,1,float16,fp8,1023,0.017089599370956422
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,16,128,1,float16,float16,2047,0.017972800135612487
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,16,128,1,float16,fp8,2047,0.01733600050210953
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,16,128,1,float16,float16,4095,0.021513600647449494
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,16,128,1,float16,fp8,4095,0.021193599700927733
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,16,128,1,float16,float16,4095,0.5112815856933594
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,16,128,1,float16,float16,8191,0.03046880066394806
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,16,128,1,float16,float16,1,0.012116800248622894
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,16,128,1,float16,fp8,1,0.012569600343704223
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,16,128,1,float16,fp8,8191,0.02719680070877075
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,16,128,1,float16,fp8,3,0.01247360035777092
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,16,128,1,float16,float16,7,0.012127999961376191
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,16,128,1,float16,fp8,7,0.012678399682044983
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,16,128,1,float16,float16,15,0.012108799815177918
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,16,128,1,float16,fp8,15,0.012545600533485413
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,16,128,1,float16,float16,31,0.012028799951076507
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,24,24,128,1,float16,fp8,3,0.022191999852657317
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,16,128,1,float16,fp8,31,0.012516799569129943
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,16,128,1,float16,float16,3,0.011790399998426437
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,16,128,1,float16,fp8,63,0.012744000554084778
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,16,128,1,float16,fp8,127,0.012545600533485413
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,16,128,1,float16,float16,255,0.013865600526332855
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,16,128,1,float16,fp8,255,0.01470080018043518
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,16,128,1,float16,float16,511,0.016889600455760954
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,16,128,1,float16,fp8,7,0.012337599694728852
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,16,128,1,float16,float16,1023,0.017955200374126436
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,16,128,1,float16,fp8,1023,0.01733119934797287
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,16,128,1,float16,float16,2047,0.019900800287723543
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,16,128,1,float16,fp8,2047,0.021083199977874757
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,16,128,1,float16,float16,4095,0.03020159900188446
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,16,128,1,float16,fp8,4095,0.026836800575256347
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,16,128,1,float16,float16,8191,0.04533120095729828
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,16,128,1,float16,fp8,511,0.017888000607490538
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,16,128,1,float16,fp8,8191,0.04137920141220093
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,16,128,1,float16,float16,1,0.02414720058441162
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,16,128,1,float16,fp8,1,0.026895999908447266
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,16,128,1,float16,float16,3,0.02425439953804016
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,16,128,1,float16,fp8,3,0.0269679993391037
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,16,128,1,float16,float16,7,0.024395200610160827
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,16,128,1,float16,fp8,7,0.02696160078048706
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,16,128,1,float16,float16,15,0.02433439940214157
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,16,128,1,float16,fp8,15,0.027107200026512145
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,16,128,1,float16,float16,31,0.024404799938201903
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,16,128,1,float16,fp8,31,0.027105599641799927
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,16,128,1,float16,float16,63,0.024376000463962554
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,16,128,1,float16,fp8,63,0.027163198590278624
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,16,128,1,float16,float16,127,0.024721600115299225
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,16,128,1,float16,fp8,127,0.027246400713920593
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,16,128,1,float16,float16,255,0.036487999558448794
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,16,128,1,float16,fp8,255,0.03519999980926514
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,16,128,1,float16,float16,511,0.04989759922027588
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,16,128,1,float16,float16,1023,0.08250719904899598
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,16,128,1,float16,fp8,1023,0.07723360061645508
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,16,128,1,float16,float16,2047,0.14063359498977662
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,16,128,1,float16,fp8,2047,0.12421760559082032
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,16,128,1,float16,float16,127,0.012198399752378464
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,16,128,1,float16,float16,4095,0.2583280086517334
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,16,128,1,float16,fp8,4095,0.2245568037033081
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,16,128,1,float16,float16,63,0.011846400052309036
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,16,128,1,float16,float16,1,0.012198399752378464
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,16,128,1,float16,fp8,1,0.012865599989891053
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,16,128,1,float16,float16,3,0.012060800194740295
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,16,128,1,float16,float16,8191,0.4963823795318604
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,16,128,1,float16,float16,7,0.012129600346088409
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,16,128,1,float16,float16,15,0.012148799747228623
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,16,128,1,float16,fp8,8191,0.41976637840270997
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,16,128,1,float16,float16,31,0.011999999731779098
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,16,128,1,float16,fp8,31,0.012681600451469422
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,16,128,1,float16,float16,63,0.01212640032172203
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,16,128,1,float16,fp8,63,0.01268640011548996
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,16,128,1,float16,float16,127,0.012195199728012085
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,16,128,1,float16,fp8,127,0.012612800300121307
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,16,128,1,float16,float16,255,0.013859200477600097
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,16,128,1,float16,fp8,255,0.014612799882888794
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,16,128,1,float16,float16,511,0.01716800034046173
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,16,128,1,float16,float16,3,0.012081599980592727
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,16,128,1,float16,float16,1023,0.019808000326156615
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,16,128,1,float16,fp8,1023,0.020891200006008147
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,16,128,1,float16,float16,2047,0.03033280074596405
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,16,128,1,float16,fp8,2047,0.02736159861087799
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,16,128,1,float16,float16,4095,0.044947201013565065
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,16,128,1,float16,float16,63,0.011948800086975098
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,16,128,1,float16,fp8,4095,0.041356799006462094
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,16,128,1,float16,float16,8191,0.07614079713821412
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,16,128,1,float16,fp8,511,0.018262399733066557
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,16,128,1,float16,float16,1,0.06993920207023621
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,16,128,1,float16,fp8,3,0.013006399571895599
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,16,128,1,float16,fp8,1,0.08317919969558715
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,16,128,1,float16,float16,3,0.07002559900283814
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,16,128,1,float16,fp8,7,0.012627199292182922
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,16,128,1,float16,fp8,3,0.08315680027008057
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,16,128,1,float16,fp8,15,0.012881599366664886
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,16,128,1,float16,float16,15,0.06986079812049865
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,16,128,1,float16,fp8,15,0.08321599960327149
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,16,128,1,float16,float16,31,0.07011520266532897
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,16,128,1,float16,fp8,31,0.08325759768486023
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,16,128,1,float16,float16,63,0.07534239888191223
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,16,128,1,float16,fp8,63,0.08374559879302979
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,16,128,1,float16,float16,127,0.07755200266838073
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,16,128,1,float16,fp8,127,0.09038879871368408
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,16,128,1,float16,float16,255,0.10712959766387939
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,16,128,1,float16,fp8,255,0.11772960424423218
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,16,128,1,float16,float16,511,0.16249760389328002
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,16,128,1,float16,fp8,511,0.018303999304771425
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,16,128,1,float16,fp8,511,0.1760848045349121
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,16,128,1,float16,float16,1023,0.2840384006500244
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,16,128,1,float16,fp8,1023,0.2720911979675293
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,16,128,1,float16,fp8,511,0.052086400985717776
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,16,128,1,float16,float16,2047,0.5293776035308838
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,16,128,1,float16,float16,1,0.1272528052330017
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,16,128,1,float16,fp8,2047,0.4581264019012451
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,16,128,1,float16,fp8,8191,0.06588640213012695
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,16,128,1,float16,fp8,1,0.15480159521102904
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,16,128,1,float16,float16,3,0.12758560180664064
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,16,128,1,float16,float16,7,0.1300495982170105
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,16,128,1,float16,fp8,7,0.15512640476226808
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,16,128,1,float16,float16,4095,1.0045472145080567
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,16,128,1,float16,fp8,4095,0.8550512313842773
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,16,128,1,float16,fp8,15,0.15506080389022828
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,16,128,1,float16,fp8,7,0.08333600163459778
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,16,128,1,float16,float16,63,0.13635679483413696
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,16,128,1,float16,fp8,31,0.15496159791946412
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,16,128,1,float16,fp8,63,0.16420799493789673
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,16,128,1,float16,float16,127,0.14056639671325682
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,16,128,1,float16,fp8,127,0.16970560550689698
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,16,128,1,float16,float16,255,0.19489439725875854
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,16,128,1,float16,fp8,255,0.22048640251159668
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,16,128,1,float16,float16,511,0.30597920417785646
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,16,128,1,float16,fp8,511,0.33734560012817383
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,16,128,1,float16,float16,1023,0.5507279872894287
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,16,128,1,float16,float16,1,0.2411423921585083
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,16,128,1,float16,fp8,1023,0.5320511817932129
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,16,128,1,float16,float16,3,0.24124319553375245
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,16,128,1,float16,fp8,1,0.2935983896255493
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,16,128,1,float16,fp8,3,0.29343199729919434
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,16,128,1,float16,float16,7,0.2416543960571289
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,16,128,1,float16,fp8,7,0.2936176061630249
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,16,128,1,float16,float16,15,0.2449712038040161
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,16,128,1,float16,fp8,15,0.2939728021621704
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,16,128,1,float16,float16,31,0.25505919456481935
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,16,128,1,float16,fp8,31,0.3042623996734619
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,16,128,1,float16,float16,63,0.26027679443359375
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,16,128,1,float16,fp8,63,0.3139280080795288
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,16,128,1,float16,float16,127,0.27084639072418215
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,16,128,1,float16,float16,7,0.06991360187530518
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,16,128,1,float16,fp8,127,0.31998720169067385
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,16,128,1,float16,float16,255,0.373142409324646
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,16,128,1,float16,float16,1,0.012452799826860428
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,16,128,1,float16,float16,31,0.13956799507141113
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,16,128,1,float16,float16,3,0.012697599828243256
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,16,128,1,float16,fp8,3,0.013020800054073333
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,16,128,1,float16,fp8,7,0.01308320015668869
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,16,128,1,float16,float16,511,0.5930704116821289
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,16,128,1,float16,float16,15,0.012383999675512314
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,16,128,1,float16,float16,31,0.012451200187206269
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,16,128,1,float16,fp8,15,0.013276800513267517
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,16,128,1,float16,fp8,31,0.013073599338531494
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,16,128,1,float16,fp8,63,0.013153600692749023
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,16,128,1,float16,float16,127,0.012646399438381195
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,16,128,1,float16,fp8,127,0.013166399300098419
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,16,128,1,float16,float16,255,0.014460800588130951
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,16,128,1,float16,fp8,255,0.015131199359893798
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,16,128,1,float16,float16,511,0.017543999850749968
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,16,128,1,float16,fp8,511,0.018680000305175783
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,16,128,1,float16,float16,1023,0.02972320020198822
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,16,128,1,float16,fp8,1023,0.026214399933815004
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,16,128,1,float16,float16,2047,0.04423680007457733
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,16,128,1,float16,fp8,2047,0.04047360122203827
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,16,128,1,float16,float16,4095,0.07455360293388366
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,16,128,1,float16,fp8,4095,0.06556959748268128
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,16,128,1,float16,float16,8191,0.13381279706954957
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,16,128,1,float16,float16,15,0.12864320278167723
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,16,128,1,float16,fp8,8191,0.11653759479522705
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,16,128,1,float16,fp8,1,0.01305440068244934
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,16,128,1,float16,float16,1,0.4694960117340088
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,16,128,1,float16,float16,7,0.012624000012874604
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,16,128,1,float16,fp8,1,0.5676368236541748
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,16,128,1,float16,fp8,511,0.6551424026489258
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,16,128,1,float16,float16,7,0.4690864086151123
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,16,128,1,float16,fp8,7,0.5693391799926758
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,16,128,1,float16,float16,15,0.4861775875091553
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,16,128,1,float16,fp8,15,0.5713263988494873
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,16,128,1,float16,fp8,31,0.6065919876098633
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,16,128,1,float16,fp8,3,0.15404319763183594
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,16,128,1,float16,fp8,63,0.6048943996429443
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,16,128,1,float16,float16,3,0.465883207321167
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,16,128,1,float16,float16,63,0.012608000636100769
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,16,128,1,float16,float16,31,0.4952752113342285
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,16,128,1,float16,float16,255,0.7280735969543457
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,16,128,1,float16,float16,63,0.4962575912475586
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,16,128,1,float16,fp8,255,0.8226448059082031
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,16,16,128,1,float16,float16,1,0.950438404083252
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,16,16,128,1,float16,fp8,1,1.111625576019287
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,16,128,1,float16,fp8,255,0.42436800003051756
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,16,16,128,1,float16,float16,3,0.9470527648925782
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,16,16,128,1,float16,fp8,3,1.1071056365966796
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,16,128,1,float16,fp8,3,0.5701183795928955
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,16,16,128,1,float16,float16,15,0.955627155303955
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,16,16,128,1,float16,fp8,7,1.1196304321289063
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,16,128,1,float16,fp8,127,0.6161903858184814
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,16,16,128,1,float16,fp8,15,1.1909423828125
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,16,16,128,1,float16,float16,31,0.9625455856323242
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,16,128,1,float16,float16,1,0.016059200465679168
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,16,128,1,float16,fp8,1,0.01717280000448227
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,16,128,1,float16,float16,3,0.01602720022201538
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,16,128,1,float16,fp8,3,0.01717599928379059
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,16,16,128,1,float16,fp8,31,1.1819104194641112
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,16,128,1,float16,float16,7,0.01600960046052933
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,16,128,1,float16,float16,15,0.016171200573444365
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,16,16,128,1,float16,float16,127,0.9992143630981445
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,16,128,1,float16,fp8,15,0.017430399358272553
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,16,128,1,float16,fp8,31,0.01743199974298477
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,16,128,1,float16,float16,31,0.01616799980401993
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,16,128,1,float16,float16,63,0.01631679981946945
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,16,16,128,1,float16,fp8,127,1.203929615020752
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,16,128,1,float16,float16,127,0.01648319959640503
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,16,128,1,float16,fp8,63,0.017550399899482726
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,16,128,1,float16,float16,255,0.020020799338817598
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,16,128,1,float16,fp8,255,0.021612800657749176
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,16,128,1,float16,float16,511,0.0296207994222641
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,16,128,1,float16,fp8,511,0.028905600309371948
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,16,128,1,float16,float16,1023,0.046804800629615784
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,16,128,1,float16,fp8,1023,0.044865599274635314
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,16,128,1,float16,float16,2047,0.07647200226783753
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,16,128,1,float16,fp8,2047,0.06903039813041686
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,16,128,1,float16,float16,4095,0.1373711943626404
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,16,128,1,float16,fp8,4095,0.11906720399856567
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,16,16,128,1,float16,float16,7,0.9528079986572265
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,12,12,128,1,float16,float16,1,0.03220799863338471
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,16,128,1,float16,float16,8191,0.25734720230102537
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,12,12,128,1,float16,fp8,1,0.03715519905090332
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,12,12,128,1,float16,float16,3,0.03251039981842041
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,12,12,128,1,float16,fp8,3,0.037190398573875426
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,12,12,128,1,float16,float16,7,0.03240639865398407
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,12,12,128,1,float16,fp8,7,0.037252798676490784
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,12,12,128,1,float16,float16,15,0.032481598854064944
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,12,12,128,1,float16,fp8,15,0.03731360137462616
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,12,12,128,1,float16,fp8,31,0.03731839954853058
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,12,12,128,1,float16,float16,63,0.03288159966468811
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,12,12,128,1,float16,fp8,63,0.03710399866104126
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,16,16,128,1,float16,float16,63,0.9689519882202149
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,12,12,128,1,float16,float16,127,0.0367792010307312
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,12,12,128,1,float16,float16,255,0.05037760138511658
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,12,12,128,1,float16,fp8,255,0.05140799880027771
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,12,12,128,1,float16,float16,511,0.07061759829521179
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,12,12,128,1,float16,fp8,511,0.07447199821472168
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,16,128,1,float16,fp8,127,0.018024000525474548
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,12,12,128,1,float16,float16,1023,0.11833759546279907
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,12,12,128,1,float16,fp8,1023,0.11222879886627198
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,12,12,128,1,float16,float16,2047,0.20724799633026122
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,12,12,128,1,float16,fp8,2047,0.18173439502716066
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,16,128,1,float16,float16,127,0.5099679946899414
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,12,12,128,1,float16,float16,4095,0.3882080078125
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,12,12,128,1,float16,float16,31,0.03251039981842041
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,12,12,128,1,float16,float16,8191,0.7662559986114502
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,12,12,128,1,float16,float16,1,0.011932799965143204
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,12,12,128,1,float16,float16,3,0.012004800140857697
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,12,12,128,1,float16,fp8,8191,0.6299215793609619
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,12,12,128,1,float16,fp8,1,0.012468799948692322
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,12,12,128,1,float16,fp8,3,0.012715199589729309
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,12,12,128,1,float16,fp8,7,0.0124719999730587
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,12,12,128,1,float16,float16,15,0.01202400028705597
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,16,16,128,1,float16,fp8,63,1.1875904083251954
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,12,12,128,1,float16,fp8,15,0.012324800342321396
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,12,12,128,1,float16,float16,63,0.011796800047159195
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,12,12,128,1,float16,fp8,63,0.01244639977812767
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,12,12,128,1,float16,fp8,31,0.012513600289821625
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,12,12,128,1,float16,fp8,127,0.012508800625801087
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,12,12,128,1,float16,float16,255,0.013347199559211731
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,12,12,128,1,float16,fp8,255,0.014484800398349762
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,12,12,128,1,float16,float16,511,0.016331200301647187
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,12,12,128,1,float16,fp8,511,0.01857919991016388
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,12,12,128,1,float16,float16,1023,0.017211200296878816
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,12,12,128,1,float16,fp8,1023,0.018352000415325163
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,12,12,128,1,float16,float16,2047,0.017615999281406402
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,12,12,128,1,float16,fp8,2047,0.018449600040912627
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,12,12,128,1,float16,float16,4095,0.020633600652217865
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,12,12,128,1,float16,fp8,4095,0.020316800475120543
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,16,128,1,float16,fp8,8191,0.21829440593719482
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,12,12,128,1,float16,float16,8191,0.02757120132446289
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,12,12,128,1,float16,float16,1,0.012223999947309494
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,12,12,128,1,float16,fp8,1,0.013099199533462525
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,12,12,128,1,float16,fp8,8191,0.02648960053920746
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,12,12,128,1,float16,float16,7,0.011779200285673141
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,12,12,128,1,float16,fp8,3,0.01284320056438446
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,16,128,1,float16,fp8,7,0.017416000366210938
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,12,12,128,1,float16,float16,7,0.01218239963054657
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,12,12,128,1,float16,fp8,127,0.03716000020503998
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,12,12,128,1,float16,fp8,7,0.013107199966907502
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,12,12,128,1,float16,float16,127,0.011939200013875962
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,12,12,128,1,float16,fp8,31,0.013070400059223174
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,12,12,128,1,float16,float16,63,0.012211199849843979
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,12,12,128,1,float16,fp8,63,0.012929600477218629
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,12,12,128,1,float16,float16,127,0.0121568001806736
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,12,12,128,1,float16,fp8,127,0.012860800325870513
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,12,12,128,1,float16,float16,255,0.013798399269580841
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,12,12,128,1,float16,fp8,255,0.01467680037021637
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,12,12,128,1,float16,float16,511,0.016785599291324615
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,12,12,128,1,float16,fp8,511,0.018486399948596955
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,12,12,128,1,float16,fp8,4095,0.33028159141540525
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,12,12,128,1,float16,fp8,1023,0.018702399730682374
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,12,12,128,1,float16,float16,2047,0.020612800121307374
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,12,12,128,1,float16,fp8,2047,0.02048639953136444
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,12,12,128,1,float16,float16,4095,0.0279664009809494
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,12,12,128,1,float16,fp8,4095,0.026232001185417176
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,12,12,128,1,float16,float16,8191,0.03980000019073486
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,12,12,128,1,float16,fp8,8191,0.038838401436805725
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,12,12,128,1,float16,float16,1,0.020772799849510193
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,12,12,128,1,float16,fp8,1,0.023038400709629057
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,12,12,128,1,float16,float16,3,0.02093279957771301
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,12,12,128,1,float16,float16,31,0.012001600116491318
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,12,12,128,1,float16,fp8,3,0.022993600368499754
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,12,12,128,1,float16,float16,15,0.012278400361537933
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,12,12,128,1,float16,fp8,7,0.02298240065574646
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,12,12,128,1,float16,float16,15,0.020737600326538087
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,12,12,128,1,float16,float16,31,0.012292800098657608
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,12,12,128,1,float16,fp8,31,0.022932800650596618
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,12,12,128,1,float16,float16,63,0.020739200711250304
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,12,12,128,1,float16,fp8,63,0.022815999388694764
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,12,12,128,1,float16,float16,127,0.020971199870109557
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,12,12,128,1,float16,fp8,127,0.022865599393844603
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,12,12,128,1,float16,float16,255,0.02946400046348572
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,12,12,128,1,float16,fp8,255,0.028726398944854736
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,12,12,128,1,float16,float16,511,0.04079039990901947
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,12,12,128,1,float16,fp8,511,0.04208320081233978
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,12,12,128,1,float16,float16,1023,0.017553600668907165
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,12,12,128,1,float16,fp8,1023,0.06228799819946289
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,12,12,128,1,float16,float16,3,0.012278400361537933
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,12,12,128,1,float16,fp8,2047,0.09782400131225585
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,12,12,128,1,float16,float16,4095,0.2006767988204956
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,12,12,128,1,float16,fp8,4095,0.1734655976295471
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,12,12,128,1,float16,float16,8191,0.38152480125427246
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,12,12,128,1,float16,float16,7,0.02093600034713745
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,12,12,128,1,float16,float16,1,0.012300799787044524
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,12,12,128,1,float16,fp8,15,0.012895999848842621
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,12,12,128,1,float16,float16,3,0.012316799908876418
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,12,12,128,1,float16,float16,31,0.02069759964942932
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,12,12,128,1,float16,fp8,3,0.012945599853992462
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,12,12,128,1,float16,float16,7,0.012383999675512314
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,12,12,128,1,float16,fp8,7,0.012966400384902954
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,12,12,128,1,float16,float16,31,0.012211199849843979
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,12,12,128,1,float16,fp8,31,0.013016000390052795
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,12,12,128,1,float16,float16,63,0.01223519966006279
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,12,12,128,1,float16,fp8,63,0.013148799538612366
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,12,12,128,1,float16,float16,127,0.012449599802494049
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,12,12,128,1,float16,fp8,127,0.013252800703048706
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,12,12,128,1,float16,float16,255,0.014219200611114502
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,12,12,128,1,float16,fp8,255,0.015209600329399109
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,12,12,128,1,float16,float16,511,0.017088000476360322
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,12,12,128,1,float16,fp8,511,0.018723200261592864
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,12,12,128,1,float16,float16,1023,0.020788800716400147
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,12,12,128,1,float16,fp8,1023,0.02491839975118637
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,12,12,128,1,float16,float16,1023,0.06505920290946961
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,12,12,128,1,float16,float16,2047,0.03416799902915955
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,12,12,128,1,float16,fp8,2047,0.03244960010051727
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,12,12,128,1,float16,float16,4095,0.05435839891433716
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,12,12,128,1,float16,fp8,4095,0.05232959985733032
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,12,12,128,1,float16,float16,2047,0.11002399921417236
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,12,12,128,1,float16,float16,8191,0.09374880194664001
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,12,12,128,1,float16,float16,1,0.05561439990997315
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,12,12,128,1,float16,fp8,1,0.0652671992778778
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,12,12,128,1,float16,float16,3,0.05577279925346375
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,12,12,128,1,float16,fp8,3,0.06528480052947998
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,12,12,128,1,float16,float16,7,0.055764800310134886
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,12,12,128,1,float16,fp8,7,0.06543359756469727
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,12,12,128,1,float16,float16,15,0.05577920079231262
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,12,12,128,1,float16,fp8,15,0.06552000045776367
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,12,12,128,1,float16,float16,31,0.05597760081291199
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,12,12,128,1,float16,fp8,8191,0.32275359630584716
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,12,12,128,1,float16,float16,63,0.06000000238418579
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,12,12,128,1,float16,float16,127,0.06294400095939637
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,12,12,128,1,float16,fp8,15,0.023108799755573273
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,12,12,128,1,float16,fp8,127,0.0707423985004425
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,12,12,128,1,float16,float16,255,0.08622400164604187
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,12,12,128,1,float16,float16,15,0.0121568001806736
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,12,12,128,1,float16,fp8,255,0.09253119826316833
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,12,12,128,1,float16,float16,511,0.12773599624633789
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,12,12,128,1,float16,fp8,511,0.13674559593200683
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,12,12,128,1,float16,float16,1023,0.220467209815979
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,12,12,128,1,float16,fp8,1023,0.20920319557189943
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,12,12,128,1,float16,float16,2047,0.39800319671630857
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,12,12,128,1,float16,fp8,2047,0.34841599464416506
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,12,12,128,1,float16,float16,1,0.09943680167198181
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,12,12,128,1,float16,float16,4095,0.764296007156372
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,12,12,128,1,float16,fp8,4095,0.6441215991973877
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,12,12,128,1,float16,fp8,1,0.12968159914016725
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,12,12,128,1,float16,float16,3,0.09942079782485962
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,12,12,128,1,float16,fp8,3,0.12743359804153442
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,12,12,128,1,float16,float16,7,0.09940159916877747
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,12,12,128,1,float16,fp8,7,0.12482080459594727
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,12,12,128,1,float16,float16,15,0.09935039877891541
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,12,12,128,1,float16,float16,31,0.1001520037651062
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,12,12,128,1,float16,fp8,31,0.12074559926986694
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,12,12,128,1,float16,fp8,63,0.06512960195541381
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,12,12,128,1,float16,fp8,1,0.012884800136089326
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,12,12,128,1,float16,float16,63,0.10695199966430664
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,12,12,128,1,float16,fp8,63,0.12723360061645508
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,12,12,128,1,float16,float16,127,0.11090559959411621
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,12,12,128,1,float16,fp8,127,0.13243520259857178
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,12,12,128,1,float16,float16,255,0.15445120334625245
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,12,12,128,1,float16,fp8,255,0.17185920476913452
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,12,12,128,1,float16,float16,511,0.23448479175567627
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,12,12,128,1,float16,fp8,511,0.259118390083313
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,12,12,128,1,float16,float16,1023,0.4253104209899902
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,12,12,128,1,float16,fp8,1023,0.40493121147155764
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,12,12,128,1,float16,fp8,1,0.22606399059295654
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,12,12,128,1,float16,float16,3,0.18605120182037355
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,12,12,128,1,float16,fp8,3,0.22585599422454833
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,12,12,128,1,float16,float16,7,0.186188805103302
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,12,12,128,1,float16,fp8,8191,0.08658720254898071
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,12,12,128,1,float16,fp8,7,0.22618720531463624
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,12,12,128,1,float16,float16,15,0.18680959939956665
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,12,12,128,1,float16,fp8,15,0.2255392074584961
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,12,12,128,1,float16,fp8,31,0.22902560234069824
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,12,12,128,1,float16,float16,63,0.19944800138473512
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,12,12,128,1,float16,fp8,63,0.24160640239715575
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,12,12,128,1,float16,fp8,15,0.12502399682998658
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,12,12,128,1,float16,float16,127,0.20587360858917236
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,12,12,128,1,float16,fp8,127,0.24843358993530273
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,12,12,128,1,float16,float16,255,0.2906496047973633
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,12,12,128,1,float16,fp8,15,0.013195200264453888
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,12,12,128,1,float16,fp8,255,0.325816011428833
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,12,12,128,1,float16,fp8,1,0.013400000333786011
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,12,12,128,1,float16,float16,3,0.012510399520397186
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,12,12,128,1,float16,float16,511,0.4493103981018066
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,12,12,128,1,float16,float16,7,0.012275200337171555
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,12,12,128,1,float16,fp8,511,0.4979536056518555
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,12,12,128,1,float16,float16,15,0.0124208003282547
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,12,12,128,1,float16,float16,31,0.012427199631929398
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,12,12,128,1,float16,fp8,15,0.013358399271965027
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,12,12,128,1,float16,fp8,31,0.013275200128555298
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,12,12,128,1,float16,float16,63,0.012511999905109405
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,12,12,128,1,float16,fp8,63,0.013344000279903411
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,12,12,128,1,float16,float16,127,0.01247519999742508
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,12,12,128,1,float16,fp8,127,0.013484799861907959
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,12,12,128,1,float16,float16,255,0.014094400405883788
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,12,12,128,1,float16,fp8,255,0.015526400506496429
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,12,12,128,1,float16,float16,511,0.017268800735473634
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,12,12,128,1,float16,float16,1023,0.031692799925804135
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,12,12,128,1,float16,fp8,1023,0.030755200982093812
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,12,12,128,1,float16,float16,2047,0.0470768004655838
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,12,12,128,1,float16,fp8,2047,0.04524320065975189
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,12,12,128,1,float16,float16,4095,0.07576320171356202
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,12,12,128,1,float16,fp8,4095,0.07112159729003906
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,12,12,128,1,float16,float16,31,0.19656480550765992
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,12,12,128,1,float16,float16,8191,0.13379520177841187
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,12,12,128,1,float16,fp8,8191,0.12382400035858154
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,12,12,128,1,float16,fp8,31,0.06552799940109252
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,12,12,128,1,float16,float16,1,0.35488319396972656
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,12,12,128,1,float16,fp8,1,0.4295152187347412
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,12,12,128,1,float16,float16,3,0.35457921028137207
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,12,12,128,1,float16,fp8,3,0.43041439056396485
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,12,12,128,1,float16,float16,7,0.3559151887893677
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,12,12,128,1,float16,fp8,3,0.01327199935913086
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,12,12,128,1,float16,fp8,7,0.429856014251709
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,12,12,128,1,float16,fp8,7,0.013329599797725678
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,12,12,128,1,float16,fp8,15,0.4307375907897949
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,12,12,128,1,float16,float16,31,0.374019193649292
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,12,12,128,1,float16,float16,1,0.1861407995223999
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,12,12,128,1,float16,fp8,63,0.4615200042724609
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,12,12,128,1,float16,fp8,511,0.019123199582099914
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,12,12,128,1,float16,float16,127,0.38972480297088624
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,12,12,128,1,float16,fp8,127,0.4647007942199707
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,12,12,128,1,float16,float16,255,0.5530752182006836
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,12,12,128,1,float16,fp8,255,0.6239552021026611
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,12,12,128,1,float16,fp8,1,0.8360383987426758
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,12,12,128,1,float16,float16,3,0.6987232208251953
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,12,12,128,1,float16,fp8,3,0.8364656448364258
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,12,12,128,1,float16,float16,7,0.7147984027862548
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,12,12,128,1,float16,float16,1,0.012579199671745301
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,12,12,128,1,float16,fp8,7,0.8379055976867675
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,12,12,128,1,float16,float16,15,0.7223455905914307
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,12,12,128,1,float16,float16,15,0.36810081005096434
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,12,12,128,1,float16,fp8,15,0.8786784172058105
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,12,12,128,1,float16,float16,31,0.7305247783660889
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,12,12,128,1,float16,float16,63,0.3793344020843506
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,12,12,128,1,float16,float16,1,0.016305600106716157
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,12,12,128,1,float16,fp8,1,0.017591999471187593
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,12,12,128,1,float16,float16,3,0.016355200111865996
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,12,12,128,1,float16,fp8,63,0.9004063606262207
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,12,12,128,1,float16,float16,7,0.01618400067090988
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,12,12,128,1,float16,float16,127,0.760152006149292
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,12,12,128,1,float16,fp8,7,0.0175135999917984
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,12,12,128,1,float16,float16,15,0.016369600594043732
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,12,12,128,1,float16,fp8,127,0.9099743843078614
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,12,12,128,1,float16,float16,1,0.6953296184539794
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,12,12,128,1,float16,fp8,15,0.01765120029449463
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,12,12,128,1,float16,float16,31,0.016313600540161132
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,12,12,128,1,float16,float16,63,0.01632480025291443
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,12,12,128,1,float16,fp8,63,0.017617599666118623
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,12,12,128,1,float16,fp8,127,0.017708800733089447
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,12,12,128,1,float16,float16,255,0.019940799474716185
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,12,12,128,1,float16,fp8,255,0.02178719937801361
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,12,12,128,1,float16,float16,511,0.02805280089378357
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,12,12,128,1,float16,fp8,511,0.029120001196861266
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,12,12,128,1,float16,float16,1023,0.04474560022354126
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,12,12,128,1,float16,fp8,1023,0.04443039894104004
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,12,12,128,1,float16,float16,2047,0.07376639842987061
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,12,12,128,1,float16,fp8,2047,0.06916959881782532
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,12,12,128,1,float16,float16,4095,0.13194080591201782
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,12,12,128,1,float16,fp8,4095,0.12144960165023803
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,12,12,128,1,float16,fp8,31,0.4608784198760986
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,12,12,128,1,float16,fp8,31,0.9001824378967285
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,12,12,128,1,float16,float16,8191,0.24038240909576417
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,8,128,1,float16,float16,1,0.024611200392246246
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,12,12,128,1,float16,fp8,3,0.017712000012397765
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,8,128,1,float16,fp8,1,0.027294400334358215
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,8,128,1,float16,float16,7,0.024353599548339842
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,8,128,1,float16,fp8,3,0.02755360007286072
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,8,128,1,float16,float16,15,0.024480000138282776
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,8,128,1,float16,fp8,15,0.02757439911365509
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,12,12,128,1,float16,fp8,31,0.01754560023546219
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,8,128,1,float16,float16,31,0.024481600522994994
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,12,12,128,1,float16,float16,127,0.016484799981117248
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,8,128,1,float16,fp8,31,0.027608001232147218
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,8,128,1,float16,float16,127,0.025209599733352663
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,8,128,1,float16,fp8,127,0.027625599503517152
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,8,128,1,float16,float16,255,0.036908799409866334
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,8,128,1,float16,fp8,255,0.036032000184059144
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,8,128,1,float16,float16,511,0.049881601333618165
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,8,128,1,float16,fp8,511,0.05225920081138611
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,8,128,1,float16,float16,1023,0.08286399841308593
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,8,128,1,float16,fp8,1023,0.07837120294570923
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,8,128,1,float16,float16,2047,0.1421615958213806
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,8,128,1,float16,fp8,2047,0.12467039823532104
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,12,12,128,1,float16,float16,63,0.7358191967010498
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,12,12,128,1,float16,fp8,8191,0.22416958808898926
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,8,128,1,float16,float16,3,0.024404799938201903
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,8,128,1,float16,fp8,4095,0.22334880828857423
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,8,128,1,float16,fp8,7,0.027427199482917785
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,8,128,1,float16,float16,1,0.01149279996752739
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,8,128,1,float16,fp8,1,0.012375999987125397
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,8,128,1,float16,float16,8191,0.5008368015289306
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,8,128,1,float16,float16,3,0.011630400270223617
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,8,128,1,float16,fp8,8191,0.4223311901092529
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,8,128,1,float16,float16,7,0.01159999966621399
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,8,128,1,float16,float16,15,0.011684799939393998
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,8,128,1,float16,fp8,3,0.012095999717712403
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,8,128,1,float16,fp8,63,0.02740960121154785
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,8,128,1,float16,float16,31,0.0115167997777462
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,8,128,1,float16,fp8,31,0.012161599844694138
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,8,128,1,float16,float16,63,0.011585599929094314
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,8,128,1,float16,float16,127,0.011598400026559829
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,8,128,1,float16,fp8,127,0.012361600250005721
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,8,128,1,float16,float16,255,0.013193599879741669
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,8,128,1,float16,fp8,255,0.014150400459766389
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,8,128,1,float16,float16,511,0.016121600568294526
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,8,128,1,float16,fp8,511,0.0177824005484581
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,8,128,1,float16,float16,1023,0.01579039990901947
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,8,128,1,float16,fp8,1023,0.016950400173664094
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,8,128,1,float16,float16,2047,0.01624000072479248
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,8,128,1,float16,fp8,2047,0.017136000096797943
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,8,128,1,float16,float16,4095,0.018007999658584593
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,8,128,1,float16,fp8,4095,0.017785599827766417
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,8,128,1,float16,float16,8191,0.021988800168037413
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,8,128,1,float16,fp8,8191,0.02173279970884323
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,8,128,1,float16,float16,16383,0.031673601269721983
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,8,128,1,float16,fp8,16383,0.02799200117588043
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,8,128,1,float16,float16,4095,0.26177918910980225
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,8,128,1,float16,fp8,1,0.01220960021018982
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,8,128,1,float16,float16,3,0.011459200084209442
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,8,128,1,float16,fp8,3,0.012334399670362473
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,8,128,1,float16,float16,7,0.011649599671363831
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,8,128,1,float16,fp8,7,0.012462399899959564
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,8,128,1,float16,float16,63,0.024489599466323852
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,8,128,1,float16,fp8,15,0.012319999933242797
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,8,128,1,float16,fp8,15,0.012366399914026261
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,8,128,1,float16,fp8,31,0.012337599694728852
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,8,128,1,float16,float16,63,0.011499200016260147
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,8,128,1,float16,fp8,63,0.012113600224256515
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,8,128,1,float16,fp8,63,0.012211199849843979
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,8,128,1,float16,float16,127,0.011393599957227708
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,8,128,1,float16,fp8,16383,0.8093055725097656
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,8,128,1,float16,float16,255,0.012995199859142303
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,8,128,1,float16,float16,511,0.01627040058374405
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,8,128,1,float16,fp8,255,0.01435520052909851
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,8,128,1,float16,fp8,511,0.01822720021009445
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,8,128,1,float16,fp8,1023,0.016942399740219116
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,8,128,1,float16,float16,1023,0.016075199842453
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,8,128,1,float16,float16,2047,0.017833599448204042
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,8,128,1,float16,fp8,2047,0.0177839994430542
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,8,128,1,float16,float16,4095,0.020982399582862854
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,8,128,1,float16,fp8,4095,0.02158239930868149
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,8,128,1,float16,float16,8191,0.02988480031490326
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,8,128,1,float16,fp8,8191,0.026814401149749756
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,8,128,1,float16,float16,16383,0.04524959921836853
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,8,128,1,float16,fp8,16383,0.04133439958095551
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,8,128,1,float16,float16,1,0.016612799465656282
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,8,128,1,float16,fp8,1,0.01791200041770935
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,8,128,1,float16,float16,3,0.016599999368190767
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,8,128,1,float16,fp8,3,0.018321600556373597
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,8,128,1,float16,float16,7,0.016603200137615202
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,8,128,1,float16,fp8,7,0.017819200456142426
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,8,128,1,float16,float16,15,0.01646080017089844
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,8,128,1,float16,fp8,15,0.017854399979114532
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,8,128,1,float16,float16,31,0.016596800088882445
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,8,128,1,float16,fp8,31,0.01801919937133789
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,8,128,1,float16,float16,63,0.016596800088882445
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,8,128,1,float16,fp8,63,0.01791519969701767
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,8,128,1,float16,float16,127,0.016667200624942778
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,8,128,1,float16,fp8,127,0.01788319945335388
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,8,128,1,float16,float16,255,0.020212799310684204
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,8,128,1,float16,fp8,255,0.022047999501228332
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,8,128,1,float16,fp8,127,0.012628799676895142
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,8,128,1,float16,float16,16383,0.9731504440307617
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,8,128,1,float16,float16,511,0.02982560098171234
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,8,128,1,float16,fp8,511,0.029662400484085083
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,8,128,1,float16,fp8,1023,0.045158401131629944
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,8,128,1,float16,float16,1023,0.0502240002155304
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,8,128,1,float16,fp8,2047,0.06904159784317017
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,8,128,1,float16,float16,4095,0.14591840505599976
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,8,128,1,float16,fp8,4095,0.11941280364990234
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,8,128,1,float16,float16,8191,0.2535504102706909
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,8,128,1,float16,fp8,8191,0.21843039989471436
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,8,128,1,float16,float16,1,0.011460799723863602
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,8,128,1,float16,fp8,16383,0.4146751880645752
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,8,128,1,float16,fp8,1,0.012359999865293504
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,8,128,1,float16,float16,3,0.01165279969573021
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,8,128,1,float16,fp8,3,0.012651200592517852
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,8,128,1,float16,float16,7,0.011841599643230439
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,8,128,1,float16,fp8,7,0.012692800164222718
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,8,128,1,float16,float16,15,0.011779200285673141
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,8,128,1,float16,fp8,15,0.012569600343704223
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,8,128,1,float16,float16,31,0.011601600050926208
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,8,128,1,float16,float16,31,0.011716800183057785
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,8,128,1,float16,fp8,31,0.012723200023174286
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,8,128,1,float16,float16,63,0.011928000301122666
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,8,128,1,float16,fp8,63,0.012572799623012543
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,8,128,1,float16,float16,127,0.012049599736928939
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,8,128,1,float16,fp8,127,0.012724800407886505
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,8,128,1,float16,float16,255,0.013833600282669067
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,8,128,1,float16,fp8,255,0.014641599357128143
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,8,128,1,float16,fp8,511,0.018192000687122345
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,8,128,1,float16,float16,1023,0.01767520010471344
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,8,128,1,float16,float16,2047,0.07675840258598328
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,8,128,1,float16,fp8,1023,0.017217600345611574
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,8,128,1,float16,float16,2047,0.019551999866962433
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,8,128,1,float16,float16,4095,0.030371201038360596
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,8,128,1,float16,float16,8191,0.044449600577354434
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,8,128,1,float16,fp8,8191,0.04166240096092224
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,8,128,1,float16,float16,16383,0.07509440183639526
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,8,128,1,float16,fp8,16383,0.0665120005607605
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,8,128,1,float16,float16,1,0.0402895987033844
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,8,128,1,float16,fp8,1,0.046798399090766905
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,8,128,1,float16,float16,16383,0.49144320487976073
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,8,128,1,float16,float16,3,0.04052160084247589
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,8,128,1,float16,float16,7,0.04029119908809662
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,8,128,1,float16,fp8,7,0.046828800439834596
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,8,128,1,float16,fp8,3,0.04679679870605469
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,8,128,1,float16,float16,15,0.04047519862651825
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,8,128,1,float16,float16,31,0.040591999888420105
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,8,128,1,float16,float16,15,0.01165440008044243
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,8,128,1,float16,fp8,31,0.04694400131702423
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,8,128,1,float16,fp8,63,0.04695520102977753
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,8,128,1,float16,float16,127,0.04671199917793274
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,8,128,1,float16,fp8,127,0.047502401471138
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,8,128,1,float16,float16,255,0.062254399061203
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,8,128,1,float16,float16,511,0.016731199622154237
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,8,128,1,float16,fp8,255,0.06466079950332641
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,8,128,1,float16,float16,511,0.0891103982925415
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,8,128,1,float16,fp8,511,0.09448639750480652
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,8,128,1,float16,fp8,2047,0.020839999616146087
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,8,128,1,float16,float16,1023,0.15272159576416017
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,8,128,1,float16,fp8,4095,0.02686080038547516
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,8,128,1,float16,fp8,1023,0.14453920125961303
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,8,128,1,float16,float16,2047,0.2698800086975098
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,8,128,1,float16,fp8,2047,0.2353663921356201
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,8,128,1,float16,float16,4095,0.5099040031433105
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,8,128,1,float16,fp8,4095,0.43576478958129883
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,8,128,1,float16,float16,1,0.07085760235786438
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,8,128,1,float16,fp8,1,0.08483999967575073
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,8,128,1,float16,float16,3,0.0712336003780365
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,8,128,1,float16,fp8,3,0.08489760160446166
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,8,128,1,float16,fp8,7,0.08487359881401062
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,8,128,1,float16,float16,8191,0.9869327545166016
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,8,128,1,float16,float16,15,0.07117599844932557
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,8,128,1,float16,fp8,15,0.04678559899330139
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,8,128,1,float16,float16,31,0.07141919732093811
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,8,128,1,float16,fp8,15,0.0902463972568512
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,8,128,1,float16,fp8,31,0.08522560000419617
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,8,128,1,float16,float16,63,0.04084959924221039
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,8,128,1,float16,float16,127,0.07906399965286255
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,8,128,1,float16,fp8,127,0.09159520268440247
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,8,128,1,float16,float16,255,0.10745439529418946
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,8,128,1,float16,fp8,255,0.11925280094146729
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,8,128,1,float16,float16,511,0.1641983985900879
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,8,128,1,float16,fp8,511,0.17858879566192626
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,8,128,1,float16,float16,1023,0.28671679496765134
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,8,128,1,float16,fp8,1023,0.2753968000411987
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,8,128,1,float16,float16,1,0.011641599982976914
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,8,128,1,float16,float16,7,0.07100639939308166
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,8,128,1,float16,float16,1,0.1300480008125305
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,8,128,1,float16,fp8,8191,0.8376031875610351
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,8,128,1,float16,float16,2047,0.5247615814208985
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,8,128,1,float16,fp8,1,0.15771520137786865
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,8,128,1,float16,fp8,2047,0.459987211227417
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,8,128,1,float16,float16,3,0.12932319641113282
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,8,128,1,float16,float16,7,0.13007040023803712
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,8,128,1,float16,float16,63,0.0762880027294159
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,8,128,1,float16,fp8,3,0.1593951940536499
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,8,128,1,float16,fp8,63,0.08810399770736695
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,8,128,1,float16,fp8,7,0.157315194606781
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,8,128,1,float16,fp8,15,0.15771679878234862
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,8,128,1,float16,float16,31,0.1326416015625
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,8,128,1,float16,fp8,31,0.15881600379943847
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,8,128,1,float16,float16,63,0.13971359729766847
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,8,128,1,float16,float16,127,0.14384959936141967
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,8,128,1,float16,fp8,127,0.1695039987564087
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,8,128,1,float16,float16,255,0.20051839351654052
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,8,128,1,float16,fp8,255,0.22353439331054686
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,8,128,1,float16,float16,511,0.3094367980957031
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,8,128,1,float16,fp8,511,0.3424720048904419
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,8,128,1,float16,float16,1,0.012107200175523757
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,8,128,1,float16,float16,3,0.012135999649763108
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,8,128,1,float16,fp8,1,0.012878400087356568
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,8,128,1,float16,fp8,3,0.013007999956607818
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,8,128,1,float16,float16,7,0.012243200093507767
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,8,128,1,float16,fp8,7,0.013052800297737121
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,8,128,1,float16,float16,15,0.0121568001806736
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,8,128,1,float16,fp8,15,0.013012799620628356
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,8,128,1,float16,float16,31,0.01199520006775856
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,8,128,1,float16,float16,63,0.012115199863910676
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,8,128,1,float16,fp8,63,0.013012799620628356
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,8,128,1,float16,float16,127,0.012324800342321396
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,8,128,1,float16,fp8,127,0.013041600584983826
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,8,128,1,float16,float16,255,0.014035199582576752
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,8,128,1,float16,fp8,255,0.01509920060634613
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,8,128,1,float16,float16,511,0.017132799327373504
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,8,128,1,float16,fp8,511,0.018401600420475006
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,8,128,1,float16,float16,1023,0.01951040029525757
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,8,128,1,float16,fp8,1023,0.021249599754810333
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,8,128,1,float16,float16,2047,0.029636800289154053
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,8,128,1,float16,float16,4095,0.0445360004901886
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,8,128,1,float16,fp8,7,0.012280000001192093
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,8,128,1,float16,fp8,4095,0.04102399945259094
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,8,128,1,float16,float16,15,0.12952959537506104
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,8,128,1,float16,float16,8191,0.0758575975894928
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,8,128,1,float16,fp8,8191,0.06628479957580566
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,8,128,1,float16,float16,16383,0.13690880537033082
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,8,128,1,float16,fp8,63,0.16684160232543946
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,8,8,128,1,float16,float16,1,0.24222080707550048
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,8,8,128,1,float16,float16,3,0.24332640171051026
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,8,8,128,1,float16,fp8,1,0.29339520931243895
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,8,8,128,1,float16,fp8,3,0.2935328006744385
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,8,8,128,1,float16,float16,7,0.24349119663238525
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,8,8,128,1,float16,fp8,7,0.2938416004180908
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,8,8,128,1,float16,float16,15,0.2437983989715576
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,8,128,1,float16,float16,1023,0.5515600204467773
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,8,8,128,1,float16,fp8,15,0.2960319995880127
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,8,8,128,1,float16,float16,31,0.2561503887176514
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,8,8,128,1,float16,fp8,31,0.30378720760345457
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,8,128,1,float16,fp8,31,0.013052800297737121
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,8,8,128,1,float16,fp8,63,0.3159264087677002
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,8,8,128,1,float16,fp8,127,0.3187328100204468
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,8,8,128,1,float16,float16,255,0.37549118995666503
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,8,8,128,1,float16,fp8,255,0.4254096031188965
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,8,8,128,1,float16,float16,511,0.5940847873687745
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,8,128,1,float16,fp8,2047,0.027347201108932497
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,8,8,128,1,float16,fp8,511,0.6580543994903565
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,8,8,128,1,float16,float16,1,0.4673744201660156
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,8,8,128,1,float16,float16,3,0.4674799919128418
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,8,8,128,1,float16,fp8,1,0.5633279800415039
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,8,8,128,1,float16,float16,7,0.46941118240356444
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,8,128,1,float16,fp8,16383,0.11643359661102295
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,8,8,128,1,float16,fp8,3,0.5647264003753663
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,8,8,128,1,float16,fp8,7,0.5662911891937256
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,8,8,128,1,float16,fp8,15,0.5707583904266358
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,8,8,128,1,float16,float16,31,0.49271359443664553
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,8,8,128,1,float16,fp8,31,0.6070528030395508
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,8,8,128,1,float16,float16,63,0.49833121299743655
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,8,128,1,float16,fp8,1023,0.535964822769165
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,8,8,128,1,float16,fp8,63,0.6070496082305908
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,8,8,128,1,float16,fp8,127,0.6124063968658447
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,8,8,128,1,float16,float16,127,0.2663136005401611
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,8,128,1,float16,fp8,1,0.012984000146389008
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,8,128,1,float16,float16,3,0.011975999921560287
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,8,128,1,float16,fp8,3,0.013079999387264252
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,8,128,1,float16,float16,7,0.0121568001806736
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,8,128,1,float16,fp8,7,0.013027200102806091
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,8,128,1,float16,float16,15,0.012172800302505494
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,8,8,128,1,float16,float16,255,0.7241216182708741
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,8,128,1,float16,float16,31,0.012115199863910676
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,8,128,1,float16,float16,63,0.012100800126791
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,8,128,1,float16,fp8,31,0.013081599771976472
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,8,128,1,float16,fp8,63,0.013084800541400909
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,8,128,1,float16,float16,127,0.012547199428081513
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,8,128,1,float16,fp8,127,0.013383999466896057
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,8,128,1,float16,float16,255,0.01451839953660965
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,8,128,1,float16,fp8,255,0.015337599813938141
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,8,128,1,float16,float16,511,0.017528000473976135
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,8,128,1,float16,fp8,511,0.01897120028734207
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,8,128,1,float16,float16,1023,0.028987199068069458
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,8,128,1,float16,fp8,1023,0.026843199133872987
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,8,128,1,float16,float16,2047,0.04427199959754944
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,8,128,1,float16,fp8,2047,0.04153760075569153
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,8,128,1,float16,float16,4095,0.07471519708633423
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,8,128,1,float16,fp8,4095,0.0668287992477417
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,8,128,1,float16,float16,8191,0.13380639553070067
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,8,8,128,1,float16,float16,127,0.5118639945983887
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,8,128,1,float16,fp8,8191,0.11678080558776856
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,8,128,1,float16,float16,1,0.012177599966526032
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,8,8,128,1,float16,float16,63,0.26320960521698
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,4,128,1,float16,float16,1,0.017211200296878816
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,8,128,1,float16,float16,16383,0.254366397857666
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,4,128,1,float16,fp8,1,0.01865919977426529
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,8,128,1,float16,fp8,15,0.013036799430847169
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,4,128,1,float16,float16,7,0.016867199540138246
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,4,128,1,float16,fp8,3,0.0185248002409935
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,4,128,1,float16,fp8,7,0.018875199556350707
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,4,128,1,float16,fp8,15,0.018535999953746794
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,4,128,1,float16,float16,31,0.0172992005944252
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,4,128,1,float16,fp8,31,0.018467199802398682
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,4,128,1,float16,float16,63,0.017342400550842286
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,4,128,1,float16,fp8,63,0.018559999763965607
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,4,128,1,float16,float16,127,0.017374399304389953
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,4,128,1,float16,fp8,127,0.018568000197410582
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,4,128,1,float16,float16,255,0.020870399475097657
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,4,128,1,float16,fp8,255,0.02266560047864914
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,4,128,1,float16,float16,511,0.031060799956321716
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,4,128,1,float16,fp8,511,0.030115199089050294
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,4,128,1,float16,float16,1023,0.04814560115337372
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,4,128,1,float16,fp8,1023,0.045879998803138734
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,4,128,1,float16,float16,2047,0.07797120213508606
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,4,128,1,float16,fp8,2047,0.06988800168037415
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,4,128,1,float16,float16,4095,0.13791999816894532
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,8,8,128,1,float16,float16,15,0.4884175777435303
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,4,128,1,float16,float16,8191,0.25630080699920654
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,4,128,1,float16,fp8,8191,0.21716639995574952
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,4,128,1,float16,float16,16383,0.4986415863037109
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,4,128,1,float16,float16,3,0.017233599722385407
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,4,128,1,float16,fp8,16383,0.4100992202758789
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,4,128,1,float16,float16,1,0.011495999991893768
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,4,128,1,float16,float16,3,0.011351999640464783
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,4,128,1,float16,fp8,1,0.012195199728012085
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,4,128,1,float16,float16,7,0.011534400284290314
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,4,128,1,float16,fp8,7,0.01210239976644516
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,4,128,1,float16,float16,15,0.011326400190591812
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,4,128,1,float16,fp8,15,0.012060800194740295
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,4,128,1,float16,float16,31,0.011336000263690948
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,4,128,1,float16,fp8,31,0.012081599980592727
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,4,128,1,float16,float16,63,0.011433599889278412
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,4,128,1,float16,fp8,32767,0.8039119720458985
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,4,128,1,float16,fp8,63,0.012214399874210358
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,4,128,1,float16,float16,32767,0.9877743721008301
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,4,128,1,float16,fp8,127,0.012241599708795547
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,4,128,1,float16,float16,127,0.011327999830245971
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,4,128,1,float16,fp8,255,0.014105600118637086
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,4,128,1,float16,float16,255,0.015771199762821198
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,4,128,1,float16,fp8,511,0.01805119961500168
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,4,128,1,float16,fp8,1023,0.017846399545669557
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,4,128,1,float16,float16,2047,0.016686399281024934
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,4,128,1,float16,fp8,4095,0.11996639966964721
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,4,128,1,float16,fp8,2047,0.018116800487041472
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,4,128,1,float16,float16,4095,0.01713760048151016
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,4,128,1,float16,float16,8191,0.018561600148677825
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,4,128,1,float16,fp8,8191,0.018598400056362152
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,4,128,1,float16,float16,16383,0.02232320010662079
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,4,128,1,float16,fp8,16383,0.02256480008363724
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,4,128,1,float16,float16,32767,0.03177599906921387
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,4,128,1,float16,fp8,32767,0.030553600192070006
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,4,128,1,float16,float16,1,0.011454399675130844
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,4,128,1,float16,fp8,1,0.012217599898576736
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,4,128,1,float16,float16,3,0.011430399864912033
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,4,128,1,float16,fp8,3,0.01218239963054657
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,4,128,1,float16,float16,7,0.011326400190591812
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,4,128,1,float16,fp8,7,0.012169600278139115
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,4,128,1,float16,float16,15,0.011526399850845337
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,4,128,1,float16,fp8,15,0.012331199645996094
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,4,128,1,float16,float16,15,0.01690080016851425
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,4,128,1,float16,fp8,3,0.012057600170373916
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,4,128,1,float16,fp8,31,0.0122079998254776
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,4,128,1,float16,float16,63,0.011420799791812897
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,4,128,1,float16,fp8,63,0.012291199713945388
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,4,128,1,float16,float16,127,0.011432000249624253
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,4,128,1,float16,fp8,127,0.012305600196123123
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,4,128,1,float16,float16,255,0.013068799674510957
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,4,128,1,float16,fp8,255,0.014305600523948669
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,4,128,1,float16,float16,511,0.016143999993801117
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,4,128,1,float16,fp8,511,0.017902399599552154
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,4,128,1,float16,float16,1023,0.0168272003531456
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,4,128,1,float16,fp8,1023,0.017795200645923614
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,4,128,1,float16,fp8,2047,0.01778080016374588
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,4,128,1,float16,float16,4095,0.018830400705337525
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,4,128,1,float16,float16,511,0.015999999642372132
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,4,128,1,float16,float16,1023,0.020083199441432952
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,4,128,1,float16,float16,8191,0.02237280011177063
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,4,128,1,float16,fp8,8191,0.022256000339984892
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,4,128,1,float16,float16,16383,0.03145439922809601
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,4,128,1,float16,fp8,16383,0.028278398513793945
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,4,128,1,float16,float16,32767,0.047577598690986635
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,4,128,1,float16,fp8,32767,0.044119998812675476
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,4,128,1,float16,float16,1,0.012807999551296235
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,4,128,1,float16,fp8,1,0.013716800510883332
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,4,128,1,float16,float16,3,0.012772800028324127
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,4,128,1,float16,fp8,3,0.013689599931240082
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,4,128,1,float16,float16,7,0.012603199481964112
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,4,128,1,float16,fp8,7,0.013468800485134125
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,8,128,1,float16,fp8,16383,0.21393280029296874
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,4,128,1,float16,float16,15,0.01276639997959137
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,4,128,1,float16,float16,31,0.012836800515651703
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,4,128,1,float16,fp8,31,0.013643200695514678
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,4,128,1,float16,fp8,15,0.013475200533866883
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,8,8,128,1,float16,fp8,255,0.8227984428405761
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,4,128,1,float16,fp8,63,0.013603200018405915
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,4,128,1,float16,fp8,127,0.013814400136470794
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,4,128,1,float16,float16,255,0.014849600195884705
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,4,128,1,float16,fp8,255,0.015558399260044098
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,4,128,1,float16,float16,511,0.017774400115013123
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,4,128,1,float16,fp8,511,0.019383999705314636
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,4,128,1,float16,float16,2047,0.01695840060710907
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,4,128,1,float16,float16,1023,0.02954080104827881
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,4,128,1,float16,fp8,1023,0.026464000344276428
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,4,128,1,float16,float16,2047,0.044363200664520264
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,4,128,1,float16,fp8,2047,0.04048640131950378
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,4,128,1,float16,float16,4095,0.07582719922065735
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,4,128,1,float16,fp8,4095,0.01836480051279068
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,4,128,1,float16,float16,8191,0.13424160480499267
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,4,128,1,float16,fp8,4095,0.018216000497341157
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,4,128,1,float16,float16,16383,0.2528496026992798
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,4,128,1,float16,fp8,16383,0.21144480705261232
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,4,128,1,float16,fp8,32767,0.4048272132873535
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,4,128,1,float16,float16,1,0.011553599685430526
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,4,128,1,float16,fp8,1,0.012139199674129486
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,4,128,1,float16,float16,63,0.012851199507713318
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,4,128,1,float16,float16,3,0.011351999640464783
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,4,128,1,float16,fp8,3,0.012390399724245072
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,4,128,1,float16,fp8,7,0.01252640038728714
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,4,128,1,float16,float16,15,0.01170400008559227
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,4,128,1,float16,fp8,15,0.012547199428081513
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,4,128,1,float16,float16,31,0.011708799749612808
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,4,128,1,float16,fp8,31,0.012582400441169738
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,4,128,1,float16,float16,63,0.01178399994969368
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,4,128,1,float16,fp8,63,0.012539200484752655
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,4,128,1,float16,float16,127,0.011856000125408172
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,4,128,1,float16,fp8,127,0.012492799758911132
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,4,128,1,float16,float16,255,0.013619199395179749
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,4,128,1,float16,float16,511,0.01645279973745346
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,4,128,1,float16,fp8,511,0.018028800189495087
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,4,128,1,float16,float16,1023,0.016969600319862367
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,4,128,1,float16,fp8,8191,0.11662240028381347
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,4,128,1,float16,fp8,1023,0.01836480051279068
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,4,128,1,float16,float16,2047,0.018745599687099455
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,4,128,1,float16,float16,4095,0.0220223993062973
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,4,128,1,float16,fp8,4095,0.02221599966287613
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,4,128,1,float16,float16,8191,0.030780801177024843
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,4,128,1,float16,fp8,8191,0.02801760137081146
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,4,128,1,float16,float16,16383,0.04572319984436035
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,4,128,1,float16,fp8,16383,0.04211359918117523
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,4,128,1,float16,float16,32767,0.07729759812355042
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,4,128,1,float16,float16,32767,0.49092960357666016
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,4,128,1,float16,fp8,32767,0.06736000180244446
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,4,128,1,float16,float16,1,0.026020801067352294
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,4,128,1,float16,fp8,1,0.028825598955154418
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,4,128,1,float16,fp8,3,0.028832000494003297
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,4,128,1,float16,float16,31,0.011588799953460693
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,4,128,1,float16,float16,7,0.025678399205207824
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,4,128,1,float16,fp8,7,0.02911840081214905
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,4,128,1,float16,float16,127,0.012804800271987915
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,4,128,1,float16,fp8,15,0.029131200909614564
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,4,128,1,float16,float16,31,0.025950399041175843
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,4,128,1,float16,fp8,31,0.029164800047874452
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,4,128,1,float16,float16,63,0.026022401452064515
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,4,128,1,float16,float16,127,0.026038399338722228
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,4,128,1,float16,fp8,127,0.02911199927330017
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,4,128,1,float16,float16,255,0.038606399297714235
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,4,128,1,float16,fp8,255,0.01435679942369461
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,4,128,1,float16,fp8,255,0.03710240125656128
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,4,128,1,float16,fp8,4095,0.06576640009880066
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,4,128,1,float16,float16,511,0.05179679989814758
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,4,128,1,float16,fp8,511,0.05380319952964783
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,4,128,1,float16,float16,1023,0.0847648024559021
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,4,128,1,float16,float16,2047,0.1439344048500061
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,4,128,1,float16,fp8,2047,0.01828320026397705
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,4,128,1,float16,fp8,2047,0.1255839943885803
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,4,128,1,float16,float16,4095,0.26129438877105715
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,4,128,1,float16,fp8,4095,0.22417120933532714
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,4,128,1,float16,float16,8191,0.5034448146820069
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,4,128,1,float16,fp8,8191,0.4219632148742676
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,4,4,128,1,float16,float16,1,0.04315840005874634
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,4,128,1,float16,float16,3,0.025806400179862975
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,4,4,128,1,float16,fp8,1,0.04880160093307495
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,4,4,128,1,float16,float16,3,0.043145599961280826
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,4,4,128,1,float16,fp8,3,0.04885919988155365
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,4,4,128,1,float16,float16,7,0.04307680130004883
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,4,4,128,1,float16,fp8,7,0.04897280037403107
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,4,128,1,float16,fp8,63,0.029105600714683533
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,4,4,128,1,float16,fp8,15,0.04903840124607086
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,4,4,128,1,float16,float16,31,0.042955198884010316
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,4,4,128,1,float16,fp8,31,0.049188798666000365
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,4,4,128,1,float16,float16,63,0.04349119961261749
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,4,4,128,1,float16,fp8,63,0.04936800003051758
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,4,4,128,1,float16,float16,127,0.047916799783706665
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,4,128,1,float16,fp8,1023,0.0794704020023346
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,4,4,128,1,float16,fp8,127,0.04954879879951477
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,4,4,128,1,float16,float16,255,0.06435359716415405
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,4,4,128,1,float16,fp8,255,0.06630240082740783
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,4,4,128,1,float16,float16,511,0.09189599752426147
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,4,4,128,1,float16,fp8,511,0.09702879786491395
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,4,4,128,1,float16,float16,1023,0.1553663969039917
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,4,4,128,1,float16,fp8,1023,0.1463711977005005
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,4,4,128,1,float16,float16,2047,0.2737312078475952
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,4,4,128,1,float16,fp8,2047,0.23836638927459716
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,4,128,1,float16,float16,7,0.01151840016245842
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,4,128,1,float16,float16,15,0.02568959891796112
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,4,4,128,1,float16,fp8,4095,0.43611679077148435
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,4,4,128,1,float16,float16,1,0.07639520168304444
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,4,4,128,1,float16,fp8,1,0.08878399729728699
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,4,128,1,float16,fp8,16383,0.810086441040039
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,4,4,128,1,float16,float16,3,0.076528000831604
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,4,4,128,1,float16,float16,15,0.042952001094818115
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,4,4,128,1,float16,float16,15,0.07692319750785828
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,4,4,128,1,float16,fp8,15,0.08951359987258911
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,4,4,128,1,float16,float16,31,0.07674239873886109
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,4,4,128,1,float16,fp8,31,0.08958240151405335
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,4,4,128,1,float16,float16,63,0.08015040159225464
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,4,4,128,1,float16,fp8,63,0.09005439877510071
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,4,4,128,1,float16,float16,127,0.0819167971611023
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,4,4,128,1,float16,fp8,127,0.0953328013420105
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,4,4,128,1,float16,float16,255,0.11342079639434814
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,4,4,128,1,float16,fp8,255,0.12251039743423461
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,4,4,128,1,float16,float16,511,0.16778719425201416
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,4,4,128,1,float16,fp8,511,0.18321759700775148
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,4,4,128,1,float16,float16,1023,0.2929631948471069
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,4,4,128,1,float16,fp8,1023,0.2808176040649414
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,4,4,128,1,float16,float16,2047,0.5370719909667969
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,4,4,128,1,float16,fp8,2047,0.46492319107055663
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,4,128,1,float16,float16,1,0.012052799761295318
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,4,128,1,float16,fp8,1,0.012720000743865967
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,4,128,1,float16,float16,3,0.012078399956226348
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,4,128,1,float16,fp8,3,0.012540799379348756
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,4,128,1,float16,float16,7,0.012667199969291687
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,4,4,128,1,float16,fp8,3,0.08921279907226562
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,4,128,1,float16,float16,15,0.012547199428081513
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,4,4,128,1,float16,float16,7,0.07608159780502319
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,4,128,1,float16,fp8,15,0.012692800164222718
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,4,128,1,float16,fp8,31,0.012417600303888322
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,4,128,1,float16,float16,63,0.011926399916410447
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,4,4,128,1,float16,fp8,7,0.08975520133972167
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,4,128,1,float16,fp8,63,0.012617599964141846
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,4,128,1,float16,float16,127,0.012092799693346024
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,4,128,1,float16,fp8,127,0.012588800489902496
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,4,128,1,float16,fp8,255,0.01478399932384491
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,4,128,1,float16,float16,511,0.016502399742603303
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,4,128,1,float16,fp8,511,0.01823199987411499
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,4,128,1,float16,float16,1023,0.018742400407791137
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,4,128,1,float16,fp8,1023,0.018328000605106354
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,4,128,1,float16,float16,2047,0.020678399503231047
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,4,128,1,float16,fp8,2047,0.02228800058364868
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,4,128,1,float16,float16,4095,0.03115360140800476
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,4,128,1,float16,fp8,4095,0.028255999088287354
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,4,128,1,float16,float16,8191,0.04603840112686157
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,4,128,1,float16,fp8,8191,0.0422111988067627
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,4,128,1,float16,float16,16383,0.07718560099601746
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,4,128,1,float16,fp8,16383,0.06700639724731446
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,4,128,1,float16,float16,32767,0.1360111951828003
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,4,128,1,float16,fp8,32767,0.11786880493164062
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,4,4,128,1,float16,fp8,1,0.16215200424194337
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,4,4,128,1,float16,float16,3,0.13884479999542237
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,4,4,128,1,float16,fp8,3,0.16276479959487916
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,4,4,128,1,float16,fp8,7,0.16227999925613404
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,4,4,128,1,float16,float16,15,0.1382383942604065
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,4,4,128,1,float16,fp8,15,0.16268960237503052
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,4,4,128,1,float16,float16,31,0.13977760076522827
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,4,4,128,1,float16,float16,4095,0.5142127990722656
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,4,128,1,float16,float16,16383,0.9730575561523438
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,4,128,1,float16,fp8,7,0.01252799928188324
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,4,4,128,1,float16,float16,63,0.14616639614105226
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,4,128,1,float16,float16,31,0.012489599734544754
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,4,4,128,1,float16,float16,127,0.14652960300445556
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,4,4,128,1,float16,fp8,127,0.174180805683136
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,4,4,128,1,float16,float16,255,0.20461599826812743
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,4,4,128,1,float16,fp8,255,0.2275536060333252
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,4,4,128,1,float16,float16,511,0.31019840240478513
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,4,4,128,1,float16,fp8,511,0.3500335931777954
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,4,4,128,1,float16,float16,1,0.1378368020057678
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,4,4,128,1,float16,float16,1023,0.5555520057678223
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,4,4,128,1,float16,fp8,1023,0.5408847808837891
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,4,4,128,1,float16,float16,7,0.13881440162658693
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,4,4,128,1,float16,float16,3,0.2589936017990112
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,4,4,128,1,float16,float16,7,0.258955192565918
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,4,4,128,1,float16,fp8,3,0.30622880458831786
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,4,4,128,1,float16,fp8,7,0.3070080041885376
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,4,4,128,1,float16,float16,15,0.25922720432281493
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,4,4,128,1,float16,fp8,15,0.3070784091949463
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,4,4,128,1,float16,fp8,31,0.3179471969604492
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,4,4,128,1,float16,float16,63,0.2686239957809448
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,4,4,128,1,float16,fp8,63,0.17562719583511352
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,4,128,1,float16,float16,255,0.01387840062379837
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,4,4,128,1,float16,fp8,63,0.32625279426574705
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,4,4,128,1,float16,float16,127,0.2724607944488525
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,4,4,128,1,float16,fp8,127,0.3285808086395264
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,4,4,128,1,float16,float16,255,0.37841598987579345
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,4,4,128,1,float16,fp8,255,0.432696008682251
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,4,4,128,1,float16,float16,1,0.2605760097503662
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,4,128,1,float16,float16,1,0.011926399916410447
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,4,128,1,float16,fp8,1,0.012838399410247803
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,4,128,1,float16,float16,3,0.012174399942159653
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,4,128,1,float16,fp8,3,0.012726399302482604
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,4,4,128,1,float16,fp8,1,0.3067663908004761
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,4,128,1,float16,float16,7,0.01215839982032776
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,4,128,1,float16,fp8,7,0.012720000743865967
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,4,128,1,float16,float16,15,0.011952000111341477
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,4,128,1,float16,float16,31,0.01199520006775856
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,4,4,128,1,float16,fp8,511,0.669268798828125
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,4,128,1,float16,fp8,31,0.01311199963092804
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,4,128,1,float16,float16,127,0.012187200039625168
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,4,128,1,float16,fp8,127,0.013043199479579926
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,4,128,1,float16,float16,255,0.013939200341701508
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,4,128,1,float16,fp8,255,0.015414400398731232
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,4,128,1,float16,float16,511,0.017030400037765504
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,4,128,1,float16,fp8,511,0.018768000602722167
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,4,128,1,float16,float16,1023,0.020844799280166627
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,4,128,1,float16,fp8,1023,0.022230400145053862
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,4,128,1,float16,float16,2047,0.03129439949989319
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,4,128,1,float16,fp8,2047,0.028095999360084535
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,4,128,1,float16,float16,4095,0.046435201168060304
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,4,128,1,float16,fp8,4095,0.0424703985452652
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,4,128,1,float16,float16,8191,0.07855839729309082
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,4,128,1,float16,fp8,8191,0.06744959950447083
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,4,128,1,float16,float16,16383,0.13842240571975709
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,4,128,1,float16,fp8,16383,0.11881279945373535
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,4,128,1,float16,float16,32767,0.2608191967010498
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,4,128,1,float16,fp8,15,0.013368000090122224
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,4,128,1,float16,float16,63,0.01218239963054657
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,4,128,1,float16,fp8,32767,0.21557118892669677
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,4,128,1,float16,fp8,63,0.01284320056438446
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,3,0.013097600638866424
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,2,2,128,1,float16,fp8,1,0.014340800046920777
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,7,0.013598400354385375
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,2,2,128,1,float16,fp8,7,0.014342400431632995
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,15,0.013660800457000733
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,2,2,128,1,float16,fp8,15,0.014433600008487701
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,2,2,128,1,float16,fp8,31,0.014532800018787383
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,63,0.013734400272369385
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,2,2,128,1,float16,fp8,63,0.0143327996134758
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,127,0.01356319934129715
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,2,2,128,1,float16,fp8,127,0.014256000518798828
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,255,0.01563999950885773
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,2,2,128,1,float16,fp8,255,0.01629440039396286
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,511,0.018676799535751343
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,2,2,128,1,float16,fp8,511,0.020203199982643128
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,1023,0.030302399396896364
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,2,2,128,1,float16,fp8,1023,0.027419200539588927
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,2047,0.045131200551986696
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,4095,0.07585440278053283
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,2,2,128,1,float16,fp8,2047,0.0401856005191803
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,2,2,128,1,float16,fp8,4095,0.06640639901161194
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,8191,0.13518240451812744
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,2,2,128,1,float16,fp8,8191,0.11647200584411621
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,4,4,128,1,float16,float16,511,0.594704008102417
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,16383,0.2544480085372925
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,2,2,128,1,float16,fp8,16383,0.21380960941314697
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,2,2,128,1,float16,fp8,3,0.01451520025730133
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,32767,0.4915264129638672
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,4,4,128,1,float16,float16,31,0.26510560512542725
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,2,2,128,1,float16,fp8,32767,0.40782718658447265
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,1,0.011435200273990632
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,3,0.011451199650764465
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,4,4,128,1,float16,fp8,31,0.1632192015647888
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,7,0.011364799737930299
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,15,0.011363200098276138
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,2,2,128,1,float16,fp8,15,0.012241599708795547
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,2,2,128,1,float16,fp8,7,0.012035199999809265
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,31,0.011475200206041336
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,2,2,128,1,float16,fp8,31,0.01202239990234375
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,63,0.011481600254774094
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,2,2,128,1,float16,fp8,63,0.012015999853610992
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,127,0.011271999776363372
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,2,2,128,1,float16,fp8,127,0.011999999731779098
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,2,2,128,1,float16,fp8,65535,0.7945551872253418
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,255,0.012895999848842621
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,2,2,128,1,float16,fp8,255,0.013963200151920319
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,2,2,128,1,float16,fp8,511,0.01799200028181076
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,511,0.01594880074262619
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,2,2,128,1,float16,fp8,1023,0.017723199725151063
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,2,2,128,1,float16,fp8,2047,0.018041600286960603
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,2047,0.01687999963760376
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,4095,0.017161600291728973
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,2,2,128,1,float16,fp8,4095,0.01807200014591217
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,8191,0.01889439970254898
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,2,2,128,1,float16,fp8,8191,0.019868800044059755
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,16383,0.0204815998673439
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,2,2,128,1,float16,fp8,16383,0.02034880071878433
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,32767,0.02428639978170395
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,2,2,128,1,float16,fp8,32767,0.024531200528144836
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,65535,0.034143999218940735
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,2,2,128,1,float16,fp8,65535,0.032523199915885925
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,1,0.01146399974822998
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,2,2,128,1,float16,fp8,1,0.012076800316572189
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,3,0.011318399757146835
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,1,0.013342399895191193
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,2,2,128,1,float16,fp8,3,0.012099199742078782
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,7,0.011462400108575821
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,2,2,128,1,float16,fp8,7,0.012092799693346024
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,15,0.011547199636697768
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,31,0.011497599631547928
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,31,0.0135343998670578
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,63,0.011310400068759918
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,2,2,128,1,float16,fp8,1,0.012052799761295318
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,2,2,128,1,float16,fp8,3,0.012099199742078782
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,2,2,128,1,float16,fp8,127,0.012111999839544297
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,255,0.013121600449085235
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,65535,0.9580160140991211
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,511,0.01618880033493042
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,1023,0.016760000586509706
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,2,2,128,1,float16,fp8,511,0.01806560009717941
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,1023,0.016927999258041383
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,2,2,128,1,float16,fp8,1023,0.01804320067167282
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,2,2,128,1,float16,fp8,2047,0.01815200001001358
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,4095,0.017308799922466277
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,2,2,128,1,float16,fp8,4095,0.018345600366592406
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,8191,0.02078080028295517
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,2,2,128,1,float16,fp8,8191,0.020334400236606598
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,16383,0.023771199584007262
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,2,2,128,1,float16,fp8,16383,0.02409279942512512
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,32767,0.033687999844551085
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,2,2,128,1,float16,fp8,32767,0.030156800150871278
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,65535,0.04915199875831604
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,2,2,128,1,float16,fp8,15,0.012104000151157378
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,2,2,128,1,float16,fp8,65535,0.04595839977264404
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,2,2,128,1,float16,fp8,1,0.013350400328636169
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,3,0.012780800461769104
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,2,2,128,1,float16,fp8,3,0.01356319934129715
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,7,0.012852799892425538
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,2,2,128,1,float16,fp8,7,0.013620799779891968
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,127,0.011369600147008895
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,2,2,128,1,float16,fp8,63,0.012015999853610992
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,2,2,128,1,float16,fp8,15,0.013449600338935852
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,31,0.012230399996042252
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,2,2,128,1,float16,fp8,31,0.013545599579811097
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,63,0.012345600128173827
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,2,2,128,1,float16,fp8,63,0.013484799861907959
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,2,2,128,1,float16,fp8,127,0.01356000006198883
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,255,0.014382399618625641
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,2,2,128,1,float16,fp8,255,0.015449599921703338
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,511,0.01727679967880249
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,2,2,128,1,float16,fp8,511,0.019019199907779692
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,1023,0.021315200626850127
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,2047,0.016923199594020843
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,2,2,128,1,float16,fp8,1023,0.023051199316978455
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,2047,0.0322735995054245
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,2,2,128,1,float16,fp8,2047,0.02871679961681366
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,4095,0.047182399034500125
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,2,2,128,1,float16,fp8,4095,0.04246399998664856
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,8191,0.08104320168495179
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,16383,0.14023200273513795
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,2,2,128,1,float16,fp8,16383,0.11910879611968994
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,32767,0.2582560062408447
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,2,2,128,1,float16,fp8,32767,0.21600160598754883
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,1,0.01250240057706833
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,1,0.011532799899578094
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,2,2,128,1,float16,fp8,31,0.012240000069141388
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,2,2,128,1,float16,fp8,65535,0.4095151901245117
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,3,0.011417599767446518
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,7,0.011604800075292587
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,2,2,128,1,float16,fp8,7,0.012406399846076966
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,15,0.012830400466918945
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,15,0.011609599739313126
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,2,2,128,1,float16,fp8,15,0.012323199957609176
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,31,0.011603199690580369
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,2,2,128,1,float16,fp8,31,0.012191999703645706
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,2,2,128,1,float16,fp8,255,0.014246399700641631
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,2,2,128,1,float16,fp8,63,0.012268800288438797
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,127,0.012412799894809723
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,127,0.011555200070142746
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,2,2,128,1,float16,fp8,127,0.012142399698495865
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,255,0.013140800595283508
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,2,2,128,1,float16,fp8,255,0.014345599710941315
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,511,0.016171200573444365
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,2,2,128,1,float16,fp8,511,0.018161599338054658
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,1023,0.017470400035381316
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,2,2,128,1,float16,fp8,1023,0.01812639981508255
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,2047,0.01728159934282303
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,2,2,128,1,float16,fp8,2047,0.018347199261188506
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,4095,0.0191103994846344
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,2,2,128,1,float16,fp8,4095,0.018617600202560425
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,8191,0.024060800671577454
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,2,2,128,1,float16,fp8,8191,0.02422720044851303
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,16383,0.0333840012550354
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,2,2,128,1,float16,fp8,16383,0.03000960052013397
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,32767,0.049099200963974
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,2,2,128,1,float16,fp8,32767,0.04604159891605377
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,65535,0.0797648012638092
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,2,2,128,1,float16,float16,1,0.01820479929447174
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,2,2,128,1,float16,fp8,65535,0.07326239943504334
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,2,2,128,1,float16,fp8,8191,0.06912320256233215
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,2,2,128,1,float16,float16,3,0.01876160055398941
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,2,2,128,1,float16,float16,7,0.018438400328159334
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,2,2,128,1,float16,fp8,7,0.020343999564647674
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,2,2,128,1,float16,float16,15,0.01876640021800995
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,2,2,128,1,float16,fp8,15,0.02048960030078888
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,2,2,128,1,float16,float16,31,0.01863519996404648
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,2,2,128,1,float16,fp8,31,0.02027360051870346
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,2,2,128,1,float16,float16,63,0.018611200153827667
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,2,2,128,1,float16,fp8,63,0.020257599651813507
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,2,2,128,1,float16,float16,127,0.018772800266742707
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,2,2,128,1,float16,fp8,127,0.02038719952106476
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,2,2,128,1,float16,float16,255,0.022064000368118286
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,65535,0.49912638664245607
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,2,2,128,1,float16,fp8,255,0.0244159996509552
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,2,2,128,1,float16,fp8,3,0.01218400001525879
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,2,2,128,1,float16,float16,1023,0.04994400143623352
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,2,2,128,1,float16,fp8,1023,0.04675039947032929
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,63,0.011593600362539291
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,2,2,128,1,float16,float16,2047,0.08005599975585938
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,2,2,128,1,float16,fp8,2047,0.07081120014190674
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,2,2,128,1,float16,float16,4095,0.13993279933929442
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,2,2,128,1,float16,fp8,4095,0.12018239498138428
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,2,2,128,1,float16,float16,8191,0.2584304094314575
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,2,2,128,1,float16,fp8,1,0.019696000218391418
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,2,2,128,1,float16,fp8,3,0.020155200362205507
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,2,2,128,1,float16,float16,16383,0.49986720085144043
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,2,2,128,1,float16,fp8,16383,0.41460480690002444
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,2,2,128,1,float16,float16,1,0.02810559868812561
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,2,2,128,1,float16,fp8,1,0.03100000023841858
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,2,2,128,1,float16,float16,3,0.02795040011405945
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,2,2,128,1,float16,fp8,1,0.012332800030708312
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,2,2,128,1,float16,float16,32767,0.9492912292480469
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,2,2,128,1,float16,float16,7,0.028188800811767577
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,2,2,128,1,float16,fp8,511,0.031758400797843936
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,2,2,128,1,float16,fp8,7,0.03255040049552917
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,2,2,128,1,float16,fp8,32767,0.8234047889709473
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,2,2,128,1,float16,fp8,15,0.031244799494743347
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,2,2,128,1,float16,float16,31,0.029262399673461913
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,2,2,128,1,float16,float16,63,0.028233599662780762
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,2,2,128,1,float16,fp8,63,0.03177120089530945
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,2,2,128,1,float16,fp8,127,0.03206560015678406
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,2,2,128,1,float16,float16,255,0.04112319946289063
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,2,2,128,1,float16,fp8,255,0.04051199853420258
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,2,2,128,1,float16,float16,511,0.05538240075111389
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,2,2,128,1,float16,fp8,511,0.057334399223327635
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,2,2,128,1,float16,fp8,8191,0.2191312074661255
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,2,2,128,1,float16,float16,1023,0.08823840022087097
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,2,2,128,1,float16,fp8,1023,0.08246399760246277
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,2,2,128,1,float16,float16,2047,0.14614880084991455
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,2,2,128,1,float16,fp8,2047,0.12826240062713623
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,2,2,128,1,float16,float16,4095,0.26358880996704104
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,2,2,128,1,float16,fp8,4095,0.22810239791870118
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,2,2,128,1,float16,float16,8191,0.5
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,2,2,128,1,float16,fp8,1,0.0542959988117218
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,2,2,128,1,float16,fp8,3,0.030768001079559328
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,2,2,128,1,float16,float16,3,0.04846400022506714
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,2,2,128,1,float16,float16,511,0.03404319882392883
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,2,2,128,1,float16,float16,7,0.048332801461219786
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,2,2,128,1,float16,float16,15,0.04831359982490539
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,2,2,128,1,float16,fp8,7,0.054992002248764035
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,2,2,128,1,float16,fp8,15,0.05422080159187317
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,2,2,128,1,float16,fp8,31,0.03229919970035553
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,2,2,128,1,float16,fp8,31,0.054385602474212646
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,2,2,128,1,float16,float16,63,0.04890879988670349
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,2,2,128,1,float16,fp8,63,0.05447040200233459
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,2,2,128,1,float16,float16,127,0.053065598011016846
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,2,2,128,1,float16,fp8,127,0.054955202341079715
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,2,2,128,1,float16,float16,255,0.07011680006980896
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,2,2,128,1,float16,fp8,255,0.07160800099372863
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,2,2,128,1,float16,float16,511,0.09816960096359253
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,2,2,128,1,float16,fp8,511,0.10275520086288452
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,2,2,128,1,float16,float16,1023,0.16206079721450806
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,2,2,128,1,float16,fp8,1023,0.15305440425872802
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,2,2,128,1,float16,float16,2047,0.27957758903503416
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,2,2,128,1,float16,fp8,2047,0.24546239376068116
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,2,2,128,1,float16,float16,1,0.04837920069694519
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,2,2,128,1,float16,float16,4095,0.5214096069335937
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,1,0.011811199784278869
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,2,2,128,1,float16,fp8,1,0.012559999525547028
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,3,0.011707200109958649
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,2,2,128,1,float16,fp8,3,0.01265919953584671
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,2,2,128,1,float16,float16,15,0.028230398893356323
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,2,2,128,1,float16,fp8,7,0.01316000074148178
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,15,0.011785600334405899
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,31,0.011726400256156922
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,2,2,128,1,float16,fp8,31,0.012494400143623352
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,2,2,128,1,float16,float16,31,0.048974400758743285
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,63,0.011649599671363831
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,2,2,128,1,float16,fp8,63,0.012486399710178375
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,127,0.01189119964838028
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,2,2,128,1,float16,fp8,127,0.012511999905109405
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,255,0.013391999900341034
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,2,2,128,1,float16,fp8,255,0.014659200608730317
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,511,0.016487999260425566
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,2,2,128,1,float16,fp8,511,0.01828480064868927
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,1023,0.017323200404644013
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,2,2,128,1,float16,fp8,1023,0.018328000605106354
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,2047,0.01889760047197342
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,2,2,128,1,float16,fp8,2047,0.018644799292087556
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,4095,0.02253440022468567
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,2,2,128,1,float16,fp8,4095,0.022785599529743194
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,8191,0.03309600055217743
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,2,2,128,1,float16,fp8,8191,0.030326399207115173
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,16383,0.04795680046081543
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,2,2,128,1,float16,fp8,16383,0.04443840086460114
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,32767,0.07903680205345154
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,2,2,128,1,float16,fp8,32767,0.06959199905395508
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,65535,0.1394544005393982
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,2,2,128,1,float16,fp8,4095,0.4422624111175537
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,2,2,128,1,float16,fp8,65535,0.12030400037765503
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,2,2,128,1,float16,float16,1,0.08276159763336181
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,2,2,128,1,float16,fp8,1,0.0944591999053955
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,2,2,128,1,float16,fp8,3,0.05447999835014343
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,2,2,128,1,float16,fp8,3,0.09453439712524414
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,2,2,128,1,float16,float16,7,0.08243039846420289
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,2,2,128,1,float16,fp8,15,0.013222399353981017
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,2,2,128,1,float16,fp8,7,0.09465600252151489
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,2,2,128,1,float16,float16,15,0.08279359936714173
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,2,2,128,1,float16,fp8,15,0.09526079893112183
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,2,2,128,1,float16,float16,31,0.0829967975616455
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,2,2,128,1,float16,float16,127,0.028067201375961304
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,2,2,128,1,float16,float16,63,0.08716639876365662
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,2,2,128,1,float16,fp8,63,0.09557759761810303
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,2,2,128,1,float16,fp8,127,0.10183520317077636
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,2,2,128,1,float16,float16,255,0.12046560049057006
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,2,2,128,1,float16,fp8,255,0.12924799919128419
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,2,2,128,1,float16,float16,511,0.17476160526275636
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,2,2,128,1,float16,fp8,511,0.19098080396652223
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,2,2,128,1,float16,float16,1023,0.2975471973419189
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,2,2,128,1,float16,fp8,1023,0.28845279216766356
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,2,2,128,1,float16,float16,2047,0.5385503768920898
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,2,2,128,1,float16,fp8,2047,0.47321438789367676
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,2,2,128,1,float16,float16,1,0.14873119592666625
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,2,2,128,1,float16,fp8,1,0.17273600101470948
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,2,2,128,1,float16,float16,3,0.1489375948905945
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,7,0.01157120019197464
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,2,2,128,1,float16,fp8,3,0.17248159646987915
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,2,2,128,1,float16,float16,7,0.14953600168228148
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,2,2,128,1,float16,fp8,7,0.17133760452270508
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,2,2,128,1,float16,fp8,15,0.17249280214309692
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,2,2,128,1,float16,float16,31,0.15010240077972412
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,2,2,128,1,float16,fp8,31,0.17185440063476562
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,2,2,128,1,float16,float16,127,0.09011039733886719
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,2,2,128,1,float16,float16,127,0.1602671980857849
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,2,2,128,1,float16,fp8,127,0.1855936050415039
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,2,2,128,1,float16,float16,255,0.21847679615020751
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,2,2,128,1,float16,fp8,255,0.2382767915725708
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,2,2,128,1,float16,float16,511,0.322323203086853
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,2,2,128,1,float16,fp8,511,0.36055679321289064
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,2,2,128,1,float16,float16,1023,0.5687935829162598
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,1,0.011772800236940384
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,2,2,128,1,float16,fp8,1023,0.5523087978363037
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,2,2,128,1,float16,fp8,8191,0.42812957763671877
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,2,2,128,1,float16,float16,3,0.08343039751052857
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,2,2,128,1,float16,fp8,3,0.012726399302482604
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,7,0.012055999785661697
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,2,2,128,1,float16,fp8,7,0.012585599720478059
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,15,0.011931200325489045
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,2,2,128,1,float16,fp8,15,0.012614400684833526
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,31,0.011915200203657151
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,2,2,128,1,float16,fp8,31,0.012625600397586822
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,63,0.011984000355005265
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,2,2,128,1,float16,fp8,31,0.09465759992599487
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,127,0.011879999935626984
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,2,2,128,1,float16,float16,63,0.15639840364456176
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,2,2,128,1,float16,fp8,127,0.012942400574684144
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,255,0.013628800213336945
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,2,2,128,1,float16,fp8,255,0.01440960019826889
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,511,0.016756799817085267
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,2,2,128,1,float16,fp8,511,0.018160000443458557
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,1023,0.018956799805164338
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,2,2,128,1,float16,fp8,1023,0.01857440024614334
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,2047,0.0209184005856514
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,4095,0.03145920038223267
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,2,2,128,1,float16,fp8,4095,0.028388801217079162
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,8191,0.048256000876426695
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,2,2,128,1,float16,fp8,8191,0.044577598571777344
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,16383,0.07984319925308228
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,2,2,128,1,float16,fp8,16383,0.06919519901275635
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,32767,0.14088159799575806
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,2,2,128,1,float16,fp8,32767,0.11908479928970336
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,65535,0.2618016004562378
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,2,2,128,1,float16,fp8,65535,0.21586720943450927
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,1,0.013459199666976928
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,3,0.013473600149154663
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,1,1,128,1,float16,fp8,1,0.01424960047006607
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,7,0.013265599310398103
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,1,1,128,1,float16,fp8,3,0.014129599928855896
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,1,1,128,1,float16,fp8,7,0.014227199554443359
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,1,1,128,1,float16,fp8,15,0.01414559930562973
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,2,2,128,1,float16,fp8,1,0.01252640038728714
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,31,0.01345279961824417
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,3,0.011761599779129028
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,1,1,128,1,float16,fp8,31,0.01417279988527298
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,127,0.013335999846458436
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,1,1,128,1,float16,fp8,63,0.013868799805641175
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,1,1,128,1,float16,fp8,127,0.014060799777507783
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,255,0.015382400155067444
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,1,1,128,1,float16,fp8,255,0.016387200355529784
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,511,0.018460799753665925
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,1,1,128,1,float16,fp8,511,0.019897599518299103
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,2,2,128,1,float16,fp8,63,0.18223199844360352
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,1023,0.02237440049648285
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,2,2,128,1,float16,fp8,2047,0.022708800435066224
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,1,1,128,1,float16,fp8,1023,0.023769600689411162
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,4095,0.04794079959392548
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,1,1,128,1,float16,fp8,4095,0.043433600664138795
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,8191,0.08124480247497559
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,1,1,128,1,float16,fp8,8191,0.07023040056228638
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,16383,0.1430896043777466
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,1,1,128,1,float16,fp8,16383,0.12355040311813355
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,15,0.013291199505329133
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,63,0.013143999874591828
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,1,1,128,1,float16,fp8,32767,0.22188799381256102
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,2,2,128,1,float16,float16,15,0.14931999444961547
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,2,2,128,1,float16,fp8,63,0.012507200241088867
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,65535,0.4925536155700684
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,1,1,128,1,float16,fp8,65535,0.41630401611328127
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,1,0.011403200030326844
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,2047,0.03277280032634735
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,1,1,128,1,float16,fp8,1,0.012104000151157378
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,3,0.011376000195741653
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,1,1,128,1,float16,fp8,3,0.011988800019025803
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,7,0.011233600229024887
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,1,1,128,1,float16,fp8,7,0.012272000312805176
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,15,0.011257600039243698
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,1,1,128,1,float16,fp8,15,0.01196959987282753
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,31,0.011430399864912033
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,1,1,128,1,float16,fp8,31,0.012070400267839431
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,63,0.011283200234174728
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,127,0.011326400190591812
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,1,1,128,1,float16,fp8,127,0.012003199756145477
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,255,0.013006399571895599
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,1,1,128,1,float16,fp8,255,0.01417279988527298
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,511,0.0158160001039505
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,1,1,128,1,float16,fp8,511,0.01823520064353943
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,1,1,128,1,float16,fp8,131071,0.8027935981750488
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,2047,0.016748799383640288
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,4095,0.016763199865818024
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,1,1,128,1,float16,fp8,2047,0.01809920072555542
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,1,1,128,1,float16,fp8,4095,0.01785600036382675
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,1,1,128,1,float16,fp8,8191,0.0199072003364563
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,16383,0.022224000096321105
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,1,1,128,1,float16,fp8,16383,0.023425599932670592
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,32767,0.02401600033044815
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,1,1,128,1,float16,fp8,32767,0.02417439967393875
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,65535,0.02760159969329834
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,32767,0.26230559349060056
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,1,1,128,1,float16,fp8,65535,0.027563199400901794
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,1,1,128,1,float16,fp8,131071,0.03588480055332184
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,1,0.011419200152158738
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,1,1,128,1,float16,fp8,1,0.012249600142240524
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,3,0.011432000249624253
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,1,1,128,1,float16,fp8,3,0.012201599776744843
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,7,0.011427199840545655
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,1,1,128,1,float16,fp8,7,0.012121599912643433
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,15,0.011217600107192994
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,1,1,128,1,float16,fp8,15,0.012030400335788727
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,31,0.011292800307273865
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,1,1,128,1,float16,fp8,31,0.012004800140857697
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,63,0.011451199650764465
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,1,1,128,1,float16,fp8,63,0.012281599640846252
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,127,0.011548800021409988
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,1,1,128,1,float16,fp8,127,0.012246400117874146
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,1,1,128,1,float16,fp8,63,0.01231520026922226
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,131071,0.9660096168518066
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,1,1,128,1,float16,fp8,255,0.014251199364662171
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,1,1,128,1,float16,fp8,511,0.018116800487041472
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,1,1,128,1,float16,fp8,1023,0.017902399599552154
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,1023,0.016681599617004394
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,1,1,128,1,float16,fp8,1023,0.018216000497341157
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,1,1,128,1,float16,fp8,2047,0.0182096004486084
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,8191,0.01887200027704239
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,1,1,128,1,float16,fp8,4095,0.018267199397087097
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,8191,0.01900320053100586
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,1,1,128,1,float16,fp8,8191,0.020110400021076204
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,16383,0.02383359968662262
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,1,1,128,1,float16,fp8,16383,0.023713600635528565
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,32767,0.02712480127811432
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,131071,0.03861759901046753
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,65535,0.03680799901485443
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,1,1,128,1,float16,fp8,65535,0.033025598526000975
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,131071,0.05281599760055542
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,1,1,128,1,float16,fp8,131071,0.049275198578834535
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,1,0.012529599666595458
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,1,1,128,1,float16,fp8,1,0.013227200508117676
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,3,0.012521600723266602
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,1,1,128,1,float16,fp8,3,0.013116799294948578
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,7,0.01231039986014366
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,1,1,128,1,float16,fp8,7,0.013153600692749023
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,1,1,128,1,float16,fp8,2047,0.029761600494384765
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,15,0.012291199713945388
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,1,1,128,1,float16,fp8,15,0.013415999710559845
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,255,0.013148799538612366
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,1,1,128,1,float16,fp8,31,0.013400000333786011
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,63,0.012518399953842163
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,1,1,128,1,float16,fp8,63,0.013049599528312684
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,1023,0.016672000288963318
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,511,0.016118399798870087
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,1,1,128,1,float16,fp8,127,0.013500800728797913
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,255,0.01454399973154068
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,2047,0.016857600212097167
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,1,1,128,1,float16,fp8,255,0.015449599921703338
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,511,0.01767359972000122
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,1,1,128,1,float16,fp8,511,0.018639999628067016
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,1023,0.01966879963874817
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,4095,0.01703999936580658
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,1,1,128,1,float16,fp8,2047,0.022972799837589264
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,1,1,128,1,float16,fp8,1023,0.019433599710464478
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,4095,0.0323168009519577
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,1,1,128,1,float16,fp8,4095,0.0288783997297287
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,8191,0.04923839867115021
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,1,1,128,1,float16,fp8,8191,0.044644799828529355
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,16383,0.08401119709014893
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,1,1,128,1,float16,fp8,16383,0.0732479989528656
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,32767,0.1444111943244934
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,1,1,128,1,float16,fp8,32767,0.12429599761962891
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,1,1,128,1,float16,fp8,32767,0.027811199426651
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,65535,0.2632256031036377
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,1,1,128,1,float16,fp8,65535,0.2226191997528076
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,1,0.011500799655914306
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,1,1,128,1,float16,fp8,1,0.012161599844694138
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,3,0.011534400284290314
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,1,1,128,1,float16,fp8,3,0.01220960021018982
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,1,1,128,1,float16,fp8,7,0.012246400117874146
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,15,0.011409600079059602
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,1,1,128,1,float16,fp8,131071,0.4161327838897705
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,1,1,128,1,float16,fp8,15,0.012127999961376191
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,1,1,128,1,float16,fp8,31,0.012132800370454788
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,31,0.011406400054693223
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,1,1,128,1,float16,fp8,63,0.012124799937009812
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,127,0.011585599929094314
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,1,1,128,1,float16,fp8,127,0.012316799908876418
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,255,0.013236799836158752
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,1,1,128,1,float16,fp8,255,0.014345599710941315
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,31,0.012326399981975555
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,1,1,128,1,float16,fp8,511,0.017867200076580048
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,1023,0.016862399876117706
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,127,0.012323199957609176
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,1,1,128,1,float16,fp8,1023,0.018111999332904815
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,2047,0.01680160015821457
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,1,1,128,1,float16,fp8,2047,0.018246400356292724
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,4095,0.017097599804401398
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,1,1,128,1,float16,fp8,4095,0.018347199261188506
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,8191,0.02050720006227493
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,1,1,128,1,float16,fp8,8191,0.02003040015697479
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,2047,0.021462400257587434
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,16383,0.027142399549484254
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,1,1,128,1,float16,fp8,16383,0.026976001262664796
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,1,1,128,1,float16,fp8,32767,0.03350560069084167
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,65535,0.05270559787750244
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,1,1,128,1,float16,fp8,65535,0.04916479885578155
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,131071,0.08421279788017273
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,1,1,128,1,float16,fp8,131071,0.07647519707679748
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,1,0.014470399916172027
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,1,1,128,1,float16,fp8,1,0.015358400344848634
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,1,1,128,1,float16,fp8,3,0.015307199954986573
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,7,0.014459200203418732
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,1,1,128,1,float16,fp8,7,0.015475200116634369
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,15,0.014612799882888794
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,1,1,128,1,float16,fp8,15,0.015531200170516967
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,7,0.011566399782896041
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,31,0.014388799667358398
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,1,1,128,1,float16,fp8,31,0.01626880019903183
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,63,0.014595200121402741
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,1,1,128,1,float16,fp8,63,0.016020800173282623
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,127,0.014483200013637542
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,1,1,128,1,float16,fp8,127,0.01604959964752197
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,1,1,128,1,float16,fp8,255,0.018119999766349794
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,511,0.02011680006980896
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,1,1,128,1,float16,fp8,511,0.021499200165271758
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,1023,0.03203839957714081
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,1,1,128,1,float16,fp8,1023,0.029660800099372865
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,511,0.016198399662971496
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,2047,0.047435200214385985
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,1,1,128,1,float16,fp8,2047,0.04303520023822784
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,4095,0.07755360007286072
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,1,1,128,1,float16,fp8,4095,0.06771680116653442
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,8191,0.13672959804534912
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,1,1,128,1,float16,fp8,8191,0.11762880086898804
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,16383,0.25529119968414304
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,32767,0.03693920075893402
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,32767,0.4867856025695801
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,3,0.014478400349617004
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,1,1,128,1,float16,fp8,32767,0.4084047794342041
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,1,1,128,1,float16,float16,1,0.02099040001630783
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,1,1,128,1,float16,float16,3,0.02101919949054718
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,1,1,128,1,float16,fp8,3,0.022551999986171724
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,1,1,128,1,float16,float16,7,0.021086399257183076
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,1,1,128,1,float16,fp8,7,0.02272319942712784
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,131071,0.4942944049835205
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,65535,0.9503104209899902
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,255,0.01637440025806427
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,1,1,128,1,float16,fp8,65535,0.8012928009033203
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,1,1,128,1,float16,fp8,31,0.02260479927062988
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,1,1,128,1,float16,float16,63,0.020955200493335723
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,1,1,128,1,float16,fp8,63,0.022809599339962006
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,1,1,128,1,float16,float16,127,0.021184000372886657
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,1,1,128,1,float16,fp8,127,0.02308479994535446
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,1,1,128,1,float16,float16,255,0.024401600658893585
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,1,1,128,1,float16,fp8,255,0.026731199026107787
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,1,1,128,1,float16,float16,511,0.03582879900932312
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,1,1,128,1,float16,fp8,511,0.03397760093212128
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,1,1,128,1,float16,float16,1023,0.052939200401306154
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,1,1,128,1,float16,fp8,1023,0.0499343991279602
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,1,1,128,1,float16,fp8,16383,0.21466400623321533
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,1,1,128,1,float16,float16,2047,0.08324959874153137
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,1,1,128,1,float16,fp8,2047,0.07267839908599853
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,1,1,128,1,float16,float16,4095,0.14228639602661133
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,1,1,128,1,float16,fp8,4095,0.12306560277938842
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,1,1,128,1,float16,fp8,1,0.02223680019378662
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,1,1,128,1,float16,float16,8191,0.25995678901672364
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,1,1,128,1,float16,fp8,8191,0.22258079051971436
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,1,1,128,1,float16,float16,1,0.03418079912662506
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,1,1,128,1,float16,float16,15,0.021132799983024596
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,1,1,128,1,float16,float16,3,0.03452320098876953
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,1,1,128,1,float16,float16,16383,0.49063358306884763
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,1,1,128,1,float16,fp8,3,0.03755039870738983
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,1,1,128,1,float16,float16,31,0.02322400063276291
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,1,1,128,1,float16,fp8,16383,0.41561598777770997
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,1,1,128,1,float16,fp8,7,0.03747040033340454
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,1,1,128,1,float16,float16,7,0.034016001224517825
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,1,1,128,1,float16,float16,15,0.034822401404380796
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,1,1,128,1,float16,float16,31,0.03420799970626831
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,1,1,128,1,float16,float16,63,0.034860798716545106
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,1,1,128,1,float16,fp8,63,0.03746399879455566
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,1,1,128,1,float16,float16,127,0.03480319976806641
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,1,1,128,1,float16,fp8,127,0.037441599369049075
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,1,1,128,1,float16,float16,255,0.04631359875202179
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,1,1,128,1,float16,fp8,255,0.045719999074935916
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,1,1,128,1,float16,float16,511,0.061956799030303954
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,1,1,128,1,float16,fp8,511,0.06268799901008607
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,1,1,128,1,float16,float16,1023,0.09545599818229675
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,1,1,128,1,float16,fp8,1023,0.08862879872322083
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,1,1,128,1,float16,float16,2047,0.15319520235061646
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,1,1,128,1,float16,fp8,2047,0.13576639890670777
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,1,1,128,1,float16,float16,4095,0.26963999271392824
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,1,1,128,1,float16,fp8,4095,0.23449440002441407
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,1,1,128,1,float16,float16,8191,0.4970384120941162
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,1,0.011748799681663513
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,1,1,128,1,float16,fp8,1,0.03732799887657166
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,3,0.01170239970088005
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,1,1,128,1,float16,fp8,15,0.02234079986810684
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,7,0.011646399646997452
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,63,0.011508800089359283
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,1,1,128,1,float16,fp8,7,0.012334399670362473
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,1,1,128,1,float16,fp8,15,0.012652799487113953
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,31,0.011495999991893768
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,1,1,128,1,float16,fp8,31,0.012636800110340119
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,63,0.01146719977259636
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,1,1,128,1,float16,fp8,63,0.012644800543785095
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,1,1,128,1,float16,fp8,127,0.012595200538635254
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,255,0.013420799374580383
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,1,1,128,1,float16,fp8,255,0.01451359987258911
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,511,0.016395199298858642
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,1,1,128,1,float16,fp8,511,0.018483200669288637
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,1023,0.017136000096797943
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,1,1,128,1,float16,fp8,1023,0.018406400084495546
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,2047,0.017433600127696992
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,1,1,128,1,float16,fp8,2047,0.018441599607467652
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,4095,0.019012799859046935
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,1,1,128,1,float16,fp8,4095,0.018787199258804323
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,8191,0.024111999571323393
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,1,1,128,1,float16,fp8,8191,0.024163199961185454
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,16383,0.03674240112304687
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,1,1,128,1,float16,fp8,16383,0.033395200967788696
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,32767,0.05294240117073059
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,1,1,128,1,float16,fp8,32767,0.049769601225852965
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,65535,0.08608800172805786
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,1,1,128,1,float16,fp8,8191,0.43085441589355467
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,1,1,128,1,float16,fp8,65535,0.07701280117034912
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,131071,0.14881279468536376
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,1,1,128,1,float16,fp8,131071,0.12934720516204834
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,1,1,128,1,float16,fp8,3,0.012495999783277511
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,1,1,128,1,float16,float16,1,0.05481439828872681
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,1,1,128,1,float16,fp8,1,0.06069120168685913
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,15,0.011655999720096588
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,1,1,128,1,float16,fp8,3,0.06083359718322754
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,1,1,128,1,float16,float16,7,0.054523199796676636
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,1,1,128,1,float16,fp8,7,0.06064959764480591
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,1,1,128,1,float16,float16,15,0.0550495982170105
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,1,1,128,1,float16,fp8,31,0.03765760064125061
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,1,1,128,1,float16,float16,31,0.054992002248764035
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,1,1,128,1,float16,fp8,31,0.060680001974105835
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,1,1,128,1,float16,float16,63,0.05491359829902649
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,1,1,128,1,float16,fp8,63,0.06100800037384033
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,1,1,128,1,float16,fp8,127,0.06106240153312683
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,1,1,128,1,float16,float16,255,0.07643839716911316
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,1,1,128,1,float16,fp8,255,0.07862079739570618
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,1,1,128,1,float16,float16,511,0.10425759553909301
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,1,1,128,1,float16,fp8,511,0.1101647973060608
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,1,1,128,1,float16,float16,1023,0.16857919692993165
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,1,1,128,1,float16,fp8,1023,0.16004799604415892
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,1,1,128,1,float16,float16,2047,0.2857743978500366
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,1,1,128,1,float16,fp8,2047,0.2534096002578735
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,1,1,128,1,float16,float16,4095,0.5171311855316162
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,1,1,128,1,float16,float16,1,0.09444640278816223
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,1,1,128,1,float16,fp8,1,0.012460800260305405
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,1,1,128,1,float16,fp8,1,0.10602879524230957
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,1,1,128,1,float16,float16,3,0.09395679831504822
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,1,1,128,1,float16,fp8,15,0.03753120005130768
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,1,1,128,1,float16,fp8,3,0.10613919496536255
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,1,1,128,1,float16,float16,7,0.09463359713554383
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,1,1,128,1,float16,float16,127,0.059806400537490846
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,1,1,128,1,float16,fp8,7,0.10622719526290894
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,1,1,128,1,float16,float16,15,0.09470080137252808
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,1,1,128,1,float16,fp8,15,0.1062000036239624
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,1,1,128,1,float16,fp8,31,0.10657279491424561
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,1,1,128,1,float16,float16,63,0.09963679909706116
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,1,1,128,1,float16,fp8,63,0.10673760175704956
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,1,1,128,1,float16,float16,127,0.10258400440216064
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,1,1,128,1,float16,fp8,127,0.11352479457855225
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,1,1,128,1,float16,float16,255,0.13201760053634642
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,1,1,128,1,float16,fp8,255,0.1415343999862671
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,1,1,128,1,float16,float16,511,0.18730239868164061
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,1,1,128,1,float16,fp8,511,0.20353279113769532
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,1,1,128,1,float16,float16,1023,0.309171199798584
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,1,1,128,1,float16,fp8,1023,0.3028287887573242
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,1,1,128,1,float16,fp8,4095,0.44950242042541505
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,1,0.011793600022792816
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,1,1,128,1,float16,fp8,1,0.012585599720478059
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,1,1,128,1,float16,float16,2047,0.5387279987335205
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,1,1,128,1,float16,float16,3,0.05487359762191772
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,7,0.011980800330638886
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,1,1,128,1,float16,fp8,7,0.012489599734544754
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,15,0.011547199636697768
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,1,1,128,1,float16,fp8,15,0.012359999865293504
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,31,0.01156959980726242
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,1,1,128,1,float16,fp8,31,0.012329600006341934
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,1,1,128,1,float16,float16,31,0.09408640265464782
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,63,0.011572799831628799
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,127,0.011992000043392181
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,1,1,128,1,float16,fp8,127,0.012707200646400452
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,255,0.013630400598049163
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,1,1,128,1,float16,fp8,255,0.014444799721240997
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,511,0.01643519997596741
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,1,1,128,1,float16,fp8,511,0.018177600204944612
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,1023,0.017521600425243377
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,1,1,128,1,float16,fp8,1023,0.018566399812698364
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,2047,0.019131200015544893
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,1,1,128,1,float16,fp8,2047,0.01876319944858551
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,4095,0.022249600291252135
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,1,1,128,1,float16,fp8,4095,0.022443200647830962
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,8191,0.033486399054527285
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,1,1,128,1,float16,fp8,8191,0.030113598704338072
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,1,1,128,1,float16,fp8,16383,0.047619199752807616
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,32767,0.08371359705924988
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,1,1,128,1,float16,fp8,32767,0.07303360104560852
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,3,0.011675199866294861
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,1,1,128,1,float16,fp8,2047,0.4880383968353271
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,1,1,128,1,float16,fp8,3,0.012331199645996094
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,1,1,128,1,float16,fp8,65535,0.12402399778366088
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,127,0.01170400008559227
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,1,128,1,float16,float16,1,0.013635200262069703
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,1,128,1,float16,fp8,1,0.01436000019311905
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,1,128,1,float16,float16,3,0.01353919953107834
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,1,128,1,float16,fp8,3,0.014313599467277527
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,1,1,128,1,float16,fp8,131071,0.2206592082977295
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,1,128,1,float16,fp8,7,0.0143327996134758
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,1,128,1,float16,fp8,15,0.014531199634075165
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,1,1,128,1,float16,fp8,63,0.01250080019235611
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,1,128,1,float16,float16,31,0.013491199910640716
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,1,128,1,float16,float16,63,0.013521599769592284
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,1,128,1,float16,fp8,63,0.014297600090503692
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,1,128,1,float16,float16,127,0.013643200695514678
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,1,128,1,float16,fp8,127,0.014420799911022186
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,1,128,1,float16,float16,255,0.01566559970378876
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,1,128,1,float16,fp8,255,0.016315199434757233
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,1,128,1,float16,float16,511,0.018990400433540344
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,1,128,1,float16,fp8,511,0.019865599274635316
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,1,128,1,float16,float16,1023,0.02792479991912842
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,1,128,1,float16,fp8,1023,0.026495999097824095
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,1,128,1,float16,float16,2047,0.03889600038528442
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,1,128,1,float16,fp8,2047,0.0345551997423172
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,2,128,1,float16,float16,1,0.013473600149154663
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,2,128,1,float16,fp8,1,0.014295999705791474
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,2,128,1,float16,float16,3,0.01329600065946579
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,2,128,1,float16,fp8,3,0.014135999977588654
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,2,128,1,float16,float16,7,0.013246400654315949
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,2,128,1,float16,fp8,7,0.014099200069904328
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,2,128,1,float16,float16,15,0.013321599364280701
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,65535,0.14197280406951904
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,2,128,1,float16,fp8,15,0.014207999408245086
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,1,1,128,1,float16,fp8,15,0.06032800078392029
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,2,128,1,float16,fp8,31,0.01436000019311905
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,2,128,1,float16,float16,63,0.013398399949073792
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,131071,0.2615056037902832
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,2,128,1,float16,float16,127,0.013655999302864074
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,1,128,1,float16,float16,7,0.01361439973115921
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,2,128,1,float16,fp8,127,0.014345599710941315
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,2,128,1,float16,fp8,255,0.01626559942960739
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,2,128,1,float16,float16,511,0.0189983993768692
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,2,128,1,float16,fp8,511,0.019948799908161164
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,2,128,1,float16,float16,1023,0.032790398597717284
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,1,128,1,float16,fp8,31,0.014263999462127686
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,2,128,1,float16,float16,2047,0.04755040109157562
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,2,128,1,float16,fp8,2047,0.04182719886302948
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,4,128,1,float16,float16,1,0.017371200025081635
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,4,128,1,float16,fp8,1,0.019038400053977965
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,4,128,1,float16,float16,3,0.017505599558353423
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,4,128,1,float16,fp8,3,0.019139200448989868
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,4,128,1,float16,float16,7,0.01743839979171753
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,4,128,1,float16,fp8,7,0.01906079947948456
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,4,128,1,float16,float16,15,0.017476800084114074
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,4,128,1,float16,fp8,15,0.018873600661754607
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,4,128,1,float16,float16,31,0.01743679940700531
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,4,128,1,float16,fp8,31,0.0188400000333786
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,16383,0.05167199969291687
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,4,128,1,float16,fp8,63,0.019073599576950075
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,4,128,1,float16,float16,127,0.017734399437904357
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,4,128,1,float16,fp8,127,0.01919520050287247
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,4,128,1,float16,float16,255,0.021137599647045136
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,4,128,1,float16,fp8,255,0.02301599979400635
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,4,128,1,float16,float16,511,0.03237760066986084
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,4,128,1,float16,fp8,511,0.030244800448417663
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,2,128,1,float16,float16,31,0.013542400300502777
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,4,128,1,float16,float16,1023,0.050811201333999634
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,2,128,1,float16,fp8,63,0.014361600577831268
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,4,128,1,float16,fp8,1023,0.04688000082969666
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,2,128,1,float16,float16,255,0.01547040045261383
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,4,128,1,float16,float16,2047,0.08144959807395935
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,8,128,1,float16,float16,1,0.02475520074367523
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,8,128,1,float16,fp8,1,0.027492800354957582
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,1,128,1,float16,float16,15,0.014163200557231904
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,8,128,1,float16,float16,3,0.02449759989976883
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,8,128,1,float16,float16,7,0.02456959933042526
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,8,128,1,float16,fp8,7,0.02786239981651306
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,8,128,1,float16,fp8,15,0.027622398734092713
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,8,128,1,float16,float16,31,0.025099200010299683
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,8,128,1,float16,fp8,31,0.02757279872894287
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,8,128,1,float16,float16,63,0.025209599733352663
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,8,128,1,float16,fp8,63,0.02757920026779175
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,8,128,1,float16,fp8,127,0.02760480046272278
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,8,128,1,float16,float16,255,0.03848479986190796
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,8,128,1,float16,fp8,255,0.03611840009689331
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,8,128,1,float16,float16,511,0.05205600261688233
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,8,128,1,float16,fp8,511,0.05269439816474915
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,8,128,1,float16,float16,1023,0.086353600025177
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,8,128,1,float16,fp8,1023,0.07872480154037476
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,4,128,1,float16,float16,63,0.017467199265956877
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,8,128,1,float16,float16,2047,0.14553600549697876
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,8,128,1,float16,fp8,2047,0.12586400508880616
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,1,128,1,float16,float16,1,0.011524800211191177
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,1,128,1,float16,float16,3,0.011515200138092041
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,1,128,1,float16,fp8,3,0.012198399752378464
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,1,128,1,float16,float16,7,0.011526399850845337
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,1,128,1,float16,fp8,7,0.012383999675512314
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,1,128,1,float16,float16,15,0.011580800265073776
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,1,128,1,float16,fp8,15,0.012387199699878693
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,1,128,1,float16,float16,31,0.011588799953460693
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,1,128,1,float16,fp8,31,0.012359999865293504
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,1,128,1,float16,float16,63,0.011593600362539291
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,4,128,1,float16,fp8,2047,0.07033920288085938
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,1,128,1,float16,fp8,63,0.012272000312805176
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,1,128,1,float16,float16,127,0.011931200325489045
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,8,128,1,float16,fp8,3,0.02762719988822937
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,1,128,1,float16,float16,255,0.013422399759292603
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,8,128,1,float16,float16,15,0.02452639937400818
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,1,128,1,float16,float16,511,0.016252799332141875
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,2,128,1,float16,fp8,1023,0.028334400057792662
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,1,128,1,float16,fp8,511,0.018273599445819855
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,1,128,1,float16,fp8,1023,0.017950400710105896
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,1,128,1,float16,fp8,2047,0.018068799376487733
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,2,128,1,float16,float16,1,0.01138240024447441
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,2,128,1,float16,fp8,1,0.012177599966526032
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,2,128,1,float16,float16,3,0.011559999734163284
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,2,128,1,float16,fp8,3,0.012387199699878693
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,2,128,1,float16,float16,7,0.011627200245857238
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,2,128,1,float16,fp8,7,0.012144000083208085
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,2,128,1,float16,float16,15,0.01141280010342598
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,2,128,1,float16,fp8,15,0.012151999771595002
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,2,128,1,float16,float16,31,0.011619199812412263
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,2,128,1,float16,fp8,31,0.012169600278139115
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,2,128,1,float16,float16,63,0.011664000153541566
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,2,128,1,float16,fp8,63,0.012201599776744843
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,2,128,1,float16,float16,127,0.01165440008044243
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,2,128,1,float16,fp8,127,0.012299200147390365
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,2,128,1,float16,float16,255,0.013492800295352936
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,2,128,1,float16,fp8,255,0.014393599331378936
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,2,128,1,float16,float16,511,0.01626240015029907
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,2,128,1,float16,fp8,511,0.018352000415325163
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,2,128,1,float16,float16,1023,0.01613280028104782
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,2,128,1,float16,fp8,1023,0.0173552006483078
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,2,128,1,float16,float16,2047,0.01642560064792633
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,1,128,1,float16,fp8,1,0.012892800569534301
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,4,128,1,float16,float16,1,0.011462400108575821
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,4,128,1,float16,fp8,1,0.012300799787044524
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,4,128,1,float16,fp8,3,0.0123648002743721
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,4,128,1,float16,float16,7,0.011521600186824799
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,4,128,1,float16,fp8,7,0.012238399684429168
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,4,128,1,float16,float16,15,0.011433599889278412
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,4,128,1,float16,fp8,15,0.012027200311422348
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,4,128,1,float16,float16,31,0.01157120019197464
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,4,128,1,float16,fp8,31,0.012347199767827988
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,4,128,1,float16,float16,63,0.011631999909877778
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,4,128,1,float16,fp8,63,0.012120000272989272
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,4,128,1,float16,float16,127,0.011462400108575821
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,1,128,1,float16,fp8,255,0.014459200203418732
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,1,128,1,float16,float16,1023,0.016473600268363954
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,1,128,1,float16,float16,2047,0.016809600591659545
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,128,8,128,1,float16,float16,127,0.025726398825645445
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,4,128,1,float16,float16,511,0.016260799765586854
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,4,128,1,float16,fp8,511,0.01804800033569336
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,4,128,1,float16,float16,1023,0.015777599811553956
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,4,128,1,float16,fp8,1023,0.016944000124931337
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,4,128,1,float16,float16,2047,0.016152000427246092
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,4,128,1,float16,fp8,2047,0.017399999499320983
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,8,128,1,float16,float16,1,0.01162559986114502
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,8,128,1,float16,fp8,1,0.01234079971909523
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,8,128,1,float16,float16,3,0.011641599982976914
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,8,128,1,float16,fp8,3,0.012401600182056428
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,8,128,1,float16,float16,7,0.01149279996752739
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,8,128,1,float16,fp8,7,0.012217599898576736
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,8,128,1,float16,float16,15,0.011428800225257874
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,2,128,1,float16,fp8,2047,0.017705599963665008
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,8,128,1,float16,float16,31,0.01148959994316101
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,8,128,1,float16,fp8,31,0.01223360002040863
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,8,128,1,float16,float16,63,0.011470399796962738
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,8,128,1,float16,fp8,63,0.012238399684429168
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,8,128,1,float16,float16,127,0.011398400366306304
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,8,128,1,float16,fp8,127,0.012355200201272964
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,8,128,1,float16,float16,255,0.013288000226020813
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,8,128,1,float16,fp8,255,0.014369599521160126
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,1,128,1,float16,fp8,127,0.012243200093507767
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,8,128,1,float16,float16,511,0.016161599755287172
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,8,128,1,float16,fp8,511,0.01813279986381531
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,8,128,1,float16,fp8,1023,0.017022399604320525
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,4,128,1,float16,fp8,127,0.012254399806261062
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,8,128,1,float16,float16,2047,0.01666560024023056
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,4,128,1,float16,float16,255,0.013044799864292144
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,8,128,1,float16,fp8,2047,0.017297600209712983
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,1,128,1,float16,float16,1,0.011700800061225891
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,1,128,1,float16,fp8,1,0.012379200011491776
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,1,128,1,float16,fp8,3,0.012392000108957291
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,1,128,1,float16,float16,7,0.011713600158691407
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,1,128,1,float16,fp8,7,0.012467200309038163
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,1,128,1,float16,float16,15,0.01175680011510849
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,1,128,1,float16,fp8,15,0.012380799651145935
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,1,128,1,float16,float16,31,0.01180960014462471
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,1,128,1,float16,fp8,31,0.012283200025558471
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,1,128,1,float16,float16,63,0.01183520033955574
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,1,128,1,float16,fp8,63,0.012337599694728852
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,1,128,1,float16,float16,127,0.011879999935626984
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,1,128,1,float16,fp8,127,0.012214399874210358
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,1,128,1,float16,float16,255,0.013495999574661254
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,1,128,1,float16,fp8,255,0.0143327996134758
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,1,128,1,float16,float16,511,0.01629119962453842
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,1,128,1,float16,fp8,511,0.018318399786949158
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,1,128,1,float16,float16,1023,0.016646400094032288
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,1,128,1,float16,fp8,1023,0.017936000227928163
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,1,128,1,float16,float16,2047,0.016899199783802034
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,1,128,1,float16,fp8,2047,0.01823360025882721
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,2,128,1,float16,fp8,1,0.012385600060224534
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,2,128,1,float16,float16,3,0.011615999788045884
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,2,128,1,float16,fp8,3,0.012334399670362473
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,2,128,1,float16,float16,7,0.01151840016245842
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,8,128,1,float16,fp8,15,0.012246400117874146
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,4,128,1,float16,float16,3,0.011478400230407715
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,2,128,1,float16,fp8,15,0.012179200351238251
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,2,128,1,float16,float16,31,0.01154559999704361
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,2,128,1,float16,fp8,31,0.01234399974346161
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,2,128,1,float16,float16,63,0.011615999788045884
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,2,128,1,float16,fp8,63,0.012302400171756744
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,2,128,1,float16,float16,127,0.011454399675130844
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,2,128,1,float16,fp8,127,0.012219200283288956
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,2,128,1,float16,float16,255,0.01326880007982254
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,8,128,1,float16,float16,1023,0.01576640009880066
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,2,128,1,float16,float16,511,0.016323199868202208
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,2,128,1,float16,fp8,511,0.018409599363803864
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,2,128,1,float16,float16,1023,0.016279999911785126
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,2,128,1,float16,fp8,1023,0.017470400035381316
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,2,128,1,float16,float16,2047,0.01639360040426254
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,2,128,1,float16,fp8,2047,0.01778080016374588
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,4,128,1,float16,float16,1,0.011548800021409988
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,4,128,1,float16,fp8,1,0.012439999729394913
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,4,128,1,float16,float16,3,0.011622399836778641
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,4,128,1,float16,fp8,3,0.01226240023970604
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,4,128,1,float16,float16,7,0.01162400022149086
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,4,128,1,float16,fp8,7,0.012246400117874146
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,4,128,1,float16,float16,15,0.011587200313806533
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,4,128,1,float16,float16,31,0.011636800318956374
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,4,128,1,float16,fp8,31,0.012280000001192093
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,4,128,1,float16,float16,63,0.011360000073909759
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,4,128,1,float16,fp8,63,0.012132800370454788
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,4,128,1,float16,float16,127,0.011436799913644791
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,2,128,1,float16,float16,1,0.011430399864912033
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,4,128,1,float16,fp8,127,0.012142399698495865
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,4,128,1,float16,float16,255,0.013251200318336487
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,4,128,1,float16,fp8,255,0.014097599685192109
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,4,128,1,float16,float16,511,0.016276800632476808
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,4,128,1,float16,fp8,511,0.017987200617790224
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,4,128,1,float16,float16,1023,0.016075199842453
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,4,128,1,float16,fp8,1023,0.017105600237846373
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,4,128,1,float16,float16,2047,0.016599999368190767
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,8,128,1,float16,float16,1,0.011718399822711945
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,2,128,1,float16,float16,15,0.011572799831628799
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,8,128,1,float16,float16,3,0.0117296002805233
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,2,128,1,float16,fp8,255,0.01432960033416748
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,8,128,1,float16,fp8,3,0.012479999661445617
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,8,128,1,float16,float16,7,0.011648000031709672
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,128,4,128,1,float16,fp8,255,0.014004799723625182
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,8,128,1,float16,float16,15,0.011635199934244157
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,8,128,1,float16,fp8,15,0.012432000041007996
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,8,128,1,float16,float16,31,0.011659199744462967
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,8,128,1,float16,fp8,31,0.012414400279521943
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,4,128,1,float16,fp8,15,0.012252800166606903
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,8,128,1,float16,float16,63,0.011751999706029892
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,8,128,1,float16,fp8,63,0.012246400117874146
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,8,128,1,float16,float16,127,0.011921600252389909
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,8,128,1,float16,fp8,127,0.012331199645996094
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,8,128,1,float16,fp8,255,0.01441120058298111
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,8,128,1,float16,float16,511,0.01640959978103638
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,8,128,1,float16,fp8,511,0.018139199912548067
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,8,128,1,float16,float16,1023,0.01639840006828308
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,8,128,1,float16,fp8,1023,0.017283199727535246
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,8,128,1,float16,float16,2047,0.018024000525474548
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,4,128,1,float16,fp8,2047,0.017347200214862822
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,8,128,1,float16,fp8,2047,0.017870399355888366
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,1,128,1,float16,float16,1,0.01271360069513321
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,8,128,1,float16,fp8,1,0.012334399670362473
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,1,128,1,float16,fp8,1,0.013631999492645264
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,1,128,1,float16,fp8,3,0.01342719942331314
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,1,128,1,float16,float16,7,0.013387200236320496
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,1,128,1,float16,fp8,7,0.0137472003698349
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,1,128,1,float16,float16,15,0.013099199533462525
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,1,128,1,float16,fp8,15,0.013635200262069703
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,1,128,1,float16,float16,31,0.012702399492263794
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,1,128,1,float16,fp8,31,0.013915200531482697
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,1,128,1,float16,float16,63,0.013099199533462525
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,1,128,1,float16,fp8,63,0.013860799372196198
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,1,128,1,float16,float16,127,0.013439999520778656
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,1,128,1,float16,fp8,127,0.013463999330997466
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,1,128,1,float16,float16,255,0.015009599924087524
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,1,128,1,float16,fp8,255,0.01557600051164627
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,1,128,1,float16,float16,511,0.017791999876499175
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,1,128,1,float16,fp8,511,0.019499200582504272
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,1,128,1,float16,float16,1023,0.02114560008049011
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,1,128,1,float16,fp8,1023,0.020971199870109557
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,1,128,1,float16,float16,2047,0.024540799856185912
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,1,128,1,float16,fp8,2047,0.024745599925518037
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,2,128,1,float16,float16,1,0.0130048006772995
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,2,128,1,float16,fp8,1,0.013686400651931763
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,2,128,1,float16,float16,3,0.012932799756526947
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,2,128,1,float16,fp8,3,0.013697600364685059
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,1,128,1,float16,float16,3,0.011699199676513672
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,2,128,1,float16,fp8,7,0.013665600121021271
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,2,128,1,float16,fp8,15,0.013627199828624726
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,2,128,1,float16,float16,31,0.012899200618267059
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,2,128,1,float16,fp8,31,0.013460800051689148
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,2,128,1,float16,float16,63,0.012756800651550293
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,2,128,1,float16,fp8,63,0.013457599282264709
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,2,128,1,float16,float16,127,0.012758399546146392
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,8,128,1,float16,float16,255,0.013515199720859527
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,2,128,1,float16,float16,255,0.014662399888038635
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,2,128,1,float16,fp8,255,0.01552959978580475
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,2,128,1,float16,float16,511,0.017764799296855927
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,2,128,1,float16,fp8,511,0.019065600633621217
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,2,128,1,float16,float16,1023,0.022100800275802614
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,2,128,1,float16,fp8,1023,0.023004800081253052
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,2,128,1,float16,float16,2047,0.03375999927520752
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,2,128,1,float16,fp8,2047,0.029025599360466003
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,4,128,1,float16,float16,1,0.012831999361515046
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,4,128,1,float16,fp8,1,0.013446399569511413
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,4,128,1,float16,float16,3,0.01279519945383072
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,4,128,1,float16,fp8,3,0.013495999574661254
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,4,128,1,float16,float16,7,0.012694400548934937
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,4,128,1,float16,fp8,7,0.013711999356746673
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,4,128,1,float16,float16,15,0.012982399761676788
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,4,128,1,float16,fp8,15,0.013755199313163758
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,4,128,1,float16,float16,31,0.012943999469280243
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,4,128,1,float16,fp8,31,0.01372320055961609
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,4,128,1,float16,float16,63,0.012966400384902954
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,4,128,1,float16,fp8,63,0.013537600636482239
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,8,128,1,float16,fp8,7,0.012449599802494049
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,4,128,1,float16,fp8,127,0.013569599390029908
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,4,128,1,float16,float16,255,0.014617599546909332
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,4,128,1,float16,fp8,255,0.01547040045261383
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,4,128,1,float16,float16,511,0.01794240027666092
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,4,128,1,float16,fp8,511,0.019190399348735808
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,2,128,1,float16,fp8,127,0.013459199666976928
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,4,128,1,float16,float16,2047,0.04541279971599579
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,4,128,1,float16,fp8,2047,0.04068480134010315
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,8,128,1,float16,float16,1,0.01664319932460785
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,8,128,1,float16,fp8,1,0.017907199263572694
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,8,128,1,float16,float16,3,0.01672320067882538
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,8,128,1,float16,fp8,3,0.01810240000486374
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,8,128,1,float16,float16,7,0.016785599291324615
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,1,128,1,float16,float16,3,0.012782399356365205
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,8,128,1,float16,fp8,7,0.018118399381637573
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,8,128,1,float16,float16,15,0.01664319932460785
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,8,128,1,float16,fp8,15,0.018147200345993042
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,8,128,1,float16,float16,31,0.01661120057106018
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,8,128,1,float16,float16,63,0.01669919937849045
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,8,128,1,float16,fp8,63,0.017982399463653563
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,8,128,1,float16,float16,127,0.016812799870967864
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,8,128,1,float16,fp8,127,0.0178399994969368
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,8,128,1,float16,float16,255,0.020310400426387785
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,2,128,1,float16,float16,7,0.012873600423336028
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,2,128,1,float16,float16,15,0.012916800379753113
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,8,128,1,float16,float16,511,0.03059200048446655
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,8,128,1,float16,fp8,511,0.030000001192092896
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,8,128,1,float16,float16,1023,0.04847840070724487
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,8,128,1,float16,fp8,1023,0.04507839977741242
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,4,128,1,float16,fp8,1023,0.02686559855937958
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,8,128,1,float16,fp8,2047,0.06855840086936951
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,1,128,1,float16,float16,1,0.01162559986114502
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,1,128,1,float16,fp8,1,0.012441600114107132
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,1,128,1,float16,float16,3,0.011907199770212174
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,128,2,128,1,float16,fp8,7,0.012200000137090683
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,1,128,1,float16,fp8,3,0.012691199779510498
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,1,128,1,float16,float16,7,0.011766400188207626
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,1,128,1,float16,float16,15,0.011819200217723846
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,1,128,1,float16,fp8,15,0.012417600303888322
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,1,128,1,float16,float16,31,0.011689600348472596
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,1,128,1,float16,fp8,31,0.012399999797344208
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,1,128,1,float16,float16,63,0.011750400066375732
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,1,128,1,float16,fp8,63,0.012390399724245072
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,1,128,1,float16,float16,127,0.011740799993276596
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,1,128,1,float16,fp8,127,0.012511999905109405
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,1,128,1,float16,float16,255,0.013504000008106231
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,1,128,1,float16,fp8,255,0.014545600116252898
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,1,128,1,float16,float16,511,0.016457599401474
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,1,128,1,float16,fp8,511,0.018545599281787874
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,4,128,1,float16,float16,127,0.013064000010490417
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,8,128,1,float16,fp8,255,0.02215680032968521
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,1,128,1,float16,float16,2047,0.01713760048151016
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,1,128,1,float16,fp8,2047,0.01820800006389618
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,2,128,1,float16,float16,1,0.011539199948310852
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,2,128,1,float16,fp8,1,0.012356799840927125
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,4,128,1,float16,float16,1023,0.030105599761009218
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,2,128,1,float16,fp8,3,0.012464000284671784
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,2,128,1,float16,float16,7,0.011519999802112579
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,2,128,1,float16,fp8,7,0.012455999851226807
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,2,128,1,float16,float16,15,0.011638399958610535
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,2,128,1,float16,fp8,15,0.01244800016283989
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,2,128,1,float16,float16,31,0.01162080019712448
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,2,128,1,float16,fp8,31,0.01249760016798973
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,2,128,1,float16,float16,63,0.011822400242090225
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,2,128,1,float16,fp8,63,0.012401600182056428
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,2,128,1,float16,float16,127,0.011822400242090225
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,2,128,1,float16,fp8,127,0.012359999865293504
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,2,128,1,float16,float16,255,0.013475200533866883
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,2,128,1,float16,fp8,255,0.014404800534248353
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,2,128,1,float16,float16,511,0.016448000073432924
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,2,128,1,float16,fp8,511,0.01802079975605011
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,2,128,1,float16,float16,1023,0.01659359931945801
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,2,128,1,float16,fp8,1023,0.017696000635623932
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,2,128,1,float16,float16,2047,0.01706880033016205
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,2,128,1,float16,fp8,2047,0.01780160069465637
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,8,128,1,float16,fp8,31,0.01815200001001358
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,4,128,1,float16,fp8,1,0.012516799569129943
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,4,128,1,float16,fp8,3,0.012511999905109405
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,4,128,1,float16,float16,7,0.011609599739313126
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,4,128,1,float16,fp8,7,0.012358400225639343
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,4,128,1,float16,float16,15,0.011727999895811081
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,4,128,1,float16,fp8,15,0.012495999783277511
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,4,128,1,float16,float16,31,0.011793600022792816
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,4,128,1,float16,fp8,31,0.012534399330615998
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,4,128,1,float16,float16,63,0.011860799789428712
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,4,128,1,float16,fp8,63,0.012601600587368011
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,4,128,1,float16,float16,127,0.011724799871444702
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,1,128,1,float16,float16,1023,0.016787199676036833
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,4,128,1,float16,float16,255,0.01329279989004135
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,4,128,1,float16,fp8,255,0.014313599467277527
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,4,128,1,float16,float16,511,0.01629119962453842
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,4,128,1,float16,fp8,511,0.0180976003408432
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,4,128,1,float16,float16,1023,0.016436800360679626
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,4,128,1,float16,fp8,1023,0.01722400039434433
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,2,128,1,float16,float16,3,0.01170400008559227
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,4,128,1,float16,fp8,2047,0.01801439970731735
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,128,8,128,1,float16,float16,2047,0.07944480180740357
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,8,128,1,float16,float16,1,0.011905600130558015
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,8,128,1,float16,fp8,1,0.012721599638462066
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,8,128,1,float16,fp8,3,0.01268640011548996
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,8,128,1,float16,float16,7,0.011812800168991089
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,8,128,1,float16,fp8,7,0.012665599584579468
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,8,128,1,float16,float16,15,0.01188960000872612
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,8,128,1,float16,fp8,15,0.012771199643611907
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,8,128,1,float16,float16,31,0.011972799897193909
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,4,128,1,float16,float16,1,0.01162559986114502
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,8,128,1,float16,float16,63,0.011812800168991089
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,4,128,1,float16,float16,3,0.011535999923944473
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,8,128,1,float16,fp8,63,0.012740799784660339
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,8,128,1,float16,float16,127,0.01178399994969368
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,8,128,1,float16,float16,255,0.013494400680065155
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,8,128,1,float16,fp8,255,0.014475199580192565
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,8,128,1,float16,float16,511,0.01658080071210861
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,8,128,1,float16,fp8,511,0.018193599581718446
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,8,128,1,float16,float16,1023,0.018027199804782866
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,8,128,1,float16,fp8,1023,0.017556799948215483
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,8,128,1,float16,float16,2047,0.0199072003364563
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,4,128,1,float16,fp8,127,0.012415999919176102
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,8,128,1,float16,fp8,2047,0.021252800524234772
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,1,128,1,float16,float16,1,0.015030400454998016
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,1,128,1,float16,fp8,1,0.01629440039396286
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,1,128,1,float16,float16,3,0.015095999836921692
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,1,128,1,float16,fp8,3,0.01613599956035614
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,1,128,1,float16,float16,7,0.015000000596046448
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,1,128,1,float16,fp8,7,0.016062399744987486
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,1,128,1,float16,float16,15,0.015160000324249268
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,1,128,1,float16,fp8,15,0.015932799875736238
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,4,128,1,float16,float16,2047,0.018267199397087097
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,1,128,1,float16,float16,31,0.015118399262428283
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,1,128,1,float16,fp8,31,0.016369600594043732
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,1,128,1,float16,float16,63,0.015294399857521058
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,1,128,1,float16,fp8,63,0.016327999532222748
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,1,128,1,float16,float16,127,0.01562879979610443
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,8,128,1,float16,float16,3,0.011886399984359742
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,1,128,1,float16,float16,255,0.01722240000963211
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,1,128,1,float16,fp8,255,0.01823520064353943
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,1,128,1,float16,float16,511,0.02143999934196472
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,1,128,1,float16,fp8,511,0.021849599480628968
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,1,128,1,float16,float16,1023,0.036206400394439696
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,1,128,1,float16,fp8,1023,0.0316431999206543
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,2,128,1,float16,float16,1,0.018488000333309173
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,2,128,1,float16,fp8,1,0.020526400208473204
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,1,128,1,float16,fp8,7,0.012680000066757202
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,2,128,1,float16,fp8,3,0.02056639939546585
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,2,128,1,float16,float16,7,0.018606400489807128
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,2,128,1,float16,fp8,7,0.020576000213623047
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,8,128,1,float16,fp8,31,0.012798400223255157
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,2,128,1,float16,float16,15,0.01870719939470291
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,2,128,1,float16,fp8,15,0.020742399990558623
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,2,128,1,float16,float16,31,0.01879359930753708
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,2,128,1,float16,float16,63,0.018859200179576874
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,2,128,1,float16,fp8,63,0.020627200603485107
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,2,128,1,float16,float16,127,0.01911199986934662
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,2,128,1,float16,fp8,127,0.020614400506019592
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,2,128,1,float16,float16,255,0.02325280010700226
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,2,128,1,float16,fp8,255,0.024542400240898134
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,2,128,1,float16,float16,511,0.03580319881439209
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,2,128,1,float16,fp8,511,0.03272800147533417
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,2,128,1,float16,float16,1023,0.05427200198173523
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,2,128,1,float16,fp8,1023,0.04873439967632294
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,4,128,1,float16,float16,1,0.026236799359321595
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,4,128,1,float16,fp8,1,0.02948960065841675
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,4,128,1,float16,float16,3,0.02632319927215576
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,4,128,1,float16,fp8,3,0.029337599873542786
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,4,128,1,float16,float16,7,0.02635520100593567
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,4,128,1,float16,fp8,7,0.02938399910926819
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,4,128,1,float16,float16,15,0.02643519937992096
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,4,128,1,float16,fp8,15,0.02929440140724182
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,4,128,1,float16,float16,31,0.02656320035457611
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,4,128,1,float16,fp8,31,0.029465600848197937
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,4,128,1,float16,float16,63,0.026548799872398377
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,4,128,1,float16,fp8,63,0.02951360046863556
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,4,128,1,float16,float16,127,0.027646398544311522
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,4,128,1,float16,fp8,127,0.029675200581550598
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,4,128,1,float16,float16,255,0.040996798872947694
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,4,128,1,float16,fp8,255,0.03792639970779419
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,4,128,1,float16,float16,511,0.05448160171508789
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,4,128,1,float16,fp8,511,0.0546176016330719
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,4,128,1,float16,float16,1023,0.0897808015346527
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,8,128,1,float16,float16,1,0.0411871999502182
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,8,128,1,float16,fp8,1,0.047337600588798524
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,8,128,1,float16,float16,3,0.041094401478767396
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,2,128,1,float16,float16,3,0.018561600148677825
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,8,128,1,float16,fp8,3,0.047286400198936464
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,8,128,1,float16,float16,7,0.04069119989871979
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,8,128,1,float16,float16,15,0.04078879952430725
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,2,128,1,float16,fp8,31,0.020579199492931365
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,8,128,1,float16,fp8,15,0.04702720046043396
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,8,128,1,float16,float16,31,0.040536001324653625
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,8,128,1,float16,fp8,31,0.047092801332473753
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,8,128,1,float16,float16,63,0.04306240081787109
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,8,128,1,float16,fp8,63,0.04724160134792328
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,8,128,1,float16,float16,127,0.04845759868621826
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,8,128,1,float16,fp8,127,0.0491104006767273
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,8,128,1,float16,float16,255,0.06382719874382019
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,8,128,1,float16,fp8,255,0.06641600131988526
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,8,128,1,float16,float16,511,0.09310399889945983
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,8,128,1,float16,float16,1023,0.15748159885406493
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,1,128,1,float16,fp8,1023,0.017803199589252472
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,8,128,1,float16,fp8,1023,0.1446336030960083
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,128,1,128,1,float16,fp8,1,0.02304159998893738
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,128,1,128,1,float16,float16,3,0.021619200706481934
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,128,1,128,1,float16,fp8,3,0.023083199560642243
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,128,1,128,1,float16,float16,7,0.02197919934988022
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,128,1,128,1,float16,fp8,7,0.023175999522209167
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,128,1,128,1,float16,float16,15,0.02194720059633255
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,128,1,128,1,float16,float16,31,0.021993599832057953
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,128,1,128,1,float16,fp8,31,0.023747199773788454
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,128,1,128,1,float16,float16,63,0.02231519967317581
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,128,1,128,1,float16,fp8,63,0.023769600689411162
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,128,1,128,1,float16,float16,127,0.023204800486564637
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,128,1,128,1,float16,fp8,127,0.023918400704860687
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,128,1,128,1,float16,float16,255,0.0305184006690979
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,128,1,128,1,float16,fp8,255,0.0276528000831604
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,128,2,128,1,float16,float16,1,0.02863520085811615
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,128,2,128,1,float16,fp8,1,0.032046398520469664
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,128,2,128,1,float16,float16,3,0.028531199693679808
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,128,2,128,1,float16,fp8,3,0.032123199105262755
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,128,2,128,1,float16,float16,7,0.02882719933986664
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,128,2,128,1,float16,fp8,7,0.03232640027999878
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,128,2,128,1,float16,float16,15,0.028756800293922424
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,128,2,128,1,float16,fp8,15,0.03237920105457306
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,128,2,128,1,float16,float16,31,0.02881760001182556
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,8,128,1,float16,fp8,511,0.0947808027267456
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,128,2,128,1,float16,fp8,31,0.031934401392936705
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,128,1,128,1,float16,float16,1,0.022094400227069856
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,128,2,128,1,float16,fp8,63,0.03175039887428284
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,128,2,128,1,float16,float16,127,0.03364480137825012
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,128,2,128,1,float16,fp8,127,0.0324400007724762
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,128,2,128,1,float16,float16,255,0.04508320093154907
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,128,2,128,1,float16,fp8,255,0.041145598888397215
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,128,4,128,1,float16,float16,1,0.04327679872512817
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,4,128,1,float16,fp8,1023,0.08081120252609253
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,128,4,128,1,float16,fp8,1,0.049516800045967105
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,128,4,128,1,float16,float16,3,0.043433600664138795
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,128,4,128,1,float16,fp8,3,0.049553599953651425
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,128,4,128,1,float16,float16,7,0.043244799971580504
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,128,4,128,1,float16,fp8,7,0.04959200024604797
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,128,4,128,1,float16,float16,15,0.043278399109840396
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,128,4,128,1,float16,fp8,15,0.04962719976902008
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,128,4,128,1,float16,float16,31,0.043568000197410583
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,128,4,128,1,float16,fp8,31,0.049716800451278687
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,8,128,1,float16,fp8,7,0.047193598747253415
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,128,4,128,1,float16,fp8,63,0.04999839961528778
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,128,4,128,1,float16,float16,127,0.05218719840049744
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,128,4,128,1,float16,fp8,127,0.0531216025352478
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,128,4,128,1,float16,float16,255,0.06729919910430908
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,128,4,128,1,float16,fp8,255,0.06873440146446227
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,128,8,128,1,float16,float16,1,0.07204800248146057
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,128,8,128,1,float16,fp8,1,0.08515679836273193
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,128,8,128,1,float16,float16,3,0.07202079892158508
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,128,8,128,1,float16,float16,7,0.07211999893188477
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,128,8,128,1,float16,fp8,7,0.08531360030174255
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,128,8,128,1,float16,float16,15,0.07239519953727722
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,128,8,128,1,float16,fp8,15,0.08552799820899963
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,128,2,128,1,float16,float16,63,0.029065600037574767
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,128,8,128,1,float16,float16,31,0.07335519790649414
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,128,8,128,1,float16,fp8,31,0.08519200086593628
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,128,8,128,1,float16,float16,63,0.08096320033073426
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,128,8,128,1,float16,fp8,63,0.08992959856987
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,128,8,128,1,float16,float16,127,0.08363999724388123
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,128,8,128,1,float16,fp8,127,0.09512320160865784
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,128,8,128,1,float16,float16,255,0.1117408037185669
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,128,1,128,1,float16,float16,1,0.0360287994146347
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,128,1,128,1,float16,fp8,127,0.016383999586105348
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,128,1,128,1,float16,fp8,1,0.038950398564338684
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,128,1,128,1,float16,float16,3,0.035872000455856326
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,128,1,128,1,float16,fp8,3,0.03896960020065308
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,128,1,128,1,float16,float16,7,0.03620480000972748
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,128,1,128,1,float16,fp8,7,0.03905439972877502
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,128,1,128,1,float16,float16,15,0.03630560040473938
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,128,1,128,1,float16,fp8,15,0.038929599523544314
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,128,1,128,1,float16,float16,31,0.03654879927635193
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,128,1,128,1,float16,fp8,31,0.03919839859008789
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,128,1,128,1,float16,float16,63,0.04015200138092041
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,128,4,128,1,float16,float16,63,0.04797599911689758
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,128,1,128,1,float16,float16,127,0.04439840018749237
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,128,1,128,1,float16,fp8,127,0.04090079963207245
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,128,2,128,1,float16,float16,1,0.0502240002155304
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,128,2,128,1,float16,fp8,1,0.05589280128479004
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,128,8,128,1,float16,fp8,127,0.012748800218105316
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,128,2,128,1,float16,fp8,3,0.05603359937667847
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,128,2,128,1,float16,float16,7,0.05006719827651977
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,128,2,128,1,float16,fp8,7,0.05607360005378723
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,128,2,128,1,float16,fp8,15,0.05597280263900757
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,128,2,128,1,float16,float16,31,0.052209597826004026
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,128,2,128,1,float16,fp8,31,0.056283199787139894
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,128,2,128,1,float16,float16,63,0.05598400235176086
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,128,2,128,1,float16,fp8,63,0.059166401624679565
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,128,2,128,1,float16,float16,127,0.05973920226097107
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,128,2,128,1,float16,fp8,127,0.06298879981040954
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,128,4,128,1,float16,float16,1,0.07811359763145446
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,128,4,128,1,float16,fp8,1,0.09063680171966552
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,128,4,128,1,float16,float16,3,0.07785919904708863
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,128,4,128,1,float16,fp8,3,0.0905247986316681
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,128,1,128,1,float16,fp8,15,0.02325280010700226
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,128,4,128,1,float16,fp8,7,0.0908959984779358
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,128,4,128,1,float16,float16,15,0.08087040185928344
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,128,4,128,1,float16,fp8,15,0.09088799953460694
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,128,4,128,1,float16,float16,31,0.08512639999389648
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,128,8,128,1,float16,fp8,255,0.12073440551757812
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,128,4,128,1,float16,float16,63,0.08753920197486878
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,128,4,128,1,float16,fp8,63,0.09736800193786621
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,128,4,128,1,float16,float16,127,0.0918079972267151
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,128,4,128,1,float16,fp8,127,0.09996320009231567
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,128,8,128,1,float16,float16,1,0.13394240140914918
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,128,8,128,1,float16,fp8,1,0.1580175995826721
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,128,8,128,1,float16,float16,3,0.13515039682388305
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,128,1,128,1,float16,fp8,63,0.039673599600791934
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,128,8,128,1,float16,fp8,3,0.15835679769515992
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,128,8,128,1,float16,float16,7,0.13710559606552125
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,128,8,128,1,float16,fp8,7,0.15898720026016236
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,128,8,128,1,float16,float16,15,0.1429136037826538
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,128,8,128,1,float16,fp8,15,0.1598096013069153
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,128,8,128,1,float16,float16,31,0.14559839963912963
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,128,8,128,1,float16,fp8,31,0.16702239513397216
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,128,8,128,1,float16,float16,63,0.1472480058670044
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,128,2,128,1,float16,float16,15,0.0503600001335144
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,128,8,128,1,float16,fp8,3,0.08509759902954102
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,128,8,128,1,float16,fp8,63,0.17144960165023804
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,1,128,1,float16,float16,1,0.012012799829244613
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,128,8,128,1,float16,fp8,127,0.177675199508667
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,1,128,1,float16,fp8,3,0.012622399628162384
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,1,128,1,float16,float16,7,0.011975999921560287
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,1,128,1,float16,fp8,7,0.012612800300121307
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,1,128,1,float16,float16,15,0.011956799775362015
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,1,128,1,float16,fp8,15,0.012620800733566284
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,1,128,1,float16,float16,31,0.012135999649763108
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,1,128,1,float16,fp8,31,0.012777599692344665
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,1,128,1,float16,float16,63,0.012134400010108948
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,1,128,1,float16,fp8,63,0.01281919926404953
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,128,4,128,1,float16,float16,7,0.07833759784698487
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,1,128,1,float16,float16,127,0.012147200107574464
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,1,128,1,float16,float16,255,0.013676799833774567
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,1,128,1,float16,fp8,255,0.014521600306034088
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,1,128,1,float16,float16,511,0.016627199947834015
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,1,128,1,float16,fp8,511,0.018486399948596955
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,128,4,128,1,float16,fp8,31,0.09287520051002503
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,1,128,1,float16,float16,1023,0.01725600063800812
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,1,128,1,float16,fp8,1023,0.018249599635601042
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,1,128,1,float16,fp8,2047,0.018617600202560425
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,2,128,1,float16,float16,1,0.012044800072908401
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,2,128,1,float16,fp8,1,0.012838399410247803
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,2,128,1,float16,float16,3,0.012169600278139115
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,2,128,1,float16,fp8,3,0.012899200618267059
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,2,128,1,float16,float16,7,0.012001600116491318
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,2,128,1,float16,fp8,7,0.012699200212955475
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,2,128,1,float16,float16,15,0.011992000043392181
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,2,128,1,float16,fp8,15,0.012720000743865967
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,2,128,1,float16,float16,31,0.011964800208806992
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,2,128,1,float16,fp8,31,0.012705600261688233
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,2,128,1,float16,float16,63,0.0121568001806736
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,2,128,1,float16,fp8,63,0.01292479932308197
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,2,128,1,float16,float16,127,0.012292800098657608
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,128,8,128,1,float16,float16,127,0.15041760206222535
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,1,128,1,float16,float16,3,0.0122079998254776
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,2,128,1,float16,float16,255,0.013532799482345582
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,2,128,1,float16,fp8,255,0.01478080004453659
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,2,128,1,float16,float16,511,0.01634240001440048
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,2,128,1,float16,fp8,511,0.018356800079345703
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,2,128,1,float16,float16,1023,0.016774399578571318
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,2,128,1,float16,fp8,1023,0.0178399994969368
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,2,128,1,float16,float16,2047,0.018433600664138794
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,2,128,1,float16,fp8,2047,0.018404799699783325
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,4,128,1,float16,float16,1,0.011987199634313583
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,4,128,1,float16,fp8,1,0.012692800164222718
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,4,128,1,float16,float16,3,0.012027200311422348
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,4,128,1,float16,fp8,3,0.012726399302482604
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,4,128,1,float16,float16,7,0.012008000165224075
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,4,128,1,float16,float16,15,0.011992000043392181
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,1,128,1,float16,fp8,127,0.012806400656700134
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,4,128,1,float16,fp8,15,0.012782399356365205
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,4,128,1,float16,float16,31,0.012014400213956833
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,4,128,1,float16,fp8,31,0.012697599828243256
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,4,128,1,float16,float16,63,0.01194079965353012
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,4,128,1,float16,fp8,63,0.012604799866676331
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,4,128,1,float16,float16,127,0.011950399726629257
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,4,128,1,float16,fp8,127,0.012641599774360657
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,4,128,1,float16,float16,255,0.013827200233936309
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,4,128,1,float16,fp8,255,0.014571200311183929
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,4,128,1,float16,float16,511,0.016864000260829924
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,1,128,1,float16,float16,2047,0.017766399681568144
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,4,128,1,float16,fp8,511,0.01812160015106201
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,4,128,1,float16,float16,1023,0.018163199722766876
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,4,128,1,float16,fp8,1023,0.01764000058174133
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,4,128,1,float16,float16,2047,0.020233599841594695
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,8,128,1,float16,float16,1,0.0118928000330925
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,8,128,1,float16,fp8,1,0.012801599502563477
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,8,128,1,float16,fp8,3,0.012811200320720672
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,8,128,1,float16,float16,7,0.012027200311422348
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,8,128,1,float16,fp8,7,0.01281919926404953
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,8,128,1,float16,float16,15,0.011964800208806992
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,8,128,1,float16,fp8,15,0.012620800733566284
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,8,128,1,float16,float16,31,0.011915200203657151
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,128,2,128,1,float16,float16,3,0.04994719922542572
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,8,128,1,float16,fp8,31,0.01276959925889969
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,8,128,1,float16,float16,63,0.01197120025753975
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,8,128,1,float16,fp8,63,0.012891200184822083
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,8,128,1,float16,float16,127,0.012238399684429168
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,8,128,1,float16,fp8,127,0.012729600071907043
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,8,128,1,float16,float16,255,0.013976000249385834
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,8,128,1,float16,fp8,255,0.014892800152301789
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,8,128,1,float16,float16,511,0.01706559956073761
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,8,128,1,float16,fp8,511,0.018459199368953703
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,8,128,1,float16,float16,1023,0.020304000377655028
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,8,128,1,float16,fp8,1023,0.021091200411319733
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,8,128,1,float16,float16,2047,0.030344000458717345
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,128,1,128,1,float16,float16,1,0.06486880183219909
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,128,1,128,1,float16,fp8,1,0.06916000247001648
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,128,1,128,1,float16,float16,3,0.0652351975440979
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,4,128,1,float16,fp8,7,0.012732799351215362
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,128,1,128,1,float16,float16,7,0.06544640064239501
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,128,1,128,1,float16,float16,15,0.06556159853935242
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,128,1,128,1,float16,fp8,15,0.06931359767913818
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,128,1,128,1,float16,float16,31,0.06631039977073669
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,128,1,128,1,float16,fp8,31,0.06998559832572937
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,128,1,128,1,float16,float16,63,0.06871520280838013
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,4,128,1,float16,fp8,2047,0.02136480063199997
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,128,1,128,1,float16,fp8,63,0.0711184024810791
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,8,128,1,float16,float16,3,0.011912000179290772
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,128,2,128,1,float16,float16,1,0.09247040152549743
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,128,2,128,1,float16,fp8,1,0.10396000146865844
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,128,2,128,1,float16,float16,3,0.09297760128974915
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,128,2,128,1,float16,fp8,3,0.10436799526214599
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,128,2,128,1,float16,float16,7,0.09272000193595886
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,128,2,128,1,float16,fp8,7,0.10460480451583862
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,128,2,128,1,float16,float16,15,0.09309279918670654
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,128,2,128,1,float16,fp8,15,0.10489120483398437
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,128,2,128,1,float16,float16,31,0.09460800290107726
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,128,2,128,1,float16,fp8,31,0.1054703950881958
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,128,2,128,1,float16,fp8,63,0.10752960443496704
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,1,128,1,float16,fp8,1,0.012627199292182922
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,2,128,1,float16,fp8,127,0.012868799269199371
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,128,4,128,1,float16,float16,1,0.15017119646072388
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,128,8,128,1,float16,fp8,2047,0.02712160050868988
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,128,4,128,1,float16,fp8,3,0.17396800518035888
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,128,4,128,1,float16,float16,7,0.1501919984817505
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,128,4,128,1,float16,fp8,7,0.17422080039978027
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,128,4,128,1,float16,float16,15,0.1501039981842041
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,128,1,128,1,float16,fp8,3,0.06901760101318359
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,128,4,128,1,float16,fp8,15,0.17505600452423095
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,128,1,128,1,float16,fp8,7,0.06957280039787292
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,128,4,128,1,float16,fp8,31,0.1762608051300049
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,128,4,128,1,float16,float16,63,0.15504640340805054
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,128,4,128,1,float16,fp8,63,0.17865439653396606
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,128,8,128,1,float16,float16,1,0.2686336040496826
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,128,8,128,1,float16,fp8,1,0.3166383981704712
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,128,8,128,1,float16,float16,3,0.26857120990753175
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,128,8,128,1,float16,fp8,3,0.3162719964981079
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,128,8,128,1,float16,float16,7,0.26900639533996584
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,128,8,128,1,float16,fp8,7,0.31744799613952634
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,128,8,128,1,float16,float16,15,0.2701119899749756
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,128,8,128,1,float16,fp8,15,0.31819679737091067
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,128,8,128,1,float16,fp8,31,0.3195055961608887
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,128,8,128,1,float16,float16,63,0.2743328094482422
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,128,2,128,1,float16,float16,63,0.09726399779319764
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,128,1,128,1,float16,float16,1,0.1133247971534729
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,128,8,128,1,float16,fp8,63,0.32160000801086425
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,128,1,128,1,float16,fp8,1,0.12393120527267457
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,128,1,128,1,float16,float16,3,0.11435840129852295
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,128,1,128,1,float16,fp8,3,0.12421120405197143
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,128,1,128,1,float16,float16,7,0.11400799751281739
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,128,4,128,1,float16,fp8,1,0.17406560182571412
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,128,1,128,1,float16,float16,15,0.11442240476608276
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,128,1,128,1,float16,fp8,15,0.12475520372390747
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,128,1,128,1,float16,float16,31,0.1162335991859436
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,128,2,128,1,float16,float16,1,0.16713279485702515
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,128,2,128,1,float16,fp8,1,0.19132640361785888
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,128,2,128,1,float16,float16,3,0.16749440431594848
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,128,4,128,1,float16,float16,31,0.15150560140609742
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,128,2,128,1,float16,fp8,3,0.19130079746246337
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,128,2,128,1,float16,fp8,7,0.19219679832458497
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,128,2,128,1,float16,float16,15,0.16779040098190307
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,128,2,128,1,float16,fp8,15,0.19273920059204103
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,128,2,128,1,float16,fp8,31,0.19415520429611205
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,128,4,128,1,float16,float16,1,0.2799328088760376
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,128,4,128,1,float16,fp8,1,0.32989599704742434
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,128,4,128,1,float16,float16,3,0.28115038871765136
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,128,8,128,1,float16,float16,31,0.27295520305633547
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,128,4,128,1,float16,fp8,3,0.3295488119125366
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,128,4,128,1,float16,float16,7,0.2807807922363281
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,128,4,128,1,float16,fp8,7,0.33028640747070315
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,128,4,128,1,float16,float16,15,0.2820784091949463
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,128,4,128,1,float16,fp8,15,0.331494402885437
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,128,4,128,1,float16,float16,31,0.2858112096786499
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,128,4,128,1,float16,fp8,31,0.33343360424041746
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,128,4,128,1,float16,float16,3,0.14977760314941407
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,128,8,128,1,float16,float16,1,0.5155295848846435
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,128,1,128,1,float16,fp8,31,0.12548320293426513
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,128,8,128,1,float16,fp8,1,0.6099936008453369
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,128,8,128,1,float16,float16,3,0.5142864227294922
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,128,8,128,1,float16,fp8,3,0.6095903873443603
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,128,8,128,1,float16,float16,7,0.5142928123474121
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,128,2,128,1,float16,float16,7,0.16730079650878907
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,128,8,128,1,float16,fp8,7,0.6107295989990235
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,128,8,128,1,float16,float16,15,0.5173935890197754
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,1,128,1,float16,fp8,1,0.013214400410652161
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,1,128,1,float16,float16,3,0.012164799869060517
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,1,128,1,float16,fp8,3,0.013209599256515502
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,1,128,1,float16,float16,7,0.012256000190973282
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,1,128,1,float16,fp8,7,0.012967999279499053
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,128,2,128,1,float16,float16,31,0.16965279579162598
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,1,128,1,float16,fp8,15,0.012881599366664886
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,1,128,1,float16,float16,31,0.012540799379348756
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,1,128,1,float16,fp8,31,0.012950399518013
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,1,128,1,float16,float16,63,0.012583999335765839
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,1,128,1,float16,fp8,63,0.013084800541400909
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,1,128,1,float16,float16,127,0.01255040019750595
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,1,128,1,float16,fp8,127,0.013079999387264252
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,1,128,1,float16,float16,255,0.014054399728775025
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,1,128,1,float16,fp8,255,0.0149167999625206
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,1,128,1,float16,float16,511,0.0171424001455307
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,1,128,1,float16,fp8,511,0.018795199692249298
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,1,128,1,float16,float16,1023,0.018292799592018127
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,1,128,1,float16,fp8,1023,0.019075199961662292
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,1,128,1,float16,float16,2047,0.020024000108242034
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,1,128,1,float16,fp8,2047,0.02027200013399124
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,2,128,1,float16,float16,1,0.012328000366687774
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,2,128,1,float16,fp8,1,0.013094399869441987
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,2,128,1,float16,float16,3,0.012003199756145477
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,2,128,1,float16,fp8,3,0.012940800189971924
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,2,128,1,float16,float16,7,0.012323199957609176
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,2,128,1,float16,fp8,7,0.013303999602794648
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,2,128,1,float16,float16,15,0.01210239976644516
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,2,128,1,float16,fp8,15,0.01290079951286316
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,2,128,1,float16,float16,31,0.012291199713945388
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,2,128,1,float16,fp8,31,0.013007999956607818
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,2,128,1,float16,float16,63,0.012328000366687774
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,2,128,1,float16,fp8,63,0.012787200510501862
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,128,1,128,1,float16,fp8,7,0.12453759908676147
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,2,128,1,float16,float16,127,0.01252640038728714
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,2,128,1,float16,fp8,127,0.012873600423336028
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,2,128,1,float16,fp8,255,0.014870400726795196
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,2,128,1,float16,float16,511,0.017555199563503265
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,2,128,1,float16,fp8,511,0.01894560009241104
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,2,128,1,float16,float16,1023,0.018966400623321535
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,2,128,1,float16,fp8,1023,0.018726399540901183
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,2,128,1,float16,float16,2047,0.021273599565029146
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,1,128,1,float16,float16,1,0.012191999703645706
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,2,128,1,float16,fp8,2047,0.022201600670814513
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,128,8,128,1,float16,fp8,15,0.6101232051849366
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,4,128,1,float16,float16,1,0.012171199917793274
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,128,8,128,1,float16,float16,31,0.5214992046356202
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,1,128,1,float16,float16,15,0.012464000284671784
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,4,128,1,float16,float16,7,0.012220799922943115
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,4,128,1,float16,fp8,7,0.012942400574684144
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,4,128,1,float16,float16,15,0.012036799639463424
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,4,128,1,float16,fp8,15,0.01316480040550232
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,4,128,1,float16,float16,31,0.012140800058841706
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,4,128,1,float16,fp8,31,0.0131632000207901
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,4,128,1,float16,fp8,63,0.013198399543762207
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,4,128,1,float16,float16,127,0.012476799637079239
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,4,128,1,float16,fp8,127,0.013167999684810638
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,4,128,1,float16,float16,255,0.014276799559593201
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,4,128,1,float16,fp8,255,0.015217599272727967
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,4,128,1,float16,float16,511,0.017272000014781953
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,4,128,1,float16,fp8,511,0.018665599822998046
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,4,128,1,float16,float16,1023,0.020241600275039674
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,4,128,1,float16,fp8,1023,0.021587200462818146
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,4,128,1,float16,float16,2047,0.03142400085926056
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,4,128,1,float16,fp8,2047,0.027796798944473268
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,8,128,1,float16,float16,1,0.012563200294971466
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,8,128,1,float16,fp8,1,0.013291199505329133
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,2,128,1,float16,float16,255,0.013896000385284425
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,8,128,1,float16,float16,3,0.012392000108957291
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,8,128,1,float16,fp8,3,0.013279999792575835
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,8,128,1,float16,float16,7,0.01239359974861145
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,8,128,1,float16,fp8,7,0.013228799402713775
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,8,128,1,float16,float16,15,0.012382400035858155
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,8,128,1,float16,fp8,15,0.013283200562000275
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,8,128,1,float16,float16,31,0.012377600371837615
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,8,128,1,float16,fp8,31,0.013123199343681335
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,8,128,1,float16,float16,63,0.012427199631929398
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,8,128,1,float16,fp8,63,0.013126400113105775
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,8,128,1,float16,fp8,127,0.013329599797725678
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,8,128,1,float16,float16,255,0.014476799964904785
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,8,128,1,float16,fp8,255,0.015411199629306793
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,4,128,1,float16,fp8,1,0.013016000390052795
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,4,128,1,float16,fp8,3,0.01286720037460327
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,8,128,1,float16,fp8,511,0.018883199989795686
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,128,8,128,1,float16,fp8,31,0.6132304191589355
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,8,128,1,float16,fp8,1023,0.02645600140094757
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,8,128,1,float16,float16,2047,0.04440479874610901
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,8,128,1,float16,fp8,2047,0.04054239988327026
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,1,128,1,float16,float16,1,0.013315199315547943
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,1,128,1,float16,fp8,1,0.014273600280284881
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,1,128,1,float16,float16,3,0.013204799592494964
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,1,128,1,float16,fp8,3,0.014270399510860444
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,1,128,1,float16,float16,7,0.013286399841308593
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,1,128,1,float16,fp8,7,0.014265599846839904
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,1,128,1,float16,float16,15,0.013372799754142762
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,1,128,1,float16,fp8,15,0.014254400134086609
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,1,128,1,float16,float16,31,0.013254399597644805
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,1,128,1,float16,fp8,31,0.014155200123786927
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,1,128,1,float16,float16,63,0.013255999982357025
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,1,128,1,float16,fp8,63,0.014164799451828003
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,1,128,1,float16,fp8,127,0.014078399538993836
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,1,128,1,float16,float16,255,0.015364800393581391
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,1,128,1,float16,fp8,255,0.016121600568294526
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,1,128,1,float16,float16,511,0.0186256006360054
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,1,128,1,float16,float16,1023,0.025998398661613464
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,1,128,1,float16,fp8,1023,0.02577440142631531
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,1,128,1,float16,float16,2047,0.037390398979187014
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,1,128,1,float16,fp8,2047,0.03247359991073608
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,2,128,1,float16,float16,1,0.013798399269580841
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,2,128,1,float16,fp8,1,0.014435200393199921
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,2,128,1,float16,float16,3,0.013670399785041809
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,8,128,1,float16,float16,127,0.01268479973077774
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,2,128,1,float16,float16,7,0.013492800295352936
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,2,128,1,float16,fp8,7,0.014337599277496338
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,2,128,1,float16,float16,15,0.013531200587749481
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,8,128,1,float16,float16,511,0.017771199345588684
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,2,128,1,float16,fp8,15,0.014270399510860444
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,2,128,1,float16,float16,31,0.013550400733947754
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,2,128,1,float16,fp8,31,0.014342400431632995
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,8,128,1,float16,float16,1023,0.029627200961112977
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,2,128,1,float16,fp8,63,0.014299200475215912
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,2,128,1,float16,float16,127,0.013644799590110779
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,2,128,1,float16,float16,255,0.01564960032701492
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,4,128,1,float16,float16,63,0.012206400185823441
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,2,128,1,float16,fp8,255,0.01618880033493042
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,2,128,1,float16,float16,511,0.01905599981546402
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,2,128,1,float16,fp8,511,0.020025600492954255
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,2,128,1,float16,float16,1023,0.0322847992181778
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,2,128,1,float16,fp8,1023,0.028548800945281984
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,2,128,1,float16,float16,2047,0.04705440104007721
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,2,128,1,float16,fp8,2047,0.041756799817085265
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,4,128,1,float16,float16,1,0.017140799760818483
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,4,128,1,float16,fp8,1,0.01855040043592453
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,4,128,1,float16,float16,3,0.01712000072002411
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,4,128,1,float16,fp8,3,0.018748800456523895
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,4,128,1,float16,float16,7,0.017294399440288544
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,4,128,1,float16,fp8,7,0.018651199340820313
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,4,128,1,float16,float16,15,0.017131200432777403
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,4,128,1,float16,fp8,15,0.018580800294876097
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,4,128,1,float16,float16,31,0.017272000014781953
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,4,128,1,float16,fp8,31,0.018774400651454925
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,4,128,1,float16,float16,63,0.017308799922466277
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,2,128,1,float16,fp8,3,0.014374400675296783
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,4,128,1,float16,fp8,63,0.01882079988718033
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,4,128,1,float16,float16,127,0.017684799432754517
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,4,128,1,float16,fp8,127,0.01884640008211136
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,4,128,1,float16,float16,255,0.021222400665283202
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,4,128,1,float16,float16,511,0.032574400305747986
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,128,4,128,1,float16,float16,3,0.012187200039625168
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,4,128,1,float16,fp8,511,0.030462399125099182
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,2,128,1,float16,float16,63,0.013531200587749481
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,4,128,1,float16,fp8,1023,0.04684160053730011
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,2,128,1,float16,fp8,127,0.014614400267601014
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,4,128,1,float16,float16,2047,0.08079040050506592
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,4,128,1,float16,fp8,2047,0.06975359916687011
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,8,128,1,float16,float16,1,0.024668799340724946
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,8,128,1,float16,fp8,1,0.027483201026916503
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,8,128,1,float16,fp8,3,0.027689599990844728
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,8,128,1,float16,float16,7,0.024868799746036528
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,8,128,1,float16,fp8,7,0.02773280143737793
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,8,128,1,float16,float16,15,0.024500800669193266
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,8,128,1,float16,fp8,15,0.027614399790763855
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,8,128,1,float16,float16,31,0.02467840015888214
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,8,128,1,float16,fp8,31,0.02767840027809143
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,8,128,1,float16,float16,63,0.02491360008716583
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,8,128,1,float16,fp8,63,0.027651199698448183
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,8,128,1,float16,float16,127,0.02553919851779938
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,8,128,1,float16,fp8,127,0.027689599990844728
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,8,128,1,float16,float16,255,0.03863840103149414
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,8,128,1,float16,fp8,255,0.03635999858379364
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,8,128,1,float16,float16,511,0.051774400472640994
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,1,128,1,float16,fp8,511,0.019862399995326997
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,8,128,1,float16,float16,1023,0.08547520041465759
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,8,128,1,float16,fp8,1023,0.07871519923210143
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,8,128,1,float16,float16,2047,0.14391679763793946
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,8,128,1,float16,fp8,2047,0.12569279670715333
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,1,128,1,float16,float16,1,0.011433599889278412
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,1,128,1,float16,fp8,1,0.01218239963054657
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,1,128,1,float16,float16,3,0.011521600186824799
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,1,128,1,float16,fp8,3,0.012291199713945388
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,1,128,1,float16,float16,7,0.011494400352239609
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,1,128,1,float16,fp8,7,0.012332800030708312
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,1,128,1,float16,float16,15,0.011638399958610535
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,1,128,1,float16,fp8,15,0.01233920007944107
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,1,128,1,float16,float16,31,0.011670400202274323
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,1,128,1,float16,fp8,31,0.012334399670362473
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,1,128,1,float16,float16,63,0.011751999706029892
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,1,128,1,float16,fp8,63,0.012329600006341934
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,1,128,1,float16,float16,127,0.011777599900960922
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,1,128,1,float16,fp8,127,0.012296000123023986
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,1,128,1,float16,float16,255,0.013353599607944489
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,1,128,1,float16,fp8,255,0.014263999462127686
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,1,128,1,float16,float16,511,0.016388800740242005
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,1,128,1,float16,fp8,511,0.01809920072555542
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,1,128,1,float16,float16,1023,0.016417600214481354
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,1,128,1,float16,fp8,1023,0.017499199509620665
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,1,128,1,float16,float16,2047,0.016663999855518342
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,1,128,1,float16,fp8,2047,0.017955200374126436
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,4,128,1,float16,float16,1023,0.05014240145683289
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,2,128,1,float16,fp8,1,0.012135999649763108
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,2,128,1,float16,float16,3,0.011500799655914306
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,2,128,1,float16,float16,7,0.011407999694347382
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,2,128,1,float16,fp8,7,0.012148799747228623
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,2,128,1,float16,float16,15,0.011358399689197541
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,2,128,1,float16,fp8,15,0.012283200025558471
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,2,128,1,float16,float16,31,0.011587200313806533
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,2,128,1,float16,fp8,31,0.012280000001192093
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,2,128,1,float16,float16,63,0.011532799899578094
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,1,128,1,float16,float16,127,0.013652800023555756
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,2,128,1,float16,fp8,127,0.012121599912643433
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,2,128,1,float16,float16,255,0.013067199289798737
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,2,128,1,float16,fp8,255,0.014127999544143677
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,8,128,1,float16,fp8,511,0.05283520221710205
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,2,128,1,float16,fp8,511,0.01835999935865402
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,2,128,1,float16,float16,1023,0.016097599267959596
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,2,128,1,float16,fp8,1023,0.017566399276256563
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,2,128,1,float16,float16,2047,0.01629599928855896
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,2,128,1,float16,fp8,2047,0.017479999363422392
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,4,128,1,float16,float16,1,0.011526399850845337
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,4,128,1,float16,fp8,1,0.01223360002040863
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,4,128,1,float16,float16,3,0.011523199826478958
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,4,128,1,float16,fp8,3,0.012368000298738479
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,4,128,1,float16,float16,7,0.011684799939393998
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,4,128,1,float16,fp8,7,0.012392000108957291
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,4,128,1,float16,fp8,255,0.022726400196552275
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,4,128,1,float16,fp8,15,0.012201599776744843
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,4,128,1,float16,float16,31,0.011448000371456147
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,2,128,1,float16,float16,1,0.011420799791812897
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,4,128,1,float16,fp8,31,0.012276799976825714
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,2,128,1,float16,fp8,3,0.012217599898576736
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,4,128,1,float16,float16,63,0.011553599685430526
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,4,128,1,float16,fp8,63,0.012294399738311767
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,4,128,1,float16,float16,127,0.011638399958610535
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,96,8,128,1,float16,float16,3,0.025040000677108765
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,4,128,1,float16,float16,255,0.013195200264453888
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,4,128,1,float16,fp8,255,0.014238399267196656
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,4,128,1,float16,fp8,511,0.018195199966430663
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,4,128,1,float16,float16,1023,0.01587519943714142
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,4,128,1,float16,fp8,1023,0.016993600130081176
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,2,128,1,float16,fp8,63,0.012241599708795547
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,4,128,1,float16,float16,2047,0.016359999775886536
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,4,128,1,float16,fp8,2047,0.017236800491809846
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,8,128,1,float16,float16,1,0.01162559986114502
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,8,128,1,float16,fp8,1,0.012086399644613267
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,8,128,1,float16,float16,3,0.011591999977827071
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,8,128,1,float16,fp8,3,0.012169600278139115
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,8,128,1,float16,float16,7,0.011505600064992905
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,2,128,1,float16,float16,511,0.01621119976043701
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,8,128,1,float16,float16,15,0.011510399729013443
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,8,128,1,float16,fp8,15,0.01228479966521263
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,8,128,1,float16,float16,31,0.01162400022149086
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,8,128,1,float16,fp8,31,0.012342400103807449
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,8,128,1,float16,float16,63,0.011643200367689132
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,8,128,1,float16,fp8,63,0.012316799908876418
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,8,128,1,float16,float16,127,0.011683200299739838
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,8,128,1,float16,fp8,127,0.012230399996042252
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,8,128,1,float16,float16,255,0.013286399841308593
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,8,128,1,float16,fp8,255,0.014168000221252442
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,8,128,1,float16,float16,511,0.016126400232315062
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,8,128,1,float16,fp8,511,0.01775840073823929
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,8,128,1,float16,float16,1023,0.015897600352764128
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,8,128,1,float16,fp8,1023,0.016840000450611115
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,8,128,1,float16,fp8,2047,0.017254400253295898
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,1,128,1,float16,float16,1,0.01133280023932457
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,1,128,1,float16,fp8,1,0.01234399974346161
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,1,128,1,float16,float16,3,0.011479999870061874
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,1,128,1,float16,fp8,3,0.012243200093507767
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,1,128,1,float16,float16,7,0.011345600336790084
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,4,128,1,float16,float16,15,0.01162400022149086
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,1,128,1,float16,float16,15,0.01154239997267723
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,1,128,1,float16,fp8,15,0.01231520026922226
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,1,128,1,float16,float16,31,0.011648000031709672
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,1,128,1,float16,fp8,31,0.012414400279521943
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,1,128,1,float16,float16,63,0.011721599847078323
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,1,128,1,float16,fp8,63,0.012366399914026261
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,1,128,1,float16,float16,127,0.011700800061225891
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,4,128,1,float16,fp8,127,0.012336000055074691
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,1,128,1,float16,fp8,127,0.012318400293588638
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,1,128,1,float16,float16,255,0.013265599310398103
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,4,128,1,float16,float16,511,0.016227200627326965
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,1,128,1,float16,float16,511,0.01621119976043701
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,1,128,1,float16,fp8,511,0.01839040070772171
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,1,128,1,float16,float16,1023,0.016414399445056915
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,1,128,1,float16,fp8,1023,0.017584000527858735
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,1,128,1,float16,float16,2047,0.01688160002231598
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,1,128,1,float16,fp8,2047,0.017791999876499175
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,2,128,1,float16,float16,1,0.011564800143241882
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,2,128,1,float16,fp8,1,0.012116800248622894
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,2,128,1,float16,float16,3,0.011539199948310852
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,2,128,1,float16,fp8,3,0.012358400225639343
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,2,128,1,float16,float16,7,0.0115167997777462
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,2,128,1,float16,fp8,7,0.01236959993839264
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,2,128,1,float16,float16,15,0.011468800157308579
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,2,128,1,float16,fp8,15,0.012291199713945388
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,2,128,1,float16,float16,31,0.011423999816179276
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,8,128,1,float16,fp8,7,0.012127999961376191
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,2,128,1,float16,fp8,31,0.012276799976825714
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,2,128,1,float16,float16,63,0.011379200220108032
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,2,128,1,float16,fp8,63,0.012135999649763108
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,2,128,1,float16,fp8,127,0.012137600034475327
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,2,128,1,float16,float16,255,0.01324319988489151
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,2,128,1,float16,float16,511,0.016256000101566314
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,2,128,1,float16,fp8,511,0.018086400628089905
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,2,128,1,float16,fp8,1023,0.017211200296878816
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,8,128,1,float16,float16,2047,0.016406400501728056
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,2,128,1,float16,fp8,2047,0.01759999990463257
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,4,128,1,float16,float16,1,0.011614400148391723
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,4,128,1,float16,fp8,1,0.012411200255155564
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,4,128,1,float16,float16,3,0.011648000031709672
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,4,128,1,float16,fp8,3,0.01242400035262108
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,4,128,1,float16,float16,7,0.011627200245857238
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,4,128,1,float16,fp8,7,0.01220960021018982
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,4,128,1,float16,float16,15,0.011552000045776367
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,4,128,1,float16,fp8,15,0.012195199728012085
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,4,128,1,float16,float16,31,0.011553599685430526
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,4,128,1,float16,fp8,31,0.012204799801111221
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,4,128,1,float16,float16,63,0.011646399646997452
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,4,128,1,float16,fp8,63,0.012352000176906585
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,4,128,1,float16,float16,127,0.011744000017642975
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,4,128,1,float16,fp8,127,0.012294399738311767
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,4,128,1,float16,float16,255,0.013198399543762207
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,4,128,1,float16,fp8,255,0.01430719941854477
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,4,128,1,float16,float16,511,0.01634240001440048
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,96,2,128,1,float16,float16,127,0.011582399904727935
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,4,128,1,float16,float16,1023,0.015908800065517426
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,4,128,1,float16,fp8,1023,0.01691199988126755
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,4,128,1,float16,float16,2047,0.0166143998503685
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,2,128,1,float16,float16,127,0.011457599699497223
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,2,128,1,float16,fp8,255,0.014265599846839904
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,8,128,1,float16,float16,1,0.011751999706029892
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,8,128,1,float16,fp8,1,0.012436799705028534
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,8,128,1,float16,float16,3,0.011710400134325028
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,8,128,1,float16,fp8,3,0.012478400021791458
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,2,128,1,float16,float16,2047,0.016622400283813475
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,8,128,1,float16,float16,7,0.01170400008559227
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,8,128,1,float16,fp8,7,0.012567999958992004
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,8,128,1,float16,fp8,15,0.012571200728416443
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,8,128,1,float16,float16,31,0.011825600266456604
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,8,128,1,float16,fp8,31,0.012582400441169738
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,8,128,1,float16,float16,63,0.011771199852228164
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,8,128,1,float16,fp8,63,0.012404800206422806
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,8,128,1,float16,float16,127,0.011648000031709672
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,8,128,1,float16,fp8,127,0.012439999729394913
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,8,128,1,float16,float16,255,0.013646399974822998
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,8,128,1,float16,fp8,255,0.014590400457382201
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,8,128,1,float16,float16,511,0.01619199961423874
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,8,128,1,float16,fp8,511,0.018028800189495087
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,8,128,1,float16,float16,1023,0.01608159989118576
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,8,128,1,float16,fp8,1023,0.017067199945449828
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,1,128,1,float16,fp8,255,0.0143312007188797
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,8,128,1,float16,fp8,2047,0.01788640022277832
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,8,128,1,float16,float16,2047,0.018190400302410127
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,1,128,1,float16,float16,1,0.01257600039243698
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,1,128,1,float16,fp8,1,0.013407999277114868
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,1,128,1,float16,float16,3,0.012547199428081513
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,4,128,1,float16,fp8,511,0.01802240014076233
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,1,128,1,float16,float16,7,0.012603199481964112
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,1,128,1,float16,fp8,7,0.013672000169754029
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,1,128,1,float16,float16,15,0.01263200044631958
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,1,128,1,float16,fp8,15,0.013676799833774567
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,1,128,1,float16,float16,31,0.012638400495052337
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,1,128,1,float16,fp8,31,0.013439999520778656
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,1,128,1,float16,float16,63,0.012740799784660339
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,1,128,1,float16,fp8,63,0.013635200262069703
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,1,128,1,float16,float16,127,0.013012799620628356
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,1,128,1,float16,fp8,127,0.013643200695514678
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,1,128,1,float16,float16,255,0.014601600170135499
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,1,128,1,float16,fp8,255,0.015414400398731232
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,4,128,1,float16,fp8,2047,0.017470400035381316
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,1,128,1,float16,float16,511,0.01751679927110672
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,1,128,1,float16,fp8,511,0.019148799777030944
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,1,128,1,float16,fp8,1023,0.01979999989271164
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,2,128,1,float16,float16,1023,0.016118399798870087
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,1,128,1,float16,fp8,2047,0.023596799373626708
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,2,128,1,float16,float16,1,0.012561599910259246
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,2,128,1,float16,fp8,1,0.013484799861907959
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,2,128,1,float16,float16,3,0.012489599734544754
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,2,128,1,float16,fp8,3,0.013350400328636169
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,2,128,1,float16,float16,7,0.012787200510501862
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,2,128,1,float16,fp8,7,0.013344000279903411
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,2,128,1,float16,float16,15,0.012665599584579468
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,2,128,1,float16,fp8,15,0.013335999846458436
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,2,128,1,float16,float16,31,0.012836800515651703
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,2,128,1,float16,fp8,31,0.013364799320697784
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,2,128,1,float16,float16,63,0.012849600613117218
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,2,128,1,float16,fp8,63,0.013352000713348388
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,2,128,1,float16,float16,127,0.013126400113105775
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,2,128,1,float16,fp8,127,0.013358399271965027
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,2,128,1,float16,float16,255,0.014667199552059173
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,2,128,1,float16,fp8,255,0.01544640064239502
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,2,128,1,float16,float16,511,0.01743839979171753
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,2,128,1,float16,fp8,511,0.01902559995651245
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,2,128,1,float16,float16,1023,0.021223999559879303
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,2,128,1,float16,fp8,1023,0.022462399303913118
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,2,128,1,float16,float16,2047,0.03288480043411255
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,2,128,1,float16,fp8,2047,0.028489598631858827
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,4,128,1,float16,float16,1,0.01266079992055893
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,4,128,1,float16,fp8,1,0.013548800349235534
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,4,128,1,float16,float16,3,0.01284639984369278
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,4,128,1,float16,float16,7,0.012750400602817536
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,4,128,1,float16,fp8,7,0.01361120045185089
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,4,128,1,float16,float16,15,0.012904000282287598
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,4,128,1,float16,fp8,15,0.013628800213336945
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,4,128,1,float16,float16,31,0.012727999687194824
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,4,128,1,float16,float16,63,0.012892800569534301
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,4,128,1,float16,fp8,63,0.01361120045185089
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,1,128,1,float16,fp8,3,0.01369599997997284
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,4,128,1,float16,float16,127,0.013084800541400909
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,4,128,1,float16,fp8,127,0.01377120018005371
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,4,128,1,float16,float16,255,0.01483519971370697
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,4,128,1,float16,fp8,255,0.015806399285793304
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,4,128,1,float16,fp8,511,0.019281600415706635
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,4,128,1,float16,float16,1023,0.030031999945640563
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,4,128,1,float16,fp8,1023,0.026876801252365114
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,4,128,1,float16,float16,2047,0.04513759911060333
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,1,128,1,float16,float16,2047,0.022635200619697572
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,4,128,1,float16,fp8,2047,0.04053280055522919
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,8,128,1,float16,float16,1,0.016833600401878358
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,8,128,1,float16,fp8,1,0.018080000579357148
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,8,128,1,float16,float16,3,0.016803200542926788
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,8,128,1,float16,fp8,3,0.01807039976119995
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,8,128,1,float16,float16,7,0.016804799437522888
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,8,128,1,float16,fp8,7,0.017947199940681457
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,8,128,1,float16,float16,15,0.016804799437522888
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,8,128,1,float16,fp8,15,0.017862400412559508
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,8,128,1,float16,float16,31,0.016820800304412842
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,8,128,1,float16,fp8,31,0.01791680008172989
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,8,128,1,float16,float16,63,0.016820800304412842
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,8,128,1,float16,fp8,63,0.018104000389575957
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,1,128,1,float16,fp8,7,0.012198399752378464
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,8,128,1,float16,fp8,127,0.017950400710105896
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,8,128,1,float16,float16,255,0.02054080069065094
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,8,128,1,float16,fp8,255,0.021891200542449953
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,8,128,1,float16,float16,511,0.030535998940467834
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,4,128,1,float16,fp8,3,0.013537600636482239
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,8,128,1,float16,fp8,511,0.02948319911956787
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,8,128,1,float16,float16,1023,0.04785279929637909
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,8,128,1,float16,fp8,1023,0.04513919949531555
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,8,128,1,float16,float16,2047,0.07831680178642272
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,8,128,1,float16,fp8,2047,0.06859840154647827
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,4,128,1,float16,fp8,31,0.013491199910640716
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,1,128,1,float16,fp8,1,0.01218239963054657
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,1,128,1,float16,float16,3,0.011436799913644791
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,1,128,1,float16,fp8,3,0.012303999811410903
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,1,128,1,float16,float16,7,0.011425600200891495
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,1,128,1,float16,fp8,7,0.01226240023970604
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,1,128,1,float16,float16,15,0.011630400270223617
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,1,128,1,float16,fp8,15,0.01247360035777092
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,1,128,1,float16,float16,31,0.01165120005607605
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,1,128,1,float16,fp8,31,0.01252480000257492
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,1,128,1,float16,float16,63,0.011580800265073776
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,1,128,1,float16,fp8,63,0.012494400143623352
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,1,128,1,float16,float16,127,0.011454399675130844
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,1,128,1,float16,fp8,127,0.012439999729394913
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,1,128,1,float16,float16,255,0.013060800731182098
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,1,128,1,float16,fp8,255,0.014431999623775482
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,1,128,1,float16,float16,511,0.016113600134849547
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,1,128,1,float16,fp8,511,0.01818079948425293
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,1,128,1,float16,float16,1023,0.0164560005068779
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,1,128,1,float16,fp8,1023,0.017591999471187593
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,1,128,1,float16,float16,2047,0.016841599345207216
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,1,128,1,float16,fp8,2047,0.01791519969701767
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,2,128,1,float16,float16,1,0.011575999855995178
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,2,128,1,float16,fp8,1,0.012172800302505494
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,2,128,1,float16,fp8,3,0.012193600088357926
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,2,128,1,float16,float16,7,0.01154559999704361
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,2,128,1,float16,fp8,7,0.012388800084590913
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,96,8,128,1,float16,float16,15,0.011694400012493134
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,2,128,1,float16,float16,15,0.011643200367689132
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,2,128,1,float16,float16,31,0.011646399646997452
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,2,128,1,float16,fp8,31,0.012409599870443344
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,2,128,1,float16,float16,63,0.011564800143241882
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,2,128,1,float16,fp8,63,0.01223199963569641
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,8,128,1,float16,float16,127,0.017000000178813934
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,2,128,1,float16,fp8,127,0.012268800288438797
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,2,128,1,float16,float16,255,0.01324480026960373
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,2,128,1,float16,fp8,255,0.014343999326229095
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,2,128,1,float16,float16,511,0.016339200735092162
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,2,128,1,float16,fp8,511,0.017987200617790224
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,2,128,1,float16,float16,1023,0.01640319973230362
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,2,128,1,float16,fp8,1023,0.017385600507259368
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,2,128,1,float16,float16,2047,0.016777600347995757
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,2,128,1,float16,fp8,2047,0.017617599666118623
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,4,128,1,float16,float16,1,0.011675199866294861
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,4,128,1,float16,fp8,1,0.012449599802494049
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,4,128,1,float16,float16,3,0.011660800129175187
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,4,128,1,float16,fp8,3,0.012548799812793731
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,4,128,1,float16,float16,7,0.011796800047159195
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,4,128,1,float16,fp8,7,0.012563200294971466
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,4,128,1,float16,fp8,15,0.01252799928188324
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,4,128,1,float16,float16,31,0.011788800358772278
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,4,128,1,float16,fp8,31,0.012428800016641617
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,4,128,1,float16,float16,63,0.011750400066375732
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,4,128,1,float16,fp8,63,0.0124208003282547
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,1,128,1,float16,float16,1023,0.020158399641513825
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,2,128,1,float16,float16,3,0.011572799831628799
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,4,128,1,float16,float16,255,0.013388800621032714
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,4,128,1,float16,fp8,255,0.0143327996134758
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,4,128,1,float16,float16,511,0.01634240001440048
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,4,128,1,float16,fp8,511,0.018060800433158875
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,2,128,1,float16,fp8,15,0.012371200323104858
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,4,128,1,float16,float16,1023,0.016172799468040466
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,4,128,1,float16,fp8,1023,0.017108799517154695
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,4,128,1,float16,fp8,2047,0.01777919977903366
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,2,128,1,float16,float16,127,0.011503999680280685
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,8,128,1,float16,float16,1,0.01183359995484352
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,8,128,1,float16,fp8,1,0.012439999729394913
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,8,128,1,float16,float16,3,0.01189119964838028
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,8,128,1,float16,fp8,3,0.01247360035777092
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,8,128,1,float16,float16,7,0.011955200135707856
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,8,128,1,float16,fp8,7,0.012452799826860428
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,8,128,1,float16,float16,15,0.0117807999253273
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,8,128,1,float16,fp8,15,0.01265919953584671
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,8,128,1,float16,float16,31,0.011779200285673141
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,8,128,1,float16,fp8,31,0.012670400738716125
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,8,128,1,float16,float16,63,0.011828800290822982
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,8,128,1,float16,fp8,63,0.012683199346065521
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,8,128,1,float16,float16,127,0.012083200365304947
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,8,128,1,float16,float16,255,0.013708800077438354
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,8,128,1,float16,fp8,255,0.014815999567508698
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,8,128,1,float16,float16,511,0.016680000722408293
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,1,128,1,float16,float16,1,0.011668799817562104
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,8,128,1,float16,fp8,511,0.018423999845981597
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,8,128,1,float16,float16,1023,0.01785600036382675
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,8,128,1,float16,fp8,1023,0.017543999850749968
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,8,128,1,float16,float16,2047,0.01985439956188202
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,8,128,1,float16,fp8,2047,0.021222400665283202
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,1,128,1,float16,float16,1,0.014739200472831726
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,96,4,128,1,float16,float16,511,0.018054400384426118
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,4,128,1,float16,float16,15,0.011879999935626984
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,1,128,1,float16,fp8,3,0.016228799521923066
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,1,128,1,float16,float16,7,0.015399999916553497
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,1,128,1,float16,fp8,7,0.016172799468040466
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,4,128,1,float16,float16,127,0.01183359995484352
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,4,128,1,float16,fp8,127,0.012520000338554382
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,1,128,1,float16,float16,31,0.014950400590896607
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,1,128,1,float16,fp8,31,0.01598079949617386
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,1,128,1,float16,float16,63,0.014998400211334228
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,1,128,1,float16,fp8,63,0.015883199870586395
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,1,128,1,float16,float16,127,0.015515199303627015
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,4,128,1,float16,float16,2047,0.017833599448204042
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,1,128,1,float16,fp8,127,0.015824000537395477
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,1,128,1,float16,float16,255,0.017591999471187593
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,1,128,1,float16,fp8,255,0.01748799979686737
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,1,128,1,float16,float16,511,0.020897600054740905
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,1,128,1,float16,fp8,511,0.021240000426769257
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,1,128,1,float16,float16,1023,0.035252800583839415
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,1,128,1,float16,fp8,1023,0.030294400453567506
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,2,128,1,float16,float16,1,0.018713599443435668
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,2,128,1,float16,fp8,1,0.02004159986972809
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,2,128,1,float16,float16,3,0.018563200533390046
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,2,128,1,float16,fp8,3,0.020151999592781068
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,2,128,1,float16,float16,7,0.018572799861431122
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,2,128,1,float16,fp8,7,0.020153599977493285
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,2,128,1,float16,float16,15,0.018751999735832213
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,2,128,1,float16,fp8,15,0.020084799826145174
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,2,128,1,float16,float16,31,0.01879200041294098
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,2,128,1,float16,fp8,31,0.02009119987487793
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,2,128,1,float16,float16,63,0.018831999599933626
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,2,128,1,float16,fp8,63,0.02008959949016571
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,2,128,1,float16,float16,127,0.019046400487422944
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,2,128,1,float16,fp8,127,0.02027200013399124
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,2,128,1,float16,float16,255,0.022867199778556824
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,2,128,1,float16,fp8,255,0.024182400107383727
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,2,128,1,float16,float16,511,0.03490239977836609
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,2,128,1,float16,fp8,511,0.0320576012134552
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,2,128,1,float16,float16,1023,0.05347679853439331
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,2,128,1,float16,fp8,1023,0.04847199916839599
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,1,128,1,float16,fp8,1,0.015564799308776855
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,4,128,1,float16,fp8,1,0.02924320101737976
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,1,128,1,float16,float16,3,0.015118399262428283
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,4,128,1,float16,fp8,3,0.029097598791122437
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,4,128,1,float16,float16,7,0.026099199056625368
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,4,128,1,float16,fp8,7,0.029075199365615846
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,4,128,1,float16,float16,15,0.02616479992866516
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,4,128,1,float16,fp8,15,0.02914080023765564
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,4,128,1,float16,float16,31,0.02611519992351532
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,4,128,1,float16,fp8,31,0.029099199175834655
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,4,128,1,float16,float16,63,0.026254400610923767
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,4,128,1,float16,fp8,63,0.029259198904037477
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,1,128,1,float16,fp8,15,0.01615840047597885
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,4,128,1,float16,float16,127,0.02689119875431061
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,4,128,1,float16,fp8,127,0.02916960120201111
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,4,128,1,float16,float16,255,0.039934399724006656
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,4,128,1,float16,float16,511,0.05498719811439514
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,4,128,1,float16,fp8,511,0.05428000092506409
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,4,128,1,float16,float16,1023,0.08858240246772767
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,4,128,1,float16,fp8,1023,0.08072479963302612
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,8,128,1,float16,float16,1,0.04068320095539093
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,8,128,1,float16,float16,3,0.04065119922161102
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,8,128,1,float16,fp8,3,0.0468176007270813
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,8,128,1,float16,float16,7,0.04038879871368408
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,8,128,1,float16,fp8,7,0.04681279957294464
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,8,128,1,float16,float16,15,0.04056319892406464
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,8,128,1,float16,fp8,15,0.046928000450134275
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,96,8,128,1,float16,fp8,127,0.012747199833393097
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,8,128,1,float16,float16,31,0.04054880142211914
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,8,128,1,float16,fp8,31,0.047049599885940555
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,8,128,1,float16,fp8,63,0.04688639938831329
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,8,128,1,float16,float16,127,0.0477183997631073
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,8,128,1,float16,float16,255,0.06283519864082336
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,8,128,1,float16,fp8,255,0.06599519848823547
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,4,128,1,float16,float16,1,0.02617279887199402
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,8,128,1,float16,float16,511,0.09175840020179749
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,4,128,1,float16,float16,3,0.02606559991836548
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,8,128,1,float16,fp8,511,0.09460639953613281
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,8,128,1,float16,float16,1023,0.15694719552993774
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,8,128,1,float16,fp8,1023,0.1451792001724243
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,96,1,128,1,float16,fp8,1,0.023358400166034698
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,96,1,128,1,float16,float16,3,0.021190400421619415
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,96,1,128,1,float16,fp8,3,0.022806400060653688
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,96,1,128,1,float16,float16,7,0.021166400611400606
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,96,1,128,1,float16,fp8,7,0.022852799296379088
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,96,1,128,1,float16,float16,15,0.021379199624061585
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,4,128,1,float16,fp8,255,0.03775680065155029
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,96,1,128,1,float16,fp8,15,0.02274720072746277
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,96,1,128,1,float16,float16,31,0.021222400665283202
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,96,1,128,1,float16,fp8,31,0.023511999845504762
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,96,1,128,1,float16,fp8,63,0.02280319929122925
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,96,1,128,1,float16,float16,127,0.02231519967317581
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,96,1,128,1,float16,fp8,127,0.022819200158119203
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,96,1,128,1,float16,float16,255,0.027724799513816834
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,96,1,128,1,float16,fp8,255,0.026859200000762938
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,8,128,1,float16,fp8,1,0.04686239957809448
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,96,2,128,1,float16,float16,3,0.028547200560569762
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,96,2,128,1,float16,fp8,3,0.0320607990026474
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,96,2,128,1,float16,float16,7,0.02869440019130707
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,8,128,1,float16,float16,63,0.04188799858093262
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,96,2,128,1,float16,fp8,7,0.032051199674606325
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,96,2,128,1,float16,float16,15,0.028527998924255372
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,96,2,128,1,float16,fp8,15,0.03195199966430664
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,96,2,128,1,float16,float16,31,0.028648000955581666
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,96,2,128,1,float16,fp8,31,0.03205280005931854
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,96,2,128,1,float16,float16,63,0.02900480031967163
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,96,2,128,1,float16,fp8,63,0.03222079873085022
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,96,1,128,1,float16,float16,1,0.021857599914073943
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,96,2,128,1,float16,fp8,127,0.03197120130062103
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,96,2,128,1,float16,float16,255,0.04340479969978332
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,96,2,128,1,float16,fp8,255,0.039980798959732056
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,96,4,128,1,float16,float16,1,0.04336639940738678
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,96,1,128,1,float16,float16,63,0.021217599511146545
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,96,4,128,1,float16,fp8,1,0.04967679977416992
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,96,4,128,1,float16,float16,3,0.043403199315071105
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,96,4,128,1,float16,fp8,3,0.04973439872264862
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,96,4,128,1,float16,float16,7,0.04345279932022095
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,96,2,128,1,float16,float16,1,0.0285504013299942
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,96,4,128,1,float16,float16,15,0.04336960017681122
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,96,4,128,1,float16,fp8,15,0.049635198712348935
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,96,4,128,1,float16,float16,31,0.04343999922275543
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,96,4,128,1,float16,fp8,31,0.04959200024604797
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,96,4,128,1,float16,float16,63,0.046348801255226134
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,96,4,128,1,float16,fp8,63,0.04966399967670441
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,96,4,128,1,float16,float16,127,0.051316797733306885
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,96,4,128,1,float16,fp8,127,0.05153759717941284
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,96,4,128,1,float16,float16,255,0.06693440079689025
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,96,4,128,1,float16,fp8,255,0.06839680075645446
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,96,8,128,1,float16,float16,1,0.07159680128097534
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,96,8,128,1,float16,fp8,1,0.08473600149154663
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,96,8,128,1,float16,float16,3,0.07164480090141297
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,96,8,128,1,float16,fp8,3,0.08488320112228394
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,96,8,128,1,float16,float16,7,0.07168319821357727
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,8,128,1,float16,fp8,127,0.048456001281738284
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,96,8,128,1,float16,float16,15,0.07192479968070983
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,96,8,128,1,float16,fp8,15,0.0847216010093689
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,96,8,128,1,float16,fp8,31,0.08474559783935547
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,96,8,128,1,float16,float16,63,0.07939360141754151
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,96,8,128,1,float16,fp8,63,0.08966079950332642
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,96,2,128,1,float16,float16,127,0.03181599974632263
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,96,8,128,1,float16,float16,127,0.08273599743843078
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,96,1,128,1,float16,float16,15,0.015063999593257904
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,96,8,128,1,float16,fp8,127,0.09520639777183533
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,96,8,128,1,float16,float16,255,0.11113439798355103
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,96,1,128,1,float16,float16,1,0.034995201230049136
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,96,8,128,1,float16,fp8,255,0.1205456018447876
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,96,1,128,1,float16,fp8,1,0.03831999897956848
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,96,1,128,1,float16,float16,3,0.03495840132236481
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,96,1,128,1,float16,fp8,3,0.0380160003900528
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,96,1,128,1,float16,float16,7,0.035113599896430966
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,96,1,128,1,float16,fp8,7,0.03825120031833649
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,96,1,128,1,float16,float16,15,0.03504799902439117
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,96,1,128,1,float16,fp8,15,0.038359999656677246
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,96,1,128,1,float16,float16,31,0.03545440137386322
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,96,1,128,1,float16,fp8,31,0.038334399461746216
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,96,1,128,1,float16,fp8,63,0.03913280069828033
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,96,2,128,1,float16,fp8,1,0.03144319951534271
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,96,1,128,1,float16,fp8,127,0.040174400806427
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,96,2,128,1,float16,float16,1,0.048958399891853334
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,96,2,128,1,float16,fp8,1,0.055371201038360594
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,96,2,128,1,float16,float16,3,0.04914880096912384
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,96,2,128,1,float16,fp8,3,0.05539199709892273
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,96,2,128,1,float16,fp8,7,0.05545920133590698
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,96,2,128,1,float16,float16,15,0.049532800912857056
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,96,2,128,1,float16,fp8,15,0.05544480085372925
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,96,2,128,1,float16,float16,31,0.050134402513504026
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,96,2,128,1,float16,fp8,31,0.05554720163345337
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,96,8,128,1,float16,fp8,7,0.0848688006401062
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,96,2,128,1,float16,float16,63,0.05478079915046692
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,96,8,128,1,float16,float16,31,0.0729695975780487
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,96,2,128,1,float16,float16,127,0.05855839848518372
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,96,2,128,1,float16,fp8,127,0.060915201902389526
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,96,4,128,1,float16,fp8,1,0.09037759900093079
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,96,4,128,1,float16,float16,3,0.07723680138587952
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,96,4,128,1,float16,float16,7,0.07715680003166199
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,96,4,128,1,float16,fp8,7,0.09041759967803956
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,96,4,128,1,float16,float16,15,0.07746239900588989
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,96,4,128,1,float16,fp8,15,0.09031360149383545
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,96,4,128,1,float16,float16,31,0.08194400072097778
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,96,4,128,1,float16,fp8,7,0.04952479898929596
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,96,4,128,1,float16,fp8,31,0.0908191978931427
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,96,4,128,1,float16,float16,63,0.085726398229599
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,96,4,128,1,float16,fp8,63,0.0962768018245697
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,96,4,128,1,float16,float16,127,0.08743360042572021
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,96,2,128,1,float16,float16,7,0.04918720126152039
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,96,8,128,1,float16,float16,1,0.13073439598083497
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,96,8,128,1,float16,fp8,1,0.15797439813613892
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,96,8,128,1,float16,float16,3,0.13124799728393555
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,96,8,128,1,float16,fp8,3,0.15802559852600098
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,96,8,128,1,float16,float16,7,0.1314895987510681
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,96,8,128,1,float16,fp8,7,0.15829440355300903
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,96,2,128,1,float16,fp8,63,0.05669280290603638
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,96,4,128,1,float16,float16,1,0.07724959850311279
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,96,8,128,1,float16,fp8,15,0.15802559852600098
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,96,8,128,1,float16,float16,31,0.1441488027572632
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,96,8,128,1,float16,float16,63,0.14537279605865477
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,96,8,128,1,float16,fp8,63,0.17036800384521483
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,96,4,128,1,float16,fp8,3,0.0902239978313446
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,96,8,128,1,float16,float16,127,0.15036799907684326
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,96,8,128,1,float16,fp8,127,0.17842400074005127
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,1,128,1,float16,fp8,1,0.012387199699878693
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,1,128,1,float16,float16,3,0.011744000017642975
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,1,128,1,float16,fp8,3,0.012588800489902496
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,1,128,1,float16,float16,7,0.011883199959993363
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,1,128,1,float16,fp8,7,0.012479999661445617
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,1,128,1,float16,float16,15,0.011697600036859513
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,96,1,128,1,float16,float16,63,0.036564800143241885
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,1,128,1,float16,fp8,15,0.012436799705028534
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,96,1,128,1,float16,float16,127,0.04208639860153198
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,1,128,1,float16,float16,63,0.011873599886894227
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,1,128,1,float16,fp8,63,0.012675200402736665
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,1,128,1,float16,float16,127,0.011988800019025803
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,1,128,1,float16,fp8,127,0.012715199589729309
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,1,128,1,float16,float16,255,0.013649600744247436
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,1,128,1,float16,fp8,255,0.01465280055999756
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,1,128,1,float16,float16,511,0.01640319973230362
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,1,128,1,float16,fp8,511,0.01807200014591217
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,1,128,1,float16,float16,1023,0.01679839938879013
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,1,128,1,float16,fp8,1023,0.017907199263572694
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,1,128,1,float16,float16,2047,0.017739200592041017
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,1,128,1,float16,fp8,2047,0.01841440051794052
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,2,128,1,float16,float16,1,0.011931200325489045
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,2,128,1,float16,fp8,1,0.012603199481964112
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,2,128,1,float16,float16,3,0.011907199770212174
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,96,8,128,1,float16,float16,15,0.13608800172805785
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,2,128,1,float16,fp8,3,0.012489599734544754
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,2,128,1,float16,float16,7,0.011924800276756287
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,2,128,1,float16,float16,15,0.01189119964838028
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,2,128,1,float16,fp8,15,0.012636800110340119
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,2,128,1,float16,float16,31,0.011823999881744384
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,2,128,1,float16,fp8,31,0.012489599734544754
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,2,128,1,float16,float16,63,0.01173280030488968
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,2,128,1,float16,fp8,63,0.012457600235939026
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,2,128,1,float16,float16,127,0.01189119964838028
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,2,128,1,float16,fp8,127,0.012430399656295776
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,2,128,1,float16,float16,255,0.013715200126171112
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,2,128,1,float16,fp8,255,0.014703999459743499
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,2,128,1,float16,float16,511,0.016627199947834015
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,1,128,1,float16,fp8,31,0.01241919994354248
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,2,128,1,float16,fp8,511,0.018537600338459016
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,2,128,1,float16,float16,1023,0.016568000614643096
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,2,128,1,float16,fp8,1023,0.017609600722789765
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,2,128,1,float16,float16,2047,0.018644799292087556
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,2,128,1,float16,fp8,2047,0.01791519969701767
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,4,128,1,float16,float16,1,0.012283200025558471
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,96,4,128,1,float16,fp8,127,0.09987679719924927
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,4,128,1,float16,float16,3,0.012039999663829803
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,4,128,1,float16,float16,7,0.012003199756145477
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,4,128,1,float16,fp8,7,0.012809599936008453
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,4,128,1,float16,float16,15,0.011972799897193909
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,4,128,1,float16,fp8,15,0.012656000256538392
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,4,128,1,float16,float16,31,0.011886399984359742
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,4,128,1,float16,fp8,31,0.012641599774360657
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,4,128,1,float16,float16,63,0.012051200121641159
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,4,128,1,float16,fp8,63,0.0127920001745224
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,4,128,1,float16,float16,127,0.012292800098657608
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,4,128,1,float16,fp8,127,0.012803199887275695
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,4,128,1,float16,float16,255,0.013913600146770478
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,4,128,1,float16,fp8,255,0.014735999703407287
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,4,128,1,float16,float16,511,0.01664319932460785
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,2,128,1,float16,fp8,7,0.012457600235939026
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,4,128,1,float16,fp8,511,0.018294399976730345
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,96,8,128,1,float16,fp8,31,0.16337599754333496
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,4,128,1,float16,fp8,1023,0.017800000309944154
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,4,128,1,float16,float16,2047,0.0199072003364563
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,4,128,1,float16,fp8,2047,0.021580800414085388
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,1,128,1,float16,float16,1,0.01175519973039627
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,8,128,1,float16,fp8,1,0.0127920001745224
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,8,128,1,float16,fp8,3,0.012912000715732574
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,8,128,1,float16,fp8,7,0.012945599853992462
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,8,128,1,float16,float16,15,0.012067200243473053
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,8,128,1,float16,fp8,15,0.012929600477218629
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,8,128,1,float16,float16,31,0.012019199877977371
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,8,128,1,float16,fp8,31,0.012878400087356568
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,8,128,1,float16,float16,63,0.011953599750995636
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,8,128,1,float16,fp8,63,0.012763200700283051
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,1,128,1,float16,float16,31,0.011772800236940384
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,8,128,1,float16,fp8,127,0.012759999930858612
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,8,128,1,float16,float16,255,0.01388159990310669
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,8,128,1,float16,fp8,255,0.01481119990348816
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,4,128,1,float16,fp8,1,0.012703999876976013
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,8,128,1,float16,float16,511,0.017126399278640746
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,8,128,1,float16,fp8,511,0.018639999628067016
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,8,128,1,float16,float16,1023,0.020110400021076204
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,8,128,1,float16,fp8,1023,0.02120320051908493
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,8,128,1,float16,float16,2047,0.030393600463867188
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,8,128,1,float16,fp8,2047,0.02706719934940338
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,96,1,128,1,float16,float16,1,0.061294400691986085
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,96,1,128,1,float16,fp8,1,0.06422560214996338
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,96,1,128,1,float16,float16,3,0.06166239976882935
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,96,1,128,1,float16,fp8,3,0.06422880291938782
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,96,1,128,1,float16,float16,7,0.06186720132827759
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,96,1,128,1,float16,fp8,7,0.06470720171928405
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,96,1,128,1,float16,float16,15,0.06258559823036194
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,96,1,128,1,float16,fp8,15,0.06464639902114869
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,96,1,128,1,float16,float16,31,0.06317440271377564
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,96,1,128,1,float16,fp8,31,0.06597920060157776
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,96,1,128,1,float16,float16,63,0.06500639915466308
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,96,1,128,1,float16,fp8,63,0.06848000288009644
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,96,2,128,1,float16,float16,1,0.08883839845657349
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,4,128,1,float16,float16,1023,0.01793919950723648
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,96,2,128,1,float16,fp8,1,0.0983792006969452
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,8,128,1,float16,float16,1,0.012238399684429168
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,8,128,1,float16,float16,3,0.012280000001192093
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,96,2,128,1,float16,fp8,3,0.09898880124092102
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,8,128,1,float16,float16,7,0.012041600048542022
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,96,2,128,1,float16,fp8,7,0.09952639937400817
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,96,2,128,1,float16,float16,15,0.09001280069351196
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,96,2,128,1,float16,fp8,15,0.10065280199050904
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,96,2,128,1,float16,float16,31,0.09080960154533387
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,96,2,128,1,float16,fp8,31,0.10275520086288452
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,96,2,128,1,float16,float16,63,0.09434080123901367
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,96,2,128,1,float16,fp8,63,0.1046895980834961
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,96,4,128,1,float16,float16,1,0.14766240119934082
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,96,4,128,1,float16,fp8,1,0.16880160570144653
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,96,4,128,1,float16,float16,3,0.1474560022354126
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,96,4,128,1,float16,fp8,3,0.1692304015159607
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,96,4,128,1,float16,float16,7,0.1479904055595398
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,96,4,128,1,float16,fp8,7,0.1692623972892761
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,96,4,128,1,float16,float16,15,0.14836640357971193
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,96,4,128,1,float16,fp8,15,0.17216960191726685
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,96,4,128,1,float16,float16,31,0.1494223952293396
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,4,128,1,float16,fp8,3,0.012796799838542938
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,96,4,128,1,float16,fp8,31,0.1742624044418335
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,96,4,128,1,float16,float16,63,0.15275199413299562
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,96,4,128,1,float16,fp8,63,0.17661279439926147
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,96,8,128,1,float16,float16,1,0.26683840751647947
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,96,8,128,1,float16,float16,3,0.2672352075576782
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,96,8,128,1,float16,fp8,3,0.30870239734649657
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,96,8,128,1,float16,float16,7,0.2670192003250122
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,96,8,128,1,float16,fp8,7,0.31284160614013673
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,96,8,128,1,float16,float16,15,0.2680016040802002
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,96,8,128,1,float16,fp8,15,0.31700959205627444
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,96,8,128,1,float16,float16,31,0.2701695919036865
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,96,8,128,1,float16,fp8,31,0.31876800060272215
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,96,8,128,1,float16,float16,63,0.2716736078262329
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,96,8,128,1,float16,fp8,63,0.319868803024292
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,96,2,128,1,float16,float16,3,0.08944799900054931
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,96,1,128,1,float16,fp8,1,0.12046079635620117
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,96,1,128,1,float16,float16,3,0.10985440015792847
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,96,2,128,1,float16,float16,7,0.08971840143203735
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,96,1,128,1,float16,fp8,3,0.12050080299377441
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,96,1,128,1,float16,float16,7,0.10994880199432373
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,96,1,128,1,float16,fp8,7,0.12072319984436035
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,96,1,128,1,float16,float16,15,0.11004639863967895
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,96,1,128,1,float16,float16,31,0.11123520135879517
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,96,1,128,1,float16,fp8,31,0.12174400091171264
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,96,2,128,1,float16,float16,1,0.16273119449615478
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,96,2,128,1,float16,fp8,1,0.1871600031852722
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,96,2,128,1,float16,fp8,3,0.18738240003585815
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,96,2,128,1,float16,float16,7,0.16279679536819458
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,96,2,128,1,float16,fp8,7,0.1874783992767334
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,96,2,128,1,float16,float16,15,0.16305279731750488
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,96,2,128,1,float16,fp8,15,0.1883952021598816
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,96,2,128,1,float16,float16,31,0.1644255995750427
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,96,2,128,1,float16,fp8,31,0.18948160409927367
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,96,4,128,1,float16,float16,1,0.27588639259338377
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,96,8,128,1,float16,fp8,1,0.30831360816955566
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,96,4,128,1,float16,fp8,1,0.32683520317077636
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,96,4,128,1,float16,float16,3,0.2763456106185913
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,96,4,128,1,float16,fp8,3,0.32746880054473876
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,96,4,128,1,float16,float16,7,0.27677600383758544
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,96,4,128,1,float16,float16,15,0.2775536060333252
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,96,4,128,1,float16,fp8,7,0.32655839920043944
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,96,4,128,1,float16,fp8,15,0.3288415908813477
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,96,4,128,1,float16,float16,31,0.2802256107330322
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,96,4,128,1,float16,fp8,31,0.3311199903488159
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,96,1,128,1,float16,float16,1,0.10961920022964478
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,96,8,128,1,float16,float16,1,0.5110527992248535
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,96,8,128,1,float16,float16,3,0.5111728191375733
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,96,8,128,1,float16,fp8,1,0.6102960109710693
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,96,1,128,1,float16,fp8,15,0.1207919955253601
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,96,8,128,1,float16,fp8,3,0.6112559795379638
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,96,8,128,1,float16,fp8,7,0.6086351871490479
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,96,2,128,1,float16,float16,3,0.16259520053863524
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,96,8,128,1,float16,float16,15,0.5129280090332031
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,1,128,1,float16,fp8,1,0.012811200320720672
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,96,8,128,1,float16,float16,31,0.5172351837158203
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,1,128,1,float16,fp8,3,0.012579199671745301
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,96,8,128,1,float16,fp8,31,0.612713623046875
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,1,128,1,float16,fp8,7,0.013016000390052795
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,1,128,1,float16,float16,15,0.012030400335788727
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,1,128,1,float16,float16,31,0.012137600034475327
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,1,128,1,float16,fp8,31,0.012801599502563477
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,1,128,1,float16,float16,63,0.012097600102424621
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,1,128,1,float16,fp8,63,0.012916800379753113
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,1,128,1,float16,float16,127,0.01207199990749359
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,1,128,1,float16,fp8,127,0.012756800651550293
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,1,128,1,float16,float16,255,0.013840000331401824
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,1,128,1,float16,fp8,255,0.014908799529075622
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,1,128,1,float16,float16,511,0.017020800709724428
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,1,128,1,float16,fp8,511,0.018464000523090364
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,1,128,1,float16,float16,1023,0.017833599448204042
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,1,128,1,float16,fp8,1023,0.01855680048465729
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,1,128,1,float16,float16,2047,0.019947199523448943
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,1,128,1,float16,fp8,2047,0.01912800073623657
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,2,128,1,float16,float16,1,0.012251199781894683
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,2,128,1,float16,fp8,1,0.013240000605583191
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,2,128,1,float16,float16,3,0.012203200161457062
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,96,8,128,1,float16,float16,7,0.5123519897460938
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,2,128,1,float16,fp8,3,0.012929600477218629
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,2,128,1,float16,float16,7,0.01226240023970604
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,2,128,1,float16,fp8,7,0.013128000497817992
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,96,8,128,1,float16,float16,127,0.011972799897193909
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,2,128,1,float16,float16,15,0.012281599640846252
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,2,128,1,float16,fp8,15,0.012923200428485871
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,1,128,1,float16,float16,7,0.012025599926710128
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,2,128,1,float16,fp8,31,0.012942400574684144
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,1,128,1,float16,fp8,15,0.013006399571895599
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,2,128,1,float16,fp8,63,0.012751999497413635
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,2,128,1,float16,float16,127,0.012470400333404541
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,2,128,1,float16,fp8,127,0.01311199963092804
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,2,128,1,float16,float16,255,0.014185599982738495
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,2,128,1,float16,fp8,255,0.01510239988565445
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,2,128,1,float16,float16,511,0.01716320067644119
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,2,128,1,float16,fp8,511,0.018596799671649934
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,2,128,1,float16,float16,1023,0.01852159947156906
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,2,128,1,float16,fp8,1023,0.018270400166511536
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,2,128,1,float16,float16,2047,0.020529599487781526
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,2,128,1,float16,fp8,2047,0.021913599967956544
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,4,128,1,float16,float16,1,0.01223519966006279
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,4,128,1,float16,fp8,1,0.0130048006772995
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,4,128,1,float16,float16,3,0.01218239963054657
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,4,128,1,float16,fp8,3,0.012931199371814727
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,4,128,1,float16,float16,7,0.012080000340938568
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,4,128,1,float16,fp8,7,0.013094399869441987
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,4,128,1,float16,float16,15,0.01241919994354248
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,4,128,1,float16,fp8,15,0.013180799782276154
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,4,128,1,float16,float16,31,0.012280000001192093
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,4,128,1,float16,fp8,31,0.013033600151538849
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,4,128,1,float16,float16,63,0.012380799651145935
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,4,128,1,float16,fp8,63,0.012945599853992462
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,4,128,1,float16,float16,127,0.012494400143623352
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,4,128,1,float16,fp8,127,0.012796799838542938
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,4,128,1,float16,float16,255,0.014193600416183472
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,4,128,1,float16,fp8,255,0.014987200498580933
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,4,128,1,float16,float16,511,0.017215999960899352
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,4,128,1,float16,fp8,511,0.018606400489807128
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,4,128,1,float16,float16,1023,0.020025600492954255
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,4,128,1,float16,fp8,1023,0.02131039947271347
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,4,128,1,float16,float16,2047,0.031064000725746155
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,4,128,1,float16,fp8,2047,0.02754560112953186
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,8,128,1,float16,float16,1,0.012382400035858155
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,8,128,1,float16,fp8,1,0.013220800459384919
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,8,128,1,float16,fp8,3,0.013222399353981017
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,8,128,1,float16,float16,7,0.012356799840927125
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,8,128,1,float16,fp8,7,0.013072000443935394
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,8,128,1,float16,float16,15,0.012256000190973282
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,8,128,1,float16,fp8,15,0.01305759996175766
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,8,128,1,float16,float16,31,0.012200000137090683
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,1,128,1,float16,float16,1,0.012211199849843979
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,8,128,1,float16,float16,63,0.012222400307655335
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,96,8,128,1,float16,fp8,15,0.6103856086730957
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,8,128,1,float16,fp8,63,0.013043199479579926
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,8,128,1,float16,float16,127,0.012382400035858155
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,8,128,1,float16,fp8,127,0.013166399300098419
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,8,128,1,float16,float16,255,0.014399999380111694
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,2,128,1,float16,float16,63,0.012438400089740754
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,8,128,1,float16,float16,511,0.01756799966096878
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,8,128,1,float16,fp8,511,0.018587200343608855
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,8,128,1,float16,float16,1023,0.029606398940086365
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,8,128,1,float16,fp8,1023,0.025956800580024718
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,8,128,1,float16,float16,2047,0.044300800561904906
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,8,128,1,float16,fp8,2047,0.040515199303627014
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,1,128,1,float16,float16,1,0.013684800267219544
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,1,128,1,float16,float16,3,0.013529600203037262
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,1,128,1,float16,fp8,3,0.014144000411033631
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,1,128,1,float16,float16,7,0.013068799674510957
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,1,128,1,float16,fp8,7,0.013867199420928955
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,1,128,1,float16,float16,15,0.013388800621032714
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,1,128,1,float16,fp8,15,0.01424960047006607
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,1,128,1,float16,fp8,31,0.01430400013923645
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,1,128,1,float16,float16,63,0.013710400462150574
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,1,128,1,float16,fp8,63,0.013913600146770478
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,1,128,1,float16,float16,127,0.013364799320697784
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,1,128,1,float16,fp8,127,0.014054399728775025
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,1,128,1,float16,float16,255,0.015371200442314149
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,8,128,1,float16,float16,3,0.012465599924325943
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,1,128,1,float16,float16,511,0.018699200451374055
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,1,128,1,float16,fp8,511,0.02000479996204376
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,1,128,1,float16,float16,1023,0.02293439954519272
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,1,128,1,float16,fp8,1023,0.023923200368881226
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,8,128,1,float16,fp8,31,0.013049599528312684
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,1,128,1,float16,float16,3,0.01204800009727478
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,2,128,1,float16,float16,31,0.01204639971256256
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,1,128,1,float16,float16,4095,0.05004159808158874
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,96,8,128,1,float16,fp8,255,0.0152319997549057
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,2,128,1,float16,float16,1,0.013169600069522858
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,2,128,1,float16,fp8,1,0.01454080045223236
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,2,128,1,float16,float16,3,0.013255999982357025
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,2,128,1,float16,fp8,3,0.014316800236701965
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,2,128,1,float16,float16,7,0.013262400031089782
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,2,128,1,float16,fp8,7,0.014318400621414184
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,2,128,1,float16,fp8,15,0.014305600523948669
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,2,128,1,float16,float16,31,0.01372160017490387
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,2,128,1,float16,fp8,31,0.014531199634075165
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,2,128,1,float16,float16,63,0.013760000467300415
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,2,128,1,float16,fp8,63,0.014435200393199921
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,2,128,1,float16,float16,127,0.013822400569915771
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,1,128,1,float16,float16,31,0.013504000008106231
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,2,128,1,float16,fp8,127,0.014422400295734406
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,2,128,1,float16,float16,255,0.015347200632095336
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,2,128,1,float16,fp8,255,0.016070400178432465
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,2,128,1,float16,float16,511,0.01834239959716797
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,2,128,1,float16,fp8,511,0.019991999864578246
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,1,128,1,float16,fp8,255,0.016422399878501893
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,2,128,1,float16,float16,1023,0.03140000104904175
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,2,128,1,float16,fp8,1023,0.028095999360084535
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,2,128,1,float16,float16,2047,0.046430400013923644
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,2,128,1,float16,fp8,2047,0.0417136013507843
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,2,128,1,float16,float16,4095,0.07799519896507263
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,2,128,1,float16,fp8,4095,0.0665615975856781
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,4,128,1,float16,float16,1,0.017182399332523347
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,4,128,1,float16,fp8,1,0.018775999546051025
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,4,128,1,float16,fp8,3,0.018681600689888
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,1,128,1,float16,fp8,2047,0.030219200253486633
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,4,128,1,float16,float16,7,0.017294399440288544
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,4,128,1,float16,fp8,7,0.018958400189876556
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,4,128,1,float16,float16,15,0.017313599586486816
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,4,128,1,float16,fp8,15,0.018958400189876556
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,1,128,1,float16,fp8,4095,0.043716800212860105
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,4,128,1,float16,fp8,31,0.01886080056428909
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,4,128,1,float16,float16,63,0.01759680062532425
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,4,128,1,float16,float16,127,0.017665599286556245
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,2,128,1,float16,float16,15,0.013288000226020813
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,4,128,1,float16,fp8,127,0.019054399430751802
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,4,128,1,float16,float16,255,0.020627200603485107
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,4,128,1,float16,fp8,255,0.022966399788856506
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,4,128,1,float16,float16,511,0.032025599479675294
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,4,128,1,float16,fp8,511,0.03059839904308319
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,4,128,1,float16,float16,1023,0.0497408002614975
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,4,128,1,float16,float16,2047,0.07982400059700012
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,4,128,1,float16,fp8,2047,0.07011680006980896
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,4,128,1,float16,float16,4095,0.13955520391464232
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,4,128,1,float16,fp8,4095,0.11973760128021241
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,8,128,1,float16,float16,1,0.024400000274181367
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,8,128,1,float16,fp8,1,0.027526399493217467
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,8,128,1,float16,float16,3,0.024635200202465058
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,8,128,1,float16,fp8,3,0.027670401334762573
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,8,128,1,float16,float16,7,0.024907200038433074
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,8,128,1,float16,fp8,7,0.027716800570487976
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,8,128,1,float16,float16,15,0.02473759949207306
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,8,128,1,float16,fp8,15,0.027558401226997375
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,8,128,1,float16,float16,31,0.024846400320529937
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,8,128,1,float16,fp8,31,0.027803200483322143
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,4,128,1,float16,float16,31,0.017452800273895265
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,8,128,1,float16,fp8,63,0.027880001068115234
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,4,128,1,float16,fp8,63,0.018775999546051025
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,8,128,1,float16,float16,127,0.025496000051498414
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,8,128,1,float16,fp8,127,0.027806401252746582
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,8,128,1,float16,float16,255,0.037811198830604555
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,8,128,1,float16,fp8,255,0.036032000184059144
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,1,128,1,float16,fp8,1,0.014243200421333313
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,8,128,1,float16,float16,511,0.05140799880027771
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,8,128,1,float16,fp8,511,0.05268959999084473
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,8,128,1,float16,fp8,1023,0.07845439910888671
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,4,128,1,float16,fp8,1023,0.04618239998817444
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,8,128,1,float16,float16,2047,0.1439728021621704
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,1,128,1,float16,float16,1,0.011495999991893768
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,8,128,1,float16,float16,4095,0.2595760107040405
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,1,128,1,float16,fp8,1,0.012276799976825714
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,8,128,1,float16,fp8,4095,0.22488958835601808
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,1,128,1,float16,fp8,3,0.012204799801111221
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,1,128,1,float16,float16,3,0.011539199948310852
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,1,128,1,float16,fp8,7,0.012212800234556198
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,1,128,1,float16,float16,7,0.011550399661064147
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,1,128,1,float16,fp8,15,0.012201599776744843
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,1,128,1,float16,fp8,31,0.012230399996042252
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,1,128,1,float16,float16,63,0.011486399918794632
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,1,128,1,float16,fp8,63,0.012406399846076966
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,1,128,1,float16,float16,127,0.011547199636697768
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,1,128,1,float16,fp8,127,0.012444800138473511
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,1,128,1,float16,float16,255,0.01324319988489151
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,1,128,1,float16,fp8,255,0.014455999433994293
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,1,128,1,float16,float16,511,0.016118399798870087
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,1,128,1,float16,fp8,511,0.018156799674034118
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,1,128,1,float16,float16,1023,0.015894399583339693
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,4,128,1,float16,float16,3,0.017073599994182585
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,1,128,1,float16,fp8,1023,0.017105600237846373
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,1,128,1,float16,float16,2047,0.01597760021686554
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,1,128,1,float16,fp8,2047,0.01732639968395233
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,1,128,1,float16,float16,4095,0.01666239947080612
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,1,128,1,float16,fp8,4095,0.017715199291706084
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,2,128,1,float16,float16,1,0.011350400000810622
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,2,128,1,float16,fp8,1,0.012176000326871873
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,2,128,1,float16,float16,3,0.01143999993801117
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,2,128,1,float16,fp8,3,0.012212800234556198
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,2,128,1,float16,float16,7,0.011472000181674958
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,2,128,1,float16,fp8,7,0.012200000137090683
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,2,128,1,float16,float16,15,0.011497599631547928
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,2,128,1,float16,fp8,15,0.012345600128173827
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,2,128,1,float16,float16,31,0.011528000235557556
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,2,128,1,float16,fp8,31,0.012414400279521943
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,2,128,1,float16,float16,63,0.011505600064992905
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,2,128,1,float16,fp8,63,0.012455999851226807
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,2,128,1,float16,float16,127,0.011459200084209442
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,2,128,1,float16,fp8,127,0.012326399981975555
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,2,128,1,float16,float16,255,0.013006399571895599
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,2,128,1,float16,fp8,255,0.014343999326229095
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,2,128,1,float16,float16,511,0.016016000509262086
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,2,128,1,float16,fp8,511,0.018156799674034118
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,8,128,1,float16,float16,1023,0.08444479703903199
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,2,128,1,float16,float16,1023,0.01570879966020584
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,2,128,1,float16,fp8,1023,0.016787199676036833
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,2,128,1,float16,fp8,2047,0.01703519970178604
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,2,128,1,float16,float16,4095,0.016942399740219116
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,2,128,1,float16,fp8,4095,0.017899200320243835
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,4,128,1,float16,float16,1,0.011617600172758102
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,4,128,1,float16,fp8,1,0.01231039986014366
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,4,128,1,float16,float16,3,0.011657600104808808
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,4,128,1,float16,fp8,3,0.012671999633312225
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,4,128,1,float16,float16,7,0.011659199744462967
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,4,128,1,float16,fp8,7,0.012139199674129486
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,4,128,1,float16,float16,15,0.01149120032787323
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,4,128,1,float16,fp8,15,0.012110400199890136
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,4,128,1,float16,float16,31,0.011475200206041336
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,4,128,1,float16,fp8,31,0.012110400199890136
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,4,128,1,float16,float16,63,0.011959999799728394
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,4,128,1,float16,fp8,63,0.01228799968957901
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,4,128,1,float16,float16,127,0.011662399768829346
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,1,128,1,float16,float16,2047,0.03532640039920807
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,4,128,1,float16,fp8,127,0.012166400253772736
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,4,128,1,float16,fp8,255,0.014238399267196656
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,4,128,1,float16,float16,511,0.01621599942445755
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,4,128,1,float16,fp8,511,0.018228800594806673
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,8,128,1,float16,float16,63,0.025033599138259886
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,4,128,1,float16,float16,1023,0.01589599996805191
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,4,128,1,float16,float16,2047,0.016016000509262086
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,4,128,1,float16,fp8,2047,0.017267200350761413
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,4,128,1,float16,float16,4095,0.01687999963760376
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,4,128,1,float16,fp8,4095,0.017556799948215483
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,8,128,1,float16,float16,1,0.01170400008559227
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,8,128,1,float16,fp8,1,0.01218400001525879
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,8,128,1,float16,float16,3,0.011505600064992905
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,8,128,1,float16,fp8,3,0.012324800342321396
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,8,128,1,float16,float16,7,0.01154559999704361
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,2,128,1,float16,float16,2047,0.01597920060157776
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,8,128,1,float16,fp8,7,0.01233920007944107
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,8,128,1,float16,float16,15,0.011376000195741653
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,8,128,1,float16,fp8,15,0.012375999987125397
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,8,128,1,float16,float16,31,0.011420799791812897
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,8,128,1,float16,fp8,31,0.012331199645996094
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,8,128,1,float16,float16,63,0.01141439974308014
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,8,128,1,float16,fp8,63,0.0122079998254776
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,8,128,1,float16,float16,127,0.011406400054693223
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,8,128,1,float16,fp8,127,0.012140800058841706
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,1,128,1,float16,float16,15,0.011644800007343293
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,8,128,1,float16,fp8,255,0.014313599467277527
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,1,128,1,float16,float16,31,0.011627200245857238
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,8,128,1,float16,float16,511,0.016271999478340148
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,8,128,1,float16,float16,1023,0.015966400504112244
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,8,128,1,float16,fp8,1023,0.016748799383640288
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,8,128,1,float16,float16,2047,0.016212800145149232
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,8,128,1,float16,fp8,2047,0.017084799706935883
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,8,128,1,float16,float16,4095,0.0181536003947258
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,8,128,1,float16,fp8,4095,0.01791200041770935
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,1,128,1,float16,float16,1,0.01138240024447441
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,1,128,1,float16,fp8,1,0.012153600156307221
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,1,128,1,float16,float16,3,0.011486399918794632
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,1,128,1,float16,fp8,3,0.012267199903726577
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,1,128,1,float16,float16,7,0.011526399850845337
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,1,128,1,float16,float16,15,0.011561600118875503
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,1,128,1,float16,fp8,15,0.012352000176906585
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,1,128,1,float16,float16,31,0.011550399661064147
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,1,128,1,float16,fp8,31,0.012331199645996094
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,4,128,1,float16,float16,255,0.013232000172138214
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,1,128,1,float16,float16,63,0.011457599699497223
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,1,128,1,float16,fp8,63,0.012406399846076966
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,1,128,1,float16,float16,127,0.011383999884128571
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,1,128,1,float16,fp8,127,0.0124208003282547
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,1,128,1,float16,float16,255,0.01316159963607788
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,1,128,1,float16,fp8,255,0.014467200636863709
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,1,128,1,float16,float16,511,0.016166399419307708
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,1,128,1,float16,float16,1023,0.016150400042533875
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,1,128,1,float16,fp8,1023,0.017392000555992125
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,1,128,1,float16,float16,2047,0.01643519997596741
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,1,128,1,float16,float16,4095,0.017159999907016756
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,1,128,1,float16,fp8,4095,0.017979200184345245
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,2,128,1,float16,float16,1,0.011537600308656693
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,2,128,1,float16,fp8,1,0.012375999987125397
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,2,128,1,float16,float16,3,0.011619199812412263
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,64,8,128,1,float16,fp8,2047,0.12513920068740844
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,2,128,1,float16,fp8,3,0.012356799840927125
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,2,128,1,float16,fp8,7,0.01239679977297783
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,2,128,1,float16,float16,15,0.01165279969573021
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,2,128,1,float16,fp8,15,0.012276799976825714
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,2,128,1,float16,float16,31,0.01146399974822998
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,8,128,1,float16,float16,255,0.013220800459384919
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,2,128,1,float16,float16,63,0.011601600050926208
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,8,128,1,float16,fp8,511,0.01791359931230545
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,2,128,1,float16,float16,127,0.011668799817562104
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,2,128,1,float16,fp8,127,0.012200000137090683
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,2,128,1,float16,float16,255,0.013092799484729767
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,2,128,1,float16,fp8,255,0.014230400323867798
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,2,128,1,float16,float16,511,0.016156800091266632
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,1,128,1,float16,fp8,7,0.012366399914026261
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,2,128,1,float16,float16,1023,0.016014400124549865
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,2,128,1,float16,fp8,1023,0.017156800627708434
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,2,128,1,float16,float16,2047,0.016392000019550323
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,64,4,128,1,float16,fp8,1023,0.01712159961462021
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,2,128,1,float16,fp8,2047,0.017367999255657195
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,1,128,1,float16,fp8,2047,0.017552000284194947
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,2,128,1,float16,float16,4095,0.01700640022754669
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,2,128,1,float16,fp8,4095,0.017819200456142426
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,4,128,1,float16,fp8,1,0.012268800288438797
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,4,128,1,float16,float16,3,0.011398400366306304
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,4,128,1,float16,fp8,3,0.012296000123023986
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,4,128,1,float16,float16,7,0.011615999788045884
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,4,128,1,float16,fp8,7,0.012321600317955017
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,2,128,1,float16,float16,7,0.011321599781513213
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,4,128,1,float16,fp8,15,0.012150400131940842
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,4,128,1,float16,float16,31,0.011430399864912033
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,4,128,1,float16,fp8,31,0.012164799869060517
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,4,128,1,float16,float16,63,0.011633600294589996
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,2,128,1,float16,fp8,63,0.012300799787044524
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,4,128,1,float16,float16,127,0.011617600172758102
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,4,128,1,float16,fp8,127,0.012350399792194367
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,4,128,1,float16,float16,255,0.013240000605583191
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,4,128,1,float16,fp8,255,0.014103999733924866
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,4,128,1,float16,float16,511,0.01611039936542511
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,2,128,1,float16,fp8,511,0.017980800569057466
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,4,128,1,float16,float16,1023,0.015964800119400026
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,4,128,1,float16,fp8,1023,0.01719679981470108
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,4,128,1,float16,float16,2047,0.016648000478744505
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,4,128,1,float16,fp8,2047,0.01749439984560013
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,4,128,1,float16,float16,4095,0.018380799889564516
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,4,128,1,float16,fp8,4095,0.01791519969701767
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,8,128,1,float16,float16,1,0.011561600118875503
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,1,128,1,float16,fp8,511,0.01831679940223694
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,8,128,1,float16,float16,3,0.011593600362539291
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,8,128,1,float16,fp8,3,0.012316799908876418
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,8,128,1,float16,float16,7,0.011428800225257874
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,4,128,1,float16,float16,1,0.01143999993801117
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,8,128,1,float16,fp8,7,0.012190400063991547
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,8,128,1,float16,float16,15,0.011462400108575821
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,8,128,1,float16,fp8,15,0.012294399738311767
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,8,128,1,float16,float16,31,0.011502400040626526
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,8,128,1,float16,fp8,31,0.01239679977297783
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,8,128,1,float16,float16,63,0.011524800211191177
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,8,128,1,float16,float16,127,0.011630400270223617
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,8,128,1,float16,fp8,127,0.012697599828243256
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,4,128,1,float16,float16,15,0.011561600118875503
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,8,128,1,float16,float16,255,0.013631999492645264
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,8,128,1,float16,fp8,255,0.014382399618625641
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,8,128,1,float16,float16,511,0.016652800142765045
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,8,128,1,float16,fp8,511,0.01828320026397705
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,2,128,1,float16,fp8,31,0.01225920021533966
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,8,128,1,float16,float16,2047,0.017679999768733978
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,4,128,1,float16,fp8,63,0.012366399914026261
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,8,128,1,float16,fp8,2047,0.01786559969186783
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,8,128,1,float16,float16,4095,0.021593600511550903
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,1,128,1,float16,float16,1,0.012441600114107132
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,1,128,1,float16,fp8,1,0.013414399325847625
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,1,128,1,float16,float16,3,0.012299200147390365
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,1,128,1,float16,fp8,3,0.01316000074148178
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,4,128,1,float16,fp8,511,0.017803199589252472
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,1,128,1,float16,fp8,7,0.013264000415802002
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,1,128,1,float16,float16,15,0.012435200065374375
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,1,128,1,float16,fp8,15,0.013265599310398103
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,1,128,1,float16,float16,31,0.012559999525547028
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,1,128,1,float16,fp8,31,0.013476799428462981
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,1,128,1,float16,float16,63,0.012411200255155564
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,1,128,1,float16,fp8,63,0.013212800025939941
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,1,128,1,float16,float16,127,0.01268800050020218
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,8,128,1,float16,fp8,1,0.012275200337171555
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,1,128,1,float16,fp8,127,0.013385599851608277
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,1,128,1,float16,float16,255,0.014289599657058717
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,1,128,1,float16,fp8,255,0.01528320014476776
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,1,128,1,float16,float16,511,0.01752000004053116
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,1,128,1,float16,fp8,511,0.019358399510383605
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,1,128,1,float16,float16,1023,0.01931679993867874
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,1,128,1,float16,float16,2047,0.022023999691009523
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,1,128,1,float16,fp8,2047,0.022422400116920472
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,1,128,1,float16,float16,4095,0.03377119898796081
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,1,128,1,float16,fp8,4095,0.0288783997297287
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,8,128,1,float16,fp8,63,0.01241919994354248
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,2,128,1,float16,fp8,1,0.013078400492668152
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,2,128,1,float16,float16,3,0.012559999525547028
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,2,128,1,float16,fp8,3,0.01340319961309433
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,2,128,1,float16,float16,7,0.012375999987125397
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,2,128,1,float16,fp8,7,0.013444800674915314
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,2,128,1,float16,fp8,15,0.01329600065946579
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,2,128,1,float16,float16,31,0.012630400061607362
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,8,128,1,float16,float16,1023,0.0160848006606102
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,2,128,1,float16,fp8,31,0.013359999656677246
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,8,128,1,float16,fp8,1023,0.017022399604320525
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,2,128,1,float16,fp8,63,0.013371199369430542
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,2,128,1,float16,float16,127,0.012915199995040894
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,2,128,1,float16,float16,255,0.014660799503326416
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,64,8,128,1,float16,fp8,4095,0.021804800629615782
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,2,128,1,float16,fp8,255,0.015497599542140961
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,2,128,1,float16,float16,511,0.017803199589252472
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,2,128,1,float16,fp8,511,0.019182400405406953
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,2,128,1,float16,float16,1023,0.021190400421619415
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,2,128,1,float16,fp8,1023,0.022073599696159362
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,2,128,1,float16,float16,2047,0.03223839998245239
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,1,128,1,float16,float16,7,0.012404800206422806
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,2,128,1,float16,float16,4095,0.046793600916862486
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,2,128,1,float16,fp8,4095,0.042368000745773314
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,4,128,1,float16,float16,1,0.012796799838542938
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,4,128,1,float16,fp8,1,0.013569599390029908
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,4,128,1,float16,float16,3,0.01276479959487915
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,4,128,1,float16,fp8,3,0.013592000305652618
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,4,128,1,float16,float16,7,0.012947200238704682
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,4,128,1,float16,fp8,7,0.01361120045185089
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,4,128,1,float16,float16,15,0.012905600666999816
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,4,128,1,float16,fp8,15,0.013601599633693695
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,4,128,1,float16,float16,31,0.01284160017967224
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,4,128,1,float16,fp8,31,0.013631999492645264
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,1,128,1,float16,fp8,1023,0.01902880072593689
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,4,128,1,float16,fp8,63,0.013510400056838989
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,4,128,1,float16,float16,127,0.012740799784660339
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,2,128,1,float16,float16,1,0.012676799297332763
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,4,128,1,float16,fp8,127,0.013363200426101684
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,4,128,1,float16,float16,255,0.014812800288200378
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,4,128,1,float16,float16,511,0.017878399789333345
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,2,128,1,float16,float16,15,0.012441600114107132
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,4,128,1,float16,fp8,511,0.019336000084877014
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,4,128,1,float16,float16,1023,0.02996160089969635
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,4,128,1,float16,fp8,1023,0.02728799879550934
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,4,128,1,float16,float16,2047,0.04514720141887665
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,4,128,1,float16,fp8,2047,0.0406143993139267
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,2,128,1,float16,float16,63,0.012729600071907043
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,2,128,1,float16,fp8,127,0.013257600367069244
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,4,128,1,float16,fp8,4095,0.06606720089912414
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,8,128,1,float16,fp8,1,0.01793439984321594
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,8,128,1,float16,fp8,3,0.01788319945335388
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,8,128,1,float16,float16,7,0.016569599509239197
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,8,128,1,float16,fp8,7,0.017849600315093993
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,8,128,1,float16,float16,15,0.016764800250530242
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,2,128,1,float16,fp8,2047,0.028174400329589844
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,8,128,1,float16,fp8,15,0.018054400384426118
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,8,128,1,float16,float16,31,0.016806399822235106
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,8,128,1,float16,fp8,31,0.018143999576568603
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,8,128,1,float16,float16,63,0.016862399876117706
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,8,128,1,float16,float16,127,0.017115199565887453
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,8,128,1,float16,fp8,127,0.018287999927997588
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,8,128,1,float16,float16,255,0.020787200331687926
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,8,128,1,float16,fp8,255,0.02234559953212738
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,8,128,1,float16,float16,511,0.030164799094200133
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,8,128,1,float16,fp8,511,0.029707199335098265
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,8,128,1,float16,float16,1023,0.04782879948616028
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,8,128,1,float16,fp8,1023,0.044772800803184507
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,4,128,1,float16,float16,63,0.013031999766826629
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,8,128,1,float16,float16,2047,0.07829440236091614
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,8,128,1,float16,fp8,2047,0.06901440024375916
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,8,128,1,float16,float16,4095,0.13711520433425903
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,4,128,1,float16,fp8,255,0.015507200360298156
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,1,128,1,float16,fp8,1,0.012470400333404541
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,1,128,1,float16,float16,1,0.011657600104808808
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,1,128,1,float16,float16,3,0.011563199758529662
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,1,128,1,float16,fp8,3,0.012409599870443344
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,1,128,1,float16,float16,7,0.011550399661064147
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,1,128,1,float16,float16,15,0.01162400022149086
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,1,128,1,float16,fp8,15,0.012280000001192093
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,4,128,1,float16,float16,4095,0.07593920230865478
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,8,128,1,float16,float16,3,0.016857600212097167
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,1,128,1,float16,float16,63,0.011726400256156922
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,1,128,1,float16,fp8,63,0.012257599830627441
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,1,128,1,float16,fp8,127,0.012212800234556198
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,1,128,1,float16,float16,255,0.013281600177288055
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,1,128,1,float16,fp8,255,0.014366400241851807
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,1,128,1,float16,float16,511,0.016278399527072905
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,8,128,1,float16,fp8,63,0.018136000633239745
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,1,128,1,float16,float16,1023,0.016353599727153778
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,1,128,1,float16,fp8,1023,0.017212800681591034
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,1,128,1,float16,float16,2047,0.016457599401474
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,1,128,1,float16,fp8,2047,0.01749120056629181
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,1,128,1,float16,float16,4095,0.017372800409793852
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,1,128,1,float16,fp8,4095,0.018244799971580506
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,2,128,1,float16,float16,1,0.011611200124025344
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,2,128,1,float16,fp8,1,0.012204799801111221
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,2,128,1,float16,float16,3,0.01141439974308014
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,2,128,1,float16,fp8,3,0.012108799815177918
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,2,128,1,float16,float16,7,0.011638399958610535
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,2,128,1,float16,fp8,7,0.01228479966521263
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,2,128,1,float16,float16,15,0.011657600104808808
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,2,128,1,float16,fp8,15,0.012332800030708312
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,2,128,1,float16,float16,31,0.01165440008044243
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,2,128,1,float16,fp8,31,0.01223199963569641
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,2,128,1,float16,float16,63,0.011510399729013443
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,1,128,1,float16,fp8,7,0.012414400279521943
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,2,128,1,float16,fp8,63,0.012188799679279327
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,2,128,1,float16,float16,127,0.011710400134325028
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,2,128,1,float16,fp8,127,0.01218239963054657
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,2,128,1,float16,float16,255,0.013382400572299957
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,2,128,1,float16,fp8,255,0.01414559930562973
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,8,128,1,float16,float16,1,0.016612799465656282
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,2,128,1,float16,float16,511,0.016281600296497344
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,2,128,1,float16,fp8,511,0.017956799268722533
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,2,128,1,float16,fp8,1023,0.017081600427627564
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,1,128,1,float16,fp8,31,0.012515200674533844
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,2,128,1,float16,fp8,2047,0.01751679927110672
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,2,128,1,float16,float16,4095,0.018467199802398682
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,2,128,1,float16,fp8,4095,0.018246400356292724
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,4,128,1,float16,float16,1,0.011456000059843064
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,1,128,1,float16,float16,127,0.011532799899578094
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,4,128,1,float16,fp8,1,0.01228640004992485
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,4,128,1,float16,float16,3,0.011446399986743927
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,4,128,1,float16,fp8,3,0.012160000205039979
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,4,128,1,float16,float16,7,0.011564800143241882
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,4,128,1,float16,fp8,7,0.012206400185823441
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,4,128,1,float16,fp8,15,0.012198399752378464
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,4,128,1,float16,float16,31,0.011564800143241882
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,4,128,1,float16,fp8,31,0.012200000137090683
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,1,128,1,float16,fp8,511,0.018145599961280824
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,4,128,1,float16,float16,63,0.011665599793195725
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,4,128,1,float16,fp8,63,0.012265600264072418
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,4,128,1,float16,float16,127,0.011587200313806533
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,4,128,1,float16,fp8,127,0.012252800166606903
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,4,128,1,float16,float16,255,0.013366399705410004
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,4,128,1,float16,fp8,255,0.01441120058298111
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,4,128,1,float16,float16,511,0.016344000399112702
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,4,128,1,float16,fp8,511,0.01788959950208664
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,4,128,1,float16,float16,1023,0.01637279987335205
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,4,128,1,float16,fp8,1023,0.017134399712085725
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,4,128,1,float16,float16,2047,0.01802240014076233
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,4,128,1,float16,fp8,2047,0.017715199291706084
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,4,128,1,float16,float16,4095,0.02125920057296753
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,4,128,1,float16,fp8,4095,0.021393600106239318
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,8,128,1,float16,float16,1,0.012064000219106674
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,8,128,1,float16,fp8,1,0.012723200023174286
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,8,128,1,float16,float16,3,0.01167680025100708
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,8,128,1,float16,fp8,3,0.012727999687194824
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,8,128,1,float16,float16,7,0.011659199744462967
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,8,128,1,float16,fp8,7,0.012529599666595458
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,8,128,1,float16,float16,15,0.011883199959993363
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,8,128,1,float16,fp8,15,0.01255040019750595
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,8,128,1,float16,float16,31,0.011894399672746659
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,8,128,1,float16,fp8,31,0.012540799379348756
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,8,128,1,float16,float16,63,0.01170239970088005
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,8,128,1,float16,fp8,63,0.01268800050020218
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,8,128,1,float16,float16,127,0.01173119992017746
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,8,128,1,float16,fp8,127,0.012559999525547028
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,8,128,1,float16,float16,255,0.013505600392818451
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,8,128,1,float16,fp8,255,0.01494240015745163
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,8,128,1,float16,float16,511,0.016897599399089813
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,8,128,1,float16,fp8,511,0.018492799997329713
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,8,128,1,float16,float16,1023,0.017822399735450745
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,8,128,1,float16,fp8,1023,0.017497600615024568
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,8,128,1,float16,float16,2047,0.019734400510787963
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,8,128,1,float16,fp8,2047,0.021265600621700288
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,8,128,1,float16,float16,4095,0.03025600016117096
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,8,128,1,float16,fp8,4095,0.027062401175498962
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,1,128,1,float16,float16,1,0.015196800231933594
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,1,128,1,float16,float16,31,0.011419200152158738
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,1,128,1,float16,float16,3,0.015329599380493164
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,1,128,1,float16,fp8,3,0.015572799742221833
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,2,128,1,float16,float16,2047,0.01656160056591034
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,1,128,1,float16,fp8,7,0.01600479930639267
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,1,128,1,float16,float16,15,0.01478559970855713
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,1,128,1,float16,fp8,15,0.015590399503707886
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,1,128,1,float16,float16,31,0.014830400049686433
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,1,128,1,float16,fp8,31,0.0159840002655983
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,1,128,1,float16,fp8,63,0.016232000291347505
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,1,128,1,float16,float16,127,0.015515199303627015
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,1,128,1,float16,fp8,127,0.01584160029888153
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,1,128,1,float16,float16,255,0.017473599314689635
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,1,128,1,float16,fp8,255,0.018219199776649476
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,1,128,1,float16,float16,511,0.020814399421215057
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,1,128,1,float16,fp8,511,0.021678400039672852
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,1,128,1,float16,float16,1023,0.03428640067577362
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,1,128,1,float16,fp8,1023,0.03043999969959259
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,1,128,1,float16,float16,2047,0.0495824009180069
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,1,128,1,float16,fp8,2047,0.04305759966373444
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,2,128,1,float16,float16,1,0.01865759938955307
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,2,128,1,float16,fp8,1,0.02008160054683685
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,2,128,1,float16,float16,3,0.018619200587272643
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,2,128,1,float16,fp8,3,0.020342400670051573
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,64,8,128,1,float16,fp8,4095,0.11941280364990234
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,2,128,1,float16,float16,7,0.018753600120544434
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,2,128,1,float16,float16,15,0.018828800320625304
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,2,128,1,float16,fp8,15,0.02038400024175644
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,2,128,1,float16,float16,31,0.018719999492168425
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,2,128,1,float16,fp8,31,0.020428800582885744
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,2,128,1,float16,float16,63,0.018747200071811677
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,1,128,1,float16,fp8,1,0.015510399639606477
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,2,128,1,float16,float16,127,0.01878879964351654
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,2,128,1,float16,float16,1023,0.01600479930639267
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,2,128,1,float16,float16,255,0.02237119972705841
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,2,128,1,float16,fp8,255,0.02402079999446869
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,2,128,1,float16,float16,511,0.03447999954223633
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,2,128,1,float16,fp8,511,0.031760001182556154
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,1,128,1,float16,float16,63,0.014920000731945039
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,2,128,1,float16,fp8,1023,0.04786880016326904
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,64,4,128,1,float16,float16,15,0.011659199744462967
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,2,128,1,float16,float16,2047,0.08208799958229065
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,2,128,1,float16,fp8,2047,0.07174239754676819
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,4,128,1,float16,fp8,1,0.02889919877052307
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,4,128,1,float16,fp8,3,0.029128000140190125
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,4,128,1,float16,float16,7,0.026259198784828186
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,4,128,1,float16,fp8,7,0.02940320074558258
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,4,128,1,float16,float16,15,0.02595199942588806
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,4,128,1,float16,fp8,15,0.028988799452781676
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,4,128,1,float16,float16,31,0.025836798548698425
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,4,128,1,float16,fp8,31,0.02901119887828827
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,4,128,1,float16,float16,63,0.025923201441764833
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,4,128,1,float16,fp8,63,0.02898559868335724
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,4,128,1,float16,float16,127,0.026606398820877075
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,2,128,1,float16,fp8,7,0.020126399397850037
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,4,128,1,float16,fp8,127,0.028988799452781676
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,4,128,1,float16,float16,255,0.040617600083351135
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,2,128,1,float16,fp8,63,0.020310400426387785
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,1,128,1,float16,float16,7,0.01515360027551651
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,4,128,1,float16,fp8,511,0.05368480086326599
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,2,128,1,float16,float16,1023,0.05284159779548645
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,4,128,1,float16,float16,1023,0.08780639767646789
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,4,128,1,float16,fp8,1023,0.07977120280265808
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,4,128,1,float16,float16,1,0.02595199942588806
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,4,128,1,float16,float16,2047,0.14717600345611573
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,4,128,1,float16,float16,3,0.025839999318122864
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,8,128,1,float16,float16,1,0.040459200739860535
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,4,128,1,float16,fp8,2047,0.12641760110855102
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,8,128,1,float16,float16,3,0.040459200739860535
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,8,128,1,float16,fp8,3,0.04667200148105621
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,8,128,1,float16,float16,7,0.04039359986782074
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,8,128,1,float16,float16,15,0.040494400262832644
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,8,128,1,float16,fp8,15,0.04665440022945404
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,8,128,1,float16,float16,31,0.04062399864196777
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,8,128,1,float16,fp8,31,0.0467056006193161
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,8,128,1,float16,float16,63,0.04156000018119812
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,8,128,1,float16,fp8,63,0.046865600347518924
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,8,128,1,float16,float16,127,0.04801119863986969
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,8,128,1,float16,fp8,127,0.04779680073261261
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,8,128,1,float16,float16,255,0.06390560269355774
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,8,128,1,float16,fp8,255,0.06552799940109252
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,8,128,1,float16,float16,511,0.09063839912414551
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,8,128,1,float16,fp8,511,0.0942031979560852
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,8,128,1,float16,float16,1023,0.15495359897613525
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,8,128,1,float16,fp8,1023,0.14470399618148805
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,8,128,1,float16,float16,2047,0.2741055965423584
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,8,128,1,float16,fp8,2047,0.23699839115142823
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,4,128,1,float16,fp8,255,0.03808799982070923
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,1,128,1,float16,float16,3,0.021273599565029146
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,1,128,1,float16,fp8,3,0.023083199560642243
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,1,128,1,float16,float16,7,0.021396799385547637
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,1,128,1,float16,fp8,7,0.02332960069179535
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,1,128,1,float16,float16,15,0.021489599347114564
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,1,128,1,float16,fp8,15,0.023366400599479677
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,1,128,1,float16,float16,31,0.021457600593566894
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,1,128,1,float16,fp8,31,0.02333119958639145
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,1,128,1,float16,float16,63,0.021352000534534454
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,1,128,1,float16,fp8,63,0.023057599365711213
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,8,128,1,float16,fp8,7,0.04673599898815155
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,1,128,1,float16,float16,127,0.021583999693393707
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,1,128,1,float16,fp8,127,0.0233024001121521
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,1,128,1,float16,float16,255,0.026305601000785828
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,1,128,1,float16,fp8,255,0.02727999985218048
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,1,128,1,float16,float16,511,0.038343998789787295
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,2,128,1,float16,float16,1,0.028404799103736878
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,2,128,1,float16,fp8,1,0.03182719945907593
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,2,128,1,float16,float16,3,0.028364801406860353
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,2,128,1,float16,fp8,3,0.031748801469802856
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,2,128,1,float16,float16,7,0.028324800729751586
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,2,128,1,float16,fp8,7,0.03192160129547119
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,2,128,1,float16,float16,15,0.028564798831939697
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,2,128,1,float16,float16,31,0.02858240008354187
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,2,128,1,float16,fp8,31,0.031865599751472476
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,2,128,1,float16,float16,63,0.028596800565719605
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,2,128,1,float16,fp8,63,0.03184640109539032
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,2,128,1,float16,float16,127,0.02940160036087036
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,2,128,1,float16,fp8,127,0.0316431999206543
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,4,128,1,float16,float16,511,0.05403519868850708
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,2,128,1,float16,float16,255,0.04321599900722504
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,2,128,1,float16,fp8,255,0.04046719968318939
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,2,128,1,float16,float16,511,0.05726879835128784
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,2,128,1,float16,fp8,511,0.05656319856643677
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,4,128,1,float16,float16,1,0.0434224009513855
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,4,128,1,float16,fp8,1,0.04901280105113983
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,4,128,1,float16,float16,3,0.04352479875087738
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,4,128,1,float16,fp8,3,0.04917919933795929
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,4,128,1,float16,float16,7,0.04351679980754852
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,4,128,1,float16,fp8,7,0.049262401461601255
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,4,128,1,float16,float16,15,0.043372800946235655
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,4,128,1,float16,fp8,15,0.049323201179504395
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,8,128,1,float16,fp8,1,0.046419200301170346
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,4,128,1,float16,fp8,31,0.04915359914302826
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,4,128,1,float16,float16,63,0.04451040029525757
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,4,128,1,float16,fp8,63,0.04915040135383606
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,4,128,1,float16,float16,127,0.049511998891830444
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,4,128,1,float16,fp8,127,0.05077760219573975
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,4,128,1,float16,float16,255,0.06538879871368408
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,1,128,1,float16,fp8,511,0.03504480123519897
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,4,128,1,float16,fp8,255,0.06774880290031433
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,4,128,1,float16,float16,511,0.09470880031585693
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,4,128,1,float16,fp8,511,0.0969760000705719
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,8,128,1,float16,fp8,1,0.0842848002910614
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,8,128,1,float16,fp8,3,0.08405759930610657
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,8,128,1,float16,float16,7,0.07142720222473145
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,8,128,1,float16,fp8,7,0.08431360125541687
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,8,128,1,float16,float16,15,0.07172319889068604
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,8,128,1,float16,fp8,15,0.08432319760322571
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,8,128,1,float16,float16,31,0.07148640155792237
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,1,128,1,float16,fp8,1,0.023068800568580627
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,8,128,1,float16,fp8,31,0.08431360125541687
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,64,2,128,1,float16,fp8,127,0.020022399723529816
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,8,128,1,float16,fp8,63,0.08568159937858581
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,8,128,1,float16,float16,127,0.0802511990070343
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,8,128,1,float16,fp8,127,0.09362559914588928
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,8,128,1,float16,float16,255,0.1102463960647583
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,8,128,1,float16,fp8,255,0.11948000192642212
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,4,128,1,float16,float16,31,0.04337440133094787
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,8,128,1,float16,fp8,511,0.1777567982673645
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,64,1,128,1,float16,float16,1,0.03465920090675354
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,64,1,128,1,float16,fp8,1,0.03787519931793213
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,64,1,128,1,float16,float16,3,0.034852799773216245
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,64,1,128,1,float16,fp8,3,0.0377263993024826
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,8,128,1,float16,float16,3,0.07176960110664368
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,64,1,128,1,float16,float16,7,0.03487359881401062
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,64,1,128,1,float16,fp8,7,0.03765600025653839
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,64,1,128,1,float16,float16,15,0.03470079898834229
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,64,1,128,1,float16,fp8,15,0.03771040141582489
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,64,1,128,1,float16,float16,31,0.0349263995885849
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,64,1,128,1,float16,fp8,31,0.037676799297332766
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,64,1,128,1,float16,float16,63,0.03514080047607422
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,64,1,128,1,float16,fp8,63,0.03794879913330078
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,64,1,128,1,float16,float16,127,0.03938399851322174
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,64,1,128,1,float16,fp8,127,0.037990400195121767
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,64,1,128,1,float16,float16,255,0.04980320036411286
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,64,1,128,1,float16,fp8,255,0.04633600115776062
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,64,2,128,1,float16,float16,1,0.04888159930706024
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,64,2,128,1,float16,fp8,1,0.055164802074432376
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,64,2,128,1,float16,float16,3,0.049028798937797546
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,1,128,1,float16,float16,1,0.021028800308704375
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,64,2,128,1,float16,fp8,3,0.05516639947891235
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,8,128,1,float16,float16,63,0.07862240076065063
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,64,2,128,1,float16,float16,15,0.04889920055866241
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,64,2,128,1,float16,float16,31,0.04899519979953766
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,64,2,128,1,float16,fp8,31,0.055105602741241454
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,64,2,128,1,float16,float16,63,0.052692800760269165
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,64,2,128,1,float16,fp8,63,0.055561602115631104
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,64,2,128,1,float16,float16,127,0.05745919942855835
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,64,2,128,1,float16,fp8,127,0.05811200141906738
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,64,2,128,1,float16,float16,255,0.07317280173301696
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,64,2,128,1,float16,fp8,255,0.07287359833717347
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,64,4,128,1,float16,float16,1,0.07723360061645508
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,64,4,128,1,float16,fp8,1,0.08980640172958373
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,8,128,1,float16,float16,1,0.07139040231704712
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,64,4,128,1,float16,float16,3,0.07718080282211304
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,64,4,128,1,float16,fp8,3,0.0900272011756897
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,64,4,128,1,float16,fp8,7,0.08996000289916992
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,64,4,128,1,float16,float16,15,0.0770143985748291
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,2,128,1,float16,fp8,15,0.03142240047454834
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,64,4,128,1,float16,fp8,15,0.0897711992263794
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,64,4,128,1,float16,float16,31,0.07895039916038513
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,64,4,128,1,float16,fp8,31,0.09000639915466309
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,64,4,128,1,float16,float16,63,0.08411840200424195
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,64,4,128,1,float16,fp8,63,0.0946016013622284
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,64,4,128,1,float16,float16,127,0.0856768012046814
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,64,2,128,1,float16,float16,7,0.04907520115375519
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,64,4,128,1,float16,fp8,127,0.0973471999168396
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,64,2,128,1,float16,fp8,7,0.05494400262832642
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,64,4,128,1,float16,fp8,255,0.12423039674758911
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,64,8,128,1,float16,float16,1,0.13114399909973146
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,64,8,128,1,float16,fp8,1,0.15834399461746215
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,64,8,128,1,float16,float16,3,0.1314128041267395
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,64,8,128,1,float16,fp8,3,0.15825599431991577
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,64,8,128,1,float16,float16,7,0.13138400316238402
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,64,8,128,1,float16,fp8,7,0.1578096032142639
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,64,8,128,1,float16,float16,15,0.13354079723358153
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,64,8,128,1,float16,fp8,15,0.15833439826965331
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,64,8,128,1,float16,float16,31,0.14215519428253173
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,64,8,128,1,float16,fp8,31,0.15863679647445678
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,64,8,128,1,float16,float16,63,0.143449604511261
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,64,8,128,1,float16,fp8,63,0.16907039880752564
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,64,8,128,1,float16,float16,127,0.14903680086135865
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,64,8,128,1,float16,float16,511,0.16577279567718506
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,64,8,128,1,float16,fp8,127,0.1747920036315918
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,64,8,128,1,float16,float16,255,0.20280640125274657
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,1,128,1,float16,fp8,1,0.012520000338554382
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,64,8,128,1,float16,fp8,255,0.2238368034362793
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,1,128,1,float16,fp8,7,0.012583999335765839
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,1,128,1,float16,float16,15,0.011744000017642975
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,1,128,1,float16,fp8,15,0.012513600289821625
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,1,128,1,float16,float16,31,0.011695999652147293
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,1,128,1,float16,fp8,31,0.012347199767827988
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,1,128,1,float16,float16,63,0.011521600186824799
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,64,4,128,1,float16,float16,7,0.07710080146789551
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,1,128,1,float16,fp8,63,0.012460800260305405
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,1,128,1,float16,float16,127,0.011747200042009354
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,1,128,1,float16,fp8,127,0.012531200051307678
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,1,128,1,float16,float16,255,0.013446399569511413
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,1,128,1,float16,fp8,255,0.014505599439144135
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,1,128,1,float16,float16,511,0.016491200029850005
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,1,128,1,float16,float16,1023,0.016492800414562227
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,1,128,1,float16,fp8,1023,0.01753759980201721
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,1,128,1,float16,float16,2047,0.017100800573825837
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,1,128,1,float16,fp8,2047,0.017694400250911714
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,1,128,1,float16,float16,4095,0.01903519928455353
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,1,128,1,float16,fp8,4095,0.018718400597572328
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,2,128,1,float16,float16,1,0.01175839975476265
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,2,128,1,float16,fp8,1,0.012433599680662155
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,64,2,128,1,float16,fp8,15,0.05518879890441895
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,2,128,1,float16,fp8,3,0.012556800246238708
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,2,128,1,float16,float16,7,0.011769600212574005
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,2,128,1,float16,fp8,7,0.01255200058221817
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,2,128,1,float16,float16,15,0.01170239970088005
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,2,128,1,float16,fp8,15,0.012408000230789185
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,2,128,1,float16,float16,31,0.011619199812412263
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,1,128,1,float16,float16,1,0.011739200353622437
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,2,128,1,float16,fp8,31,0.012408000230789185
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,1,128,1,float16,float16,7,0.0117807999253273
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,2,128,1,float16,fp8,63,0.012531200051307678
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,2,128,1,float16,float16,127,0.011923199892044068
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,2,128,1,float16,fp8,127,0.0126351997256279
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,2,128,1,float16,float16,255,0.013380800187587739
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,2,128,1,float16,fp8,255,0.014417600631713868
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,2,128,1,float16,float16,511,0.016396799683570863
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,2,128,1,float16,fp8,511,0.01804639995098114
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,2,128,1,float16,float16,1023,0.016499200463294984
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,2,128,1,float16,fp8,1023,0.017316800355911256
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,1,128,1,float16,fp8,511,0.018049600720405578
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,2,128,1,float16,float16,2047,0.018187199532985688
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,2,128,1,float16,fp8,2047,0.017924800515174866
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,2,128,1,float16,fp8,4095,0.021350400149822236
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,4,128,1,float16,float16,1,0.011811199784278869
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,4,128,1,float16,fp8,1,0.01255200058221817
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,4,128,1,float16,float16,3,0.011774399876594543
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,4,128,1,float16,fp8,3,0.012563200294971466
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,4,128,1,float16,float16,7,0.011985599994659424
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,4,128,1,float16,fp8,7,0.012747199833393097
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,2,128,1,float16,float16,3,0.011689600348472596
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,4,128,1,float16,fp8,15,0.012697599828243256
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,4,128,1,float16,float16,31,0.011872000247240066
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,4,128,1,float16,fp8,31,0.012835200130939483
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,4,128,1,float16,float16,63,0.011966399848461151
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,4,128,1,float16,fp8,63,0.012899200618267059
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,4,128,1,float16,float16,127,0.012033600360155106
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,4,128,1,float16,fp8,127,0.012780800461769104
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,4,128,1,float16,float16,255,0.013764800131320953
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,1,128,1,float16,float16,3,0.011695999652147293
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,4,128,1,float16,fp8,255,0.014494399726390838
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,2,128,1,float16,float16,63,0.011751999706029892
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,4,128,1,float16,float16,511,0.017025600373744964
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,4,128,1,float16,fp8,511,0.018160000443458557
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,4,128,1,float16,float16,1023,0.01797119975090027
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,4,128,1,float16,fp8,1023,0.017659200727939604
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,4,128,1,float16,fp8,2047,0.02150239944458008
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,4,128,1,float16,float16,4095,0.030531200766563415
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,4,128,1,float16,fp8,4095,0.02762399911880493
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,8,128,1,float16,float16,1,0.012201599776744843
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,8,128,1,float16,fp8,1,0.012863999605178833
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,8,128,1,float16,fp8,3,0.012894399464130402
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,8,128,1,float16,float16,7,0.012139199674129486
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,8,128,1,float16,fp8,7,0.012852799892425538
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,8,128,1,float16,float16,15,0.012015999853610992
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,8,128,1,float16,fp8,15,0.01281760036945343
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,8,128,1,float16,float16,31,0.012054400146007537
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,8,128,1,float16,fp8,31,0.012806400656700134
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,2,128,1,float16,float16,4095,0.021860800683498383
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,8,128,1,float16,float16,63,0.012204799801111221
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,8,128,1,float16,fp8,63,0.012806400656700134
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,8,128,1,float16,float16,127,0.012270399928092956
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,8,128,1,float16,fp8,127,0.012892800569534301
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,8,128,1,float16,float16,255,0.014070400595664978
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,8,128,1,float16,fp8,255,0.014860799908638
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,8,128,1,float16,float16,511,0.017148800194263458
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,8,128,1,float16,fp8,511,0.018559999763965607
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,8,128,1,float16,float16,1023,0.019862399995326997
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,8,128,1,float16,fp8,1023,0.02110559940338135
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,8,128,1,float16,float16,2047,0.030372801423072814
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,64,4,128,1,float16,float16,255,0.11475039720535278
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,8,128,1,float16,fp8,2047,0.027011200785636902
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,8,128,1,float16,float16,4095,0.04565120041370392
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,4,128,1,float16,float16,15,0.01194560006260872
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,64,1,128,1,float16,float16,1,0.056227201223373414
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,64,1,128,1,float16,fp8,1,0.06255040168762208
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,64,1,128,1,float16,fp8,3,0.06198559999465943
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,64,1,128,1,float16,float16,7,0.05601919889450073
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,64,1,128,1,float16,fp8,7,0.06267359852790833
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,64,1,128,1,float16,float16,15,0.056620800495147706
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,64,1,128,1,float16,float16,31,0.057918399572372437
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,64,1,128,1,float16,fp8,31,0.06275680065155029
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,64,1,128,1,float16,float16,63,0.06254240274429321
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,64,1,128,1,float16,fp8,63,0.06440640091896058
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,64,1,128,1,float16,float16,127,0.06622719764709473
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,64,1,128,1,float16,fp8,127,0.06851680278778076
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,64,2,128,1,float16,float16,1,0.0834879994392395
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,64,2,128,1,float16,fp8,1,0.09652000069618225
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,64,2,128,1,float16,float16,3,0.08406559824943542
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,64,2,128,1,float16,fp8,3,0.09631839990615845
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,64,2,128,1,float16,fp8,7,0.09604799747467041
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,64,2,128,1,float16,float16,15,0.08510400056838989
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,64,2,128,1,float16,fp8,15,0.09701759815216064
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,64,2,128,1,float16,float16,31,0.08833919763565064
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,64,2,128,1,float16,fp8,31,0.09815359711647034
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,64,2,128,1,float16,float16,63,0.09130719900131226
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,64,2,128,1,float16,fp8,63,0.10285919904708862
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,64,2,128,1,float16,float16,127,0.09628480076789855
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,64,2,128,1,float16,fp8,127,0.10595680475234985
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,8,128,1,float16,fp8,4095,0.041515201330184937
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,64,1,128,1,float16,float16,3,0.05621920228004455
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,64,4,128,1,float16,fp8,1,0.1641759991645813
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,64,1,128,1,float16,fp8,15,0.06216480135917664
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,64,4,128,1,float16,float16,3,0.1391327977180481
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,64,4,128,1,float16,fp8,3,0.16402239799499513
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,64,4,128,1,float16,float16,7,0.1409168004989624
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,64,4,128,1,float16,fp8,7,0.16396479606628417
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,64,4,128,1,float16,float16,15,0.14538880586624145
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,64,4,128,1,float16,fp8,15,0.16528639793395997
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,64,4,128,1,float16,float16,31,0.1475200057029724
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,1,128,1,float16,fp8,3,0.012481600046157837
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,64,4,128,1,float16,fp8,31,0.17072319984436035
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,64,4,128,1,float16,float16,63,0.14953279495239258
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,64,4,128,1,float16,float16,127,0.15481280088424682
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,64,4,128,1,float16,fp8,127,0.1779904007911682
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,8,128,1,float16,float16,3,0.012219200283288956
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,64,8,128,1,float16,float16,1,0.2518224000930786
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,64,8,128,1,float16,fp8,1,0.2952879905700684
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,64,8,128,1,float16,float16,3,0.25363519191741946
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,64,8,128,1,float16,fp8,3,0.2951008081436157
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,64,8,128,1,float16,float16,7,0.2583103895187378
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,64,8,128,1,float16,fp8,7,0.29577438831329345
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,64,8,128,1,float16,float16,15,0.26215519905090334
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,64,8,128,1,float16,fp8,15,0.3036672115325928
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,64,8,128,1,float16,float16,31,0.2641776084899902
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,64,8,128,1,float16,fp8,31,0.3167936086654663
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,64,8,128,1,float16,float16,63,0.26749119758605955
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,64,8,128,1,float16,fp8,63,0.31607038974761964
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,64,8,128,1,float16,float16,127,0.27438879013061523
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,64,1,128,1,float16,float16,1,0.10411839485168457
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,64,8,128,1,float16,fp8,127,0.3272464036941528
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,64,1,128,1,float16,fp8,1,0.11573920249938965
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,64,1,128,1,float16,float16,3,0.10454879999160767
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,64,1,128,1,float16,fp8,3,0.11573760509490967
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,64,1,128,1,float16,float16,7,0.10476000308990478
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,64,1,128,1,float16,fp8,7,0.11609280109405518
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,64,1,128,1,float16,float16,15,0.10489120483398437
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,64,1,128,1,float16,fp8,15,0.11670559644699097
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,64,1,128,1,float16,float16,31,0.10648479461669921
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,64,1,128,1,float16,fp8,31,0.11724640130996704
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,64,1,128,1,float16,float16,63,0.10885440111160279
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,64,2,128,1,float16,float16,1,0.1589311957359314
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,64,2,128,1,float16,float16,7,0.08441759943962097
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,64,2,128,1,float16,float16,3,0.15876799821853638
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,64,2,128,1,float16,fp8,3,0.1835968017578125
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,64,2,128,1,float16,float16,7,0.1588703989982605
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,64,2,128,1,float16,fp8,7,0.1840208053588867
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,64,2,128,1,float16,float16,15,0.15890560150146485
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,64,2,128,1,float16,fp8,15,0.1844688057899475
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,64,2,128,1,float16,fp8,31,0.1860751986503601
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,64,2,128,1,float16,float16,63,0.1642624020576477
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,64,2,128,1,float16,fp8,63,0.18891520500183107
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,64,4,128,1,float16,float16,1,0.1413264036178589
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,64,4,128,1,float16,float16,1,0.2724143981933594
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,64,4,128,1,float16,float16,3,0.2712831974029541
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,64,4,128,1,float16,fp8,3,0.323638391494751
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,64,4,128,1,float16,float16,7,0.271399998664856
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,64,4,128,1,float16,fp8,7,0.3239919900894165
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,64,4,128,1,float16,float16,15,0.27252960205078125
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,64,2,128,1,float16,fp8,1,0.18356800079345703
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,64,4,128,1,float16,fp8,15,0.325052809715271
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,64,4,128,1,float16,fp8,63,0.17504639625549318
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,64,4,128,1,float16,float16,31,0.27525599002838136
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,64,4,128,1,float16,fp8,31,0.32757279872894285
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,64,4,128,1,float16,float16,63,0.27889599800109866
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,64,4,128,1,float16,fp8,63,0.3308016061782837
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,64,8,128,1,float16,float16,1,0.5034704208374023
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,64,8,128,1,float16,float16,3,0.5049647808074951
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,64,8,128,1,float16,fp8,3,0.6090496063232422
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,64,8,128,1,float16,float16,7,0.5031807899475098
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,64,8,128,1,float16,fp8,7,0.6124303817749024
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,64,8,128,1,float16,float16,15,0.5044896125793457
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,64,8,128,1,float16,fp8,15,0.6070879936218262
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,64,8,128,1,float16,float16,31,0.5075952053070069
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,64,8,128,1,float16,fp8,31,0.6084303855895996
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,1,128,1,float16,float16,1,0.0118367999792099
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,1,128,1,float16,fp8,1,0.01289760023355484
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,1,128,1,float16,float16,3,0.012039999663829803
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,1,128,1,float16,fp8,3,0.01281599998474121
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,64,1,128,1,float16,fp8,63,0.11943199634552001
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,1,128,1,float16,float16,7,0.012139199674129486
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,1,128,1,float16,float16,15,0.012200000137090683
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,64,4,128,1,float16,float16,2047,0.019620800018310548
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,1,128,1,float16,float16,31,0.01186719983816147
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,1,128,1,float16,fp8,31,0.012606400251388549
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,1,128,1,float16,float16,63,0.012043199688196182
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,1,128,1,float16,fp8,63,0.01284479945898056
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,1,128,1,float16,float16,127,0.01191840022802353
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,1,128,1,float16,fp8,127,0.012628799676895142
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,1,128,1,float16,float16,255,0.013553600013256072
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,64,2,128,1,float16,float16,31,0.16057440042495727
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,1,128,1,float16,fp8,255,0.014511999487876893
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,1,128,1,float16,float16,511,0.016443200409412384
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,64,8,128,1,float16,fp8,1,0.6103536128997803
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,1,128,1,float16,float16,1023,0.017059199512004852
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,1,128,1,float16,fp8,1023,0.01775519996881485
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,1,128,1,float16,float16,2047,0.018632000684738158
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,1,128,1,float16,float16,4095,0.022486400604248048
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,1,128,1,float16,fp8,4095,0.021916800737380983
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,2,128,1,float16,float16,1,0.011876799911260606
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,2,128,1,float16,fp8,1,0.012705600261688233
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,2,128,1,float16,float16,3,0.012027200311422348
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,2,128,1,float16,fp8,3,0.01276959925889969
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,2,128,1,float16,float16,7,0.012001600116491318
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,2,128,1,float16,fp8,7,0.012856000661849975
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,2,128,1,float16,float16,15,0.012084800004959106
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,2,128,1,float16,fp8,15,0.01284639984369278
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,64,4,128,1,float16,fp8,1,0.32352960109710693
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,64,8,128,1,float16,float16,63,0.5136735916137696
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,2,128,1,float16,fp8,31,0.01289760023355484
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,2,128,1,float16,float16,63,0.01223680004477501
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,1,128,1,float16,fp8,7,0.012489599734544754
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,1,128,1,float16,fp8,15,0.012942400574684144
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,2,128,1,float16,float16,127,0.0119439996778965
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,2,128,1,float16,fp8,127,0.013211199641227722
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,2,128,1,float16,fp8,255,0.014876799285411834
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,2,128,1,float16,float16,511,0.0172447994351387
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,2,128,1,float16,fp8,511,0.018751999735832213
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,2,128,1,float16,float16,1023,0.01831679940223694
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,2,128,1,float16,fp8,1023,0.018006399273872375
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,2,128,1,float16,float16,2047,0.02033119946718216
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,2,128,1,float16,fp8,2047,0.021558399498462676
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,1,128,1,float16,fp8,511,0.01810079962015152
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,2,128,1,float16,float16,4095,0.03144960105419159
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,4,128,1,float16,float16,1,0.012275200337171555
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,1,128,1,float16,fp8,2047,0.018348799645900728
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,4,128,1,float16,fp8,1,0.013078400492668152
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,4,128,1,float16,float16,3,0.011980800330638886
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,4,128,1,float16,float16,7,0.011894399672746659
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,4,128,1,float16,fp8,7,0.012753599882125854
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,4,128,1,float16,float16,15,0.012089599668979645
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,4,128,1,float16,fp8,15,0.012932799756526947
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,4,128,1,float16,float16,31,0.012127999961376191
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,4,128,1,float16,fp8,31,0.01284479945898056
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,4,128,1,float16,float16,63,0.012160000205039979
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,4,128,1,float16,fp8,63,0.012729600071907043
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,4,128,1,float16,float16,127,0.012012799829244613
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,2,128,1,float16,float16,31,0.01197120025753975
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,4,128,1,float16,float16,255,0.013756799697875976
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,4,128,1,float16,fp8,255,0.014764800667762756
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,4,128,1,float16,float16,511,0.016947199404239655
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,64,8,128,1,float16,fp8,63,0.6120912075042725
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,4,128,1,float16,float16,1023,0.020503999292850496
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,2,128,1,float16,fp8,63,0.012907199561595917
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,4,128,1,float16,fp8,1023,0.02125760018825531
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,4,128,1,float16,float16,2047,0.03094879984855652
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,4,128,1,float16,fp8,2047,0.027243199944496154
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,4,128,1,float16,float16,4095,0.0460640013217926
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,4,128,1,float16,fp8,4095,0.041657599806785586
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,8,128,1,float16,float16,1,0.011937599629163742
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,8,128,1,float16,fp8,1,0.013094399869441987
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,8,128,1,float16,float16,3,0.012151999771595002
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,8,128,1,float16,fp8,3,0.013176000118255616
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,8,128,1,float16,float16,7,0.012091200053691863
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,8,128,1,float16,fp8,7,0.013152000308036805
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,8,128,1,float16,float16,15,0.01220960021018982
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,8,128,1,float16,fp8,15,0.013128000497817992
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,8,128,1,float16,float16,31,0.012366399914026261
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,8,128,1,float16,fp8,31,0.013076800107955932
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,8,128,1,float16,float16,63,0.012430399656295776
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,8,128,1,float16,fp8,63,0.012988799810409546
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,8,128,1,float16,fp8,127,0.013068799674510957
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,8,128,1,float16,float16,255,0.014289599657058717
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,2,128,1,float16,fp8,4095,0.027030399441719054
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,8,128,1,float16,float16,511,0.017099200189113616
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,4,128,1,float16,fp8,3,0.013176000118255616
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,8,128,1,float16,fp8,511,0.01858399957418442
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,8,128,1,float16,fp8,1023,0.02619200050830841
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,8,128,1,float16,float16,2047,0.044284799695014955
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,8,128,1,float16,fp8,2047,0.04089440107345581
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,8,128,1,float16,float16,4095,0.07466239929199218
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,8,128,1,float16,fp8,4095,0.06582239866256714
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,1,128,1,float16,float16,1,0.013577599823474885
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,4,128,1,float16,fp8,127,0.012703999876976013
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,1,128,1,float16,float16,3,0.013484799861907959
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,1,128,1,float16,fp8,3,0.014103999733924866
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,1,128,1,float16,float16,7,0.013447999954223633
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,1,128,1,float16,fp8,7,0.014110399782657624
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,1,128,1,float16,float16,15,0.013208000361919403
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,1,128,1,float16,fp8,15,0.01422560065984726
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,1,128,1,float16,float16,31,0.01329600065946579
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,4,128,1,float16,fp8,511,0.01839359998703003
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,1,128,1,float16,float16,63,0.013707199692726135
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,1,128,1,float16,float16,127,0.013457599282264709
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,1,128,1,float16,fp8,127,0.01414719969034195
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,1,128,1,float16,float16,255,0.015449599921703338
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,1,128,1,float16,fp8,255,0.016415999829769136
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,1,128,1,float16,float16,511,0.018620799481868743
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,1,128,1,float16,fp8,511,0.02003040015697479
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,1,128,1,float16,float16,1023,0.02162559926509857
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,1,128,1,float16,fp8,1023,0.022784000635147093
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,1,128,1,float16,float16,2047,0.03341439962387085
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,1,128,1,float16,fp8,2047,0.028972798585891725
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,8,128,1,float16,float16,127,0.0126351997256279
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,1,128,1,float16,float16,4095,0.04778079986572266
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,1,128,1,float16,fp8,4095,0.04248639941215515
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,8,128,1,float16,fp8,255,0.015070399641990662
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,1,128,1,float16,fp8,8191,0.06740639805793762
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,2,128,1,float16,float16,1,0.013710400462150574
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,2,128,1,float16,fp8,1,0.01419840008020401
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,2,128,1,float16,fp8,3,0.013982400298118591
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,8,128,1,float16,float16,1023,0.02970080077648163
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,2,128,1,float16,float16,7,0.01326880007982254
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,2,128,1,float16,fp8,7,0.014232000708580017
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,2,128,1,float16,fp8,15,0.01430879980325699
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,2,128,1,float16,float16,31,0.013465599715709686
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,2,128,1,float16,fp8,31,0.014232000708580017
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,2,128,1,float16,float16,63,0.013380800187587739
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,1,128,1,float16,fp8,1,0.014176000654697419
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,2,128,1,float16,fp8,63,0.01406719982624054
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,2,128,1,float16,float16,127,0.013468800485134125
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,2,128,1,float16,fp8,127,0.01406559944152832
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,1,128,1,float16,fp8,31,0.014567999541759491
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,2,128,1,float16,float16,255,0.015184000134468079
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,2,128,1,float16,fp8,255,0.016784000396728515
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,2,128,1,float16,float16,511,0.018305599689483643
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,2,128,1,float16,fp8,511,0.02008160054683685
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,2,128,1,float16,float16,1023,0.03135200142860413
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,2,128,1,float16,fp8,1023,0.027779200673103334
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,64,2,128,1,float16,float16,255,0.013643200695514678
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,2,128,1,float16,float16,2047,0.045684799551963806
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,2,128,1,float16,fp8,2047,0.041145598888397215
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,2,128,1,float16,float16,4095,0.07646080255508422
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,2,128,1,float16,float16,8191,0.1363952040672302
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,2,128,1,float16,fp8,8191,0.11631519794464111
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,4,128,1,float16,float16,1,0.017148800194263458
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,4,128,1,float16,fp8,1,0.018958400189876556
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,4,128,1,float16,float16,3,0.017478400468826295
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,4,128,1,float16,fp8,3,0.018937599658966065
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,4,128,1,float16,float16,7,0.01740480065345764
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,4,128,1,float16,fp8,7,0.018932799994945525
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,4,128,1,float16,float16,15,0.01733600050210953
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,4,128,1,float16,fp8,15,0.01897760033607483
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,4,128,1,float16,float16,31,0.01732800006866455
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,1,128,1,float16,float16,8191,0.07855839729309082
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,4,128,1,float16,float16,63,0.017454400658607483
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,4,128,1,float16,float16,127,0.017340800166130065
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,2,128,1,float16,float16,3,0.013715200126171112
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,4,128,1,float16,fp8,127,0.018824000656604768
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,4,128,1,float16,float16,255,0.02093600034713745
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,4,128,1,float16,fp8,255,0.023035199940204622
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,4,128,1,float16,float16,511,0.031350401043891904
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,4,128,1,float16,fp8,511,0.03078399896621704
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,4,128,1,float16,float16,1023,0.0489520013332367
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,4,128,1,float16,fp8,1023,0.04646719992160797
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,4,128,1,float16,float16,2047,0.07933279871940613
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,4,128,1,float16,float16,4095,0.13784639835357665
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,4,128,1,float16,fp8,4095,0.11835520267486573
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,1,128,1,float16,fp8,63,0.014299200475215912
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,4,128,1,float16,float16,8191,0.25585920810699464
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,8,128,1,float16,float16,1,0.024619199335575104
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,8,128,1,float16,fp8,1,0.027723199129104613
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,2,128,1,float16,fp8,4095,0.06567040085792542
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,8,128,1,float16,fp8,3,0.027323201298713684
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,8,128,1,float16,float16,7,0.0244159996509552
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,8,128,1,float16,fp8,7,0.027534401416778563
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,8,128,1,float16,float16,15,0.02492160052061081
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,8,128,1,float16,fp8,15,0.027777600288391113
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,8,128,1,float16,float16,31,0.024959999322891235
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,4,128,1,float16,fp8,31,0.01890240013599396
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,8,128,1,float16,float16,63,0.02497439980506897
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,8,128,1,float16,fp8,63,0.027668800950050355
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,8,128,1,float16,float16,127,0.025257599353790284
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,2,128,1,float16,float16,15,0.01326880007982254
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,8,128,1,float16,float16,255,0.03732160031795502
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,8,128,1,float16,fp8,255,0.035953599214553836
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,8,128,1,float16,float16,511,0.05049279928207397
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,8,128,1,float16,fp8,511,0.05227680206298828
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,8,128,1,float16,float16,1023,0.08328319787979126
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,8,128,1,float16,fp8,1023,0.0775488018989563
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,8,128,1,float16,float16,2047,0.1424623966217041
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,8,128,1,float16,fp8,2047,0.12500959634780884
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,8,128,1,float16,float16,4095,0.25979039669036863
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,8,128,1,float16,fp8,4095,0.22442240715026857
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,4,128,1,float16,fp8,8191,0.21765921115875245
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,8,128,1,float16,float16,3,0.0248416006565094
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,1,128,1,float16,float16,1,0.01146399974822998
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,1,128,1,float16,fp8,1,0.01218239963054657
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,8,128,1,float16,float16,8191,0.4987599849700928
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,8,128,1,float16,fp8,8191,0.4221807956695557
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,1,128,1,float16,fp8,3,0.012203200161457062
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,1,128,1,float16,float16,15,0.011436799913644791
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,1,128,1,float16,float16,7,0.011425600200891495
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,1,128,1,float16,fp8,15,0.012328000366687774
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,1,128,1,float16,fp8,31,0.012383999675512314
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,1,128,1,float16,float16,63,0.011558400094509124
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,1,128,1,float16,fp8,63,0.012363199889659882
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,8,128,1,float16,fp8,31,0.027692800760269164
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,1,128,1,float16,fp8,127,0.012142399698495865
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,4,128,1,float16,fp8,63,0.018654400110244752
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,1,128,1,float16,float16,255,0.012987199425697326
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,1,128,1,float16,fp8,255,0.014214399456977844
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,1,128,1,float16,float16,511,0.0160303995013237
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,1,128,1,float16,fp8,511,0.018222400546073915
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,8,128,1,float16,fp8,127,0.027580800652503967
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,1,128,1,float16,fp8,1023,0.016732800006866454
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,1,128,1,float16,float16,2047,0.0160288006067276
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,1,128,1,float16,float16,4095,0.01666239947080612
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,1,128,1,float16,fp8,4095,0.017761600017547608
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,1,128,1,float16,float16,8191,0.018334400653839112
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,1,128,1,float16,fp8,8191,0.01894879937171936
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,2,128,1,float16,float16,1,0.0114656001329422
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,2,128,1,float16,fp8,1,0.012161599844694138
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,2,128,1,float16,float16,3,0.011420799791812897
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,2,128,1,float16,fp8,3,0.01212640032172203
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,2,128,1,float16,float16,7,0.011262399703264236
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,2,128,1,float16,fp8,7,0.012116800248622894
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,2,128,1,float16,float16,15,0.011291199922561645
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,1,128,1,float16,float16,3,0.011367999762296677
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,2,128,1,float16,fp8,15,0.012185599654912949
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,1,128,1,float16,fp8,7,0.01207199990749359
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,2,128,1,float16,fp8,31,0.012086399644613267
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,2,128,1,float16,float16,63,0.01141119971871376
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,2,128,1,float16,fp8,63,0.012059199810028075
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,1,128,1,float16,float16,127,0.011635199934244157
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,2,128,1,float16,fp8,127,0.012303999811410903
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,2,128,1,float16,float16,255,0.013012799620628356
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,2,128,1,float16,fp8,255,0.01427839994430542
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,1,128,1,float16,float16,1023,0.015438400208950043
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,2,128,1,float16,fp8,511,0.0181551992893219
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,1,128,1,float16,fp8,2047,0.016956800222396852
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,2,128,1,float16,float16,1023,0.01571359932422638
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,2,128,1,float16,fp8,1023,0.016633599996566772
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,2,128,1,float16,float16,2047,0.016065600514411926
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,2,128,1,float16,fp8,2047,0.01684959977865219
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,2,128,1,float16,float16,4095,0.01687519997358322
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,2,128,1,float16,fp8,4095,0.017529599368572235
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,2,128,1,float16,fp8,8191,0.018729600310325622
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,4,128,1,float16,float16,1,0.011326400190591812
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,4,128,1,float16,fp8,1,0.012094400078058242
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,4,128,1,float16,float16,3,0.011348800361156463
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,4,128,1,float16,fp8,3,0.01204800009727478
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,4,128,1,float16,float16,7,0.011300799995660782
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,4,128,1,float16,fp8,7,0.012167999893426895
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,4,128,1,float16,float16,15,0.01154559999704361
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,4,128,1,float16,fp8,15,0.012230399996042252
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,4,128,1,float16,float16,31,0.011510399729013443
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,4,128,1,float16,fp8,31,0.012276799976825714
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,4,128,1,float16,float16,63,0.01143999993801117
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,4,128,1,float16,fp8,63,0.012089599668979645
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,4,128,1,float16,float16,127,0.01133280023932457
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,4,128,1,float16,fp8,127,0.012055999785661697
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,4,128,1,float16,float16,255,0.012996800243854523
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,4,128,1,float16,fp8,255,0.014120000600814819
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,4,128,1,float16,float16,511,0.016164800524711607
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,2,128,1,float16,float16,31,0.011478400230407715
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,4,128,1,float16,fp8,511,0.018139199912548067
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,1,128,1,float16,float16,31,0.011377599835395814
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,4,128,1,float16,fp8,1023,0.01695519983768463
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,4,128,1,float16,fp8,2047,0.0171424001455307
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,4,128,1,float16,fp8,4095,0.017633600533008574
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,2,128,1,float16,float16,127,0.011220800131559372
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,4,128,1,float16,float16,8191,0.0189968004822731
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,4,128,1,float16,fp8,8191,0.01882079988718033
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,8,128,1,float16,fp8,1,0.012011200189590454
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,8,128,1,float16,float16,3,0.01130400002002716
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,8,128,1,float16,fp8,3,0.011979199945926666
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,8,128,1,float16,float16,7,0.011404799669981003
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,8,128,1,float16,fp8,7,0.01220960021018982
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,8,128,1,float16,float16,15,0.011473599821329117
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,32,4,128,1,float16,fp8,2047,0.0695904016494751
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,8,128,1,float16,fp8,15,0.012172800302505494
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,2,128,1,float16,float16,8191,0.01822720021009445
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,8,128,1,float16,fp8,31,0.012222400307655335
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,8,128,1,float16,float16,63,0.011462400108575821
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,8,128,1,float16,fp8,63,0.01218400001525879
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,8,128,1,float16,float16,127,0.011451199650764465
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,8,128,1,float16,fp8,127,0.012031999975442886
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,8,128,1,float16,float16,255,0.013148799538612366
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,8,128,1,float16,fp8,255,0.014051200449466705
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,8,128,1,float16,float16,511,0.01616799980401993
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,8,128,1,float16,fp8,511,0.01802240014076233
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,8,128,1,float16,float16,1023,0.015769599378108977
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,8,128,1,float16,float16,2047,0.016436800360679626
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,8,128,1,float16,fp8,2047,0.017206400632858276
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,8,128,1,float16,float16,4095,0.018287999927997588
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,8,128,1,float16,fp8,4095,0.017998400330543517
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,8,128,1,float16,float16,8191,0.02172800004482269
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,8,128,1,float16,fp8,8191,0.021668800711631776
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,1,128,1,float16,float16,1,0.01143999993801117
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,1,128,1,float16,fp8,1,0.012203200161457062
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,1,128,1,float16,float16,3,0.011519999802112579
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,1,128,1,float16,fp8,3,0.012392000108957291
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,1,128,1,float16,float16,7,0.011795199662446975
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,1,128,1,float16,fp8,7,0.012371200323104858
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,4,128,1,float16,float16,2047,0.0159840002655983
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,1,128,1,float16,float16,15,0.011584000289440155
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,4,128,1,float16,float16,4095,0.016942399740219116
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,1,128,1,float16,float16,31,0.011532799899578094
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,1,128,1,float16,fp8,31,0.01226079985499382
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,1,128,1,float16,fp8,63,0.012275200337171555
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,8,128,1,float16,float16,1,0.011552000045776367
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,1,128,1,float16,fp8,127,0.012302400171756744
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,2,128,1,float16,float16,511,0.016068799793720244
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,1,128,1,float16,fp8,255,0.014283199608325959
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,1,128,1,float16,float16,511,0.016208000481128693
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,1,128,1,float16,fp8,511,0.01844480037689209
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,1,128,1,float16,float16,1023,0.01584479957818985
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,8,128,1,float16,float16,31,0.011537600308656693
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,1,128,1,float16,fp8,1023,0.016908800601959227
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,1,128,1,float16,float16,2047,0.0160863995552063
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,1,128,1,float16,fp8,2047,0.01728159934282303
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,1,128,1,float16,float16,4095,0.016633599996566772
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,1,128,1,float16,fp8,4095,0.017873600125312805
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,1,128,1,float16,fp8,8191,0.018760000169277192
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,8,128,1,float16,fp8,1023,0.01674399971961975
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,2,128,1,float16,fp8,1,0.012011200189590454
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,2,128,1,float16,float16,3,0.011486399918794632
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,32,4,128,1,float16,float16,1023,0.015692800283432007
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,1,128,1,float16,fp8,15,0.012169600278139115
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,1,128,1,float16,float16,63,0.0115167997777462
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,2,128,1,float16,fp8,7,0.0123648002743721
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,1,128,1,float16,float16,127,0.011472000181674958
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,2,128,1,float16,float16,15,0.011707200109958649
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,2,128,1,float16,fp8,15,0.012164799869060517
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,2,128,1,float16,float16,31,0.011638399958610535
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,1,128,1,float16,float16,255,0.013070400059223174
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,2,128,1,float16,fp8,31,0.012166400253772736
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,2,128,1,float16,fp8,63,0.012091200053691863
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,2,128,1,float16,float16,127,0.011472000181674958
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,2,128,1,float16,fp8,127,0.01215519979596138
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,2,128,1,float16,float16,255,0.013049599528312684
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,2,128,1,float16,fp8,255,0.014326399564743042
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,2,128,1,float16,float16,511,0.016094399988651274
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,2,128,1,float16,fp8,511,0.018225599825382233
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,2,128,1,float16,float16,1023,0.015880000591278077
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,2,128,1,float16,fp8,1023,0.016950400173664094
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,2,128,1,float16,float16,2047,0.016275200247764587
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,2,128,1,float16,fp8,2047,0.017246399819850922
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,2,128,1,float16,float16,4095,0.016956800222396852
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,2,128,1,float16,fp8,4095,0.017820799350738527
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,2,128,1,float16,float16,8191,0.01915999948978424
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,2,128,1,float16,fp8,8191,0.018993599712848662
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,4,128,1,float16,float16,1,0.011579199880361556
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,4,128,1,float16,fp8,1,0.0121568001806736
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,4,128,1,float16,float16,3,0.011575999855995178
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,4,128,1,float16,fp8,3,0.012187200039625168
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,4,128,1,float16,fp8,7,0.01218239963054657
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,4,128,1,float16,float16,15,0.011539199948310852
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,4,128,1,float16,fp8,15,0.012361600250005721
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,4,128,1,float16,float16,31,0.011604800075292587
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,4,128,1,float16,fp8,31,0.012371200323104858
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,4,128,1,float16,float16,63,0.011657600104808808
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,4,128,1,float16,fp8,63,0.012270399928092956
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,4,128,1,float16,float16,127,0.011433599889278412
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,4,128,1,float16,fp8,127,0.012196800112724305
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,4,128,1,float16,float16,255,0.013206399977207184
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,4,128,1,float16,fp8,255,0.014336000382900237
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,4,128,1,float16,float16,511,0.016257600486278535
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,4,128,1,float16,fp8,511,0.017843200266361235
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,4,128,1,float16,float16,1023,0.015662400424480437
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,4,128,1,float16,fp8,1023,0.016715200245380403
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,2,128,1,float16,float16,7,0.011308799684047698
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,4,128,1,float16,float16,2047,0.016172799468040466
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,4,128,1,float16,fp8,2047,0.017367999255657195
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,4,128,1,float16,float16,4095,0.018222400546073915
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,4,128,1,float16,fp8,4095,0.01812160015106201
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,4,128,1,float16,float16,8191,0.02166240066289902
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,4,128,1,float16,fp8,8191,0.021937599778175353
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,8,128,1,float16,float16,1,0.011846400052309036
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,2,128,1,float16,float16,63,0.011727999895811081
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,8,128,1,float16,fp8,1,0.012468799948692322
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,8,128,1,float16,fp8,3,0.01234399974346161
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,8,128,1,float16,float16,7,0.011575999855995178
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,8,128,1,float16,fp8,7,0.01231039986014366
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,8,128,1,float16,float16,15,0.011615999788045884
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,8,128,1,float16,fp8,15,0.012280000001192093
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,8,128,1,float16,float16,31,0.011604800075292587
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,8,128,1,float16,fp8,31,0.012324800342321396
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,8,128,1,float16,fp8,63,0.01231520026922226
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,8,128,1,float16,float16,127,0.011699199676513672
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,8,128,1,float16,fp8,127,0.012489599734544754
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,1,128,1,float16,float16,8191,0.018164800107479097
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,8,128,1,float16,fp8,255,0.01440960019826889
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,2,128,1,float16,float16,1,0.011552000045776367
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,8,128,1,float16,fp8,511,0.01808159947395325
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,8,128,1,float16,float16,1023,0.016065600514411926
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,8,128,1,float16,fp8,1023,0.017051200568675994
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,8,128,1,float16,float16,2047,0.017923200130462648
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,8,128,1,float16,fp8,2047,0.017508800327777862
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,8,128,1,float16,float16,4095,0.02141599953174591
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,8,128,1,float16,fp8,4095,0.021031999588012697
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,8,128,1,float16,float16,8191,0.030235201120376587
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,8,128,1,float16,fp8,8191,0.027372801303863527
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,1,128,1,float16,float16,1,0.01234079971909523
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,1,128,1,float16,fp8,1,0.013302400708198547
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,1,128,1,float16,float16,3,0.012513600289821625
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,1,128,1,float16,fp8,3,0.013284799456596375
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,1,128,1,float16,float16,7,0.012545600533485413
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,1,128,1,float16,fp8,7,0.01329279989004135
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,1,128,1,float16,float16,15,0.012539200484752655
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,1,128,1,float16,fp8,15,0.013291199505329133
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,1,128,1,float16,float16,31,0.012539200484752655
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,1,128,1,float16,fp8,31,0.013302400708198547
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,1,128,1,float16,float16,63,0.012664000689983367
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,1,128,1,float16,fp8,63,0.01326880007982254
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,1,128,1,float16,float16,127,0.012720000743865967
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,1,128,1,float16,fp8,127,0.013145600259304047
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,1,128,1,float16,float16,255,0.014292800426483154
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,1,128,1,float16,fp8,255,0.015039999783039094
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,1,128,1,float16,float16,511,0.01740960031747818
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,1,128,1,float16,fp8,511,0.018713599443435668
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,1,128,1,float16,float16,1023,0.01860159933567047
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,1,128,1,float16,fp8,1023,0.01813279986381531
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,1,128,1,float16,float16,2047,0.02066880017518997
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,1,128,1,float16,fp8,2047,0.021835200488567352
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,1,128,1,float16,float16,4095,0.0321040004491806
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,1,128,1,float16,fp8,4095,0.027718400955200194
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,1,128,1,float16,float16,8191,0.04710400104522705
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,8,128,1,float16,float16,3,0.01180799975991249
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,2,128,1,float16,float16,1,0.012854400277137756
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,2,128,1,float16,fp8,1,0.013361600041389466
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,2,128,1,float16,float16,3,0.012652799487113953
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,8,128,1,float16,float16,63,0.011598400026559829
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,2,128,1,float16,float16,7,0.012569600343704223
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,2,128,1,float16,fp8,7,0.01345279961824417
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,8,128,1,float16,float16,255,0.013356800377368926
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,2,128,1,float16,float16,15,0.012748800218105316
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,2,128,1,float16,fp8,15,0.01334560066461563
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,8,128,1,float16,float16,511,0.016334399580955505
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,4,128,1,float16,float16,7,0.011716800183057785
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,2,128,1,float16,fp8,31,0.013355199992656708
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,2,128,1,float16,float16,63,0.012803199887275695
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,2,128,1,float16,fp8,63,0.013523200154304504
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,2,128,1,float16,fp8,127,0.013596799969673157
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,2,128,1,float16,fp8,255,0.015652799606323244
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,2,128,1,float16,float16,511,0.017718400061130523
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,2,128,1,float16,fp8,511,0.018947200477123262
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,2,128,1,float16,float16,1023,0.020371200144290925
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,2,128,1,float16,fp8,1023,0.02160159945487976
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,2,128,1,float16,float16,2047,0.03184320032596588
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,2,128,1,float16,fp8,2047,0.027803200483322143
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,2,128,1,float16,float16,4095,0.04665600061416626
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,2,128,1,float16,fp8,4095,0.04179520010948181
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,32,2,128,1,float16,fp8,3,0.012350399792194367
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,2,128,1,float16,fp8,8191,0.06682400107383728
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,4,128,1,float16,float16,1,0.012703999876976013
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,4,128,1,float16,fp8,1,0.013412800431251527
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,1,128,1,float16,fp8,8191,0.04249759912490845
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,4,128,1,float16,fp8,3,0.013601599633693695
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,4,128,1,float16,float16,7,0.012748800218105316
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,4,128,1,float16,fp8,7,0.013419200479984284
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,2,128,1,float16,fp8,3,0.013438400626182557
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,4,128,1,float16,float16,15,0.012627199292182922
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,4,128,1,float16,fp8,15,0.01356000006198883
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,4,128,1,float16,float16,31,0.012753599882125854
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,4,128,1,float16,fp8,31,0.013486400246620178
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,4,128,1,float16,float16,63,0.012806400656700134
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,4,128,1,float16,fp8,63,0.013713599741458892
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,4,128,1,float16,float16,127,0.012915199995040894
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,4,128,1,float16,fp8,127,0.013689599931240082
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,4,128,1,float16,float16,255,0.014684799313545226
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,2,128,1,float16,float16,31,0.012759999930858612
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,2,128,1,float16,float16,127,0.012619200348854064
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,2,128,1,float16,float16,255,0.015031999349594117
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,4,128,1,float16,fp8,255,0.015758399665355683
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,4,128,1,float16,fp8,511,0.019392000138759614
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,4,128,1,float16,fp8,1023,0.026771199703216553
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,4,128,1,float16,float16,2047,0.04480000138282776
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,4,128,1,float16,fp8,2047,0.040468800067901614
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,4,128,1,float16,fp8,4095,0.0653711974620819
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,4,128,1,float16,float16,8191,0.134825599193573
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,8,128,1,float16,float16,1,0.016735999286174773
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,8,128,1,float16,fp8,1,0.017827199399471284
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,8,128,1,float16,float16,3,0.016516800224781036
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,8,128,1,float16,fp8,3,0.018190400302410127
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,8,128,1,float16,fp8,7,0.018025599420070648
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,8,128,1,float16,float16,15,0.01666879951953888
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,8,128,1,float16,fp8,15,0.017903999984264375
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,8,128,1,float16,float16,31,0.016574400663375854
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,4,128,1,float16,float16,3,0.012620800733566284
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,8,128,1,float16,fp8,31,0.01801919937133789
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,8,128,1,float16,float16,63,0.016476799547672272
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,8,128,1,float16,fp8,63,0.01791519969701767
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,8,128,1,float16,float16,127,0.0166703999042511
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,8,128,1,float16,float16,255,0.02048639953136444
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,8,128,1,float16,fp8,255,0.021928000450134277
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,8,128,1,float16,float16,511,0.029702401161193846
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,8,128,1,float16,fp8,511,0.029366400837898255
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,8,128,1,float16,float16,1023,0.04710879921913147
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,8,128,1,float16,fp8,1023,0.04517279863357544
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,8,128,1,float16,float16,2047,0.07683680057525635
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,4,128,1,float16,float16,511,0.017828799784183502
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,4,128,1,float16,float16,1023,0.029552000761032104
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,8,128,1,float16,float16,4095,0.13641760349273682
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,8,128,1,float16,fp8,4095,0.11872800588607788
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,4,128,1,float16,fp8,8191,0.11555039882659912
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,8,128,1,float16,float16,8191,0.2542495965957642
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,8,128,1,float16,fp8,8191,0.2184783935546875
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,1,128,1,float16,float16,1,0.011668799817562104
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,1,128,1,float16,fp8,3,0.012332800030708312
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,1,128,1,float16,float16,3,0.011539199948310852
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,1,128,1,float16,float16,7,0.01165279969573021
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,1,128,1,float16,fp8,7,0.012382400035858155
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,2,128,1,float16,float16,8191,0.07924479842185975
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,1,128,1,float16,fp8,15,0.012408000230789185
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,8,128,1,float16,float16,7,0.016468800604343414
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,1,128,1,float16,float16,31,0.011531200259923935
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,1,128,1,float16,float16,63,0.011559999734163284
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,1,128,1,float16,fp8,63,0.012244799733161926
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,1,128,1,float16,float16,127,0.011683200299739838
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,1,128,1,float16,fp8,127,0.012257599830627441
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,1,128,1,float16,float16,255,0.01327199935913086
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,1,128,1,float16,fp8,255,0.014297600090503692
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,1,128,1,float16,float16,511,0.016156800091266632
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,1,128,1,float16,fp8,511,0.018177600204944612
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,1,128,1,float16,float16,1023,0.015798400342464446
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,1,128,1,float16,fp8,1023,0.016729600727558136
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,1,128,1,float16,float16,2047,0.016254399716854096
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,1,128,1,float16,fp8,2047,0.017367999255657195
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,1,128,1,float16,float16,4095,0.017190399765968322
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,1,128,1,float16,fp8,4095,0.017932799458503724
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,1,128,1,float16,float16,8191,0.01936960071325302
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,1,128,1,float16,fp8,8191,0.019179199635982514
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,2,128,1,float16,float16,1,0.011497599631547928
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,2,128,1,float16,fp8,1,0.012195199728012085
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,2,128,1,float16,float16,3,0.011494400352239609
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,2,128,1,float16,fp8,3,0.012361600250005721
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,2,128,1,float16,float16,7,0.011615999788045884
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,2,128,1,float16,fp8,7,0.012401600182056428
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,2,128,1,float16,float16,15,0.011708799749612808
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,8,128,1,float16,fp8,2047,0.06876479983329772
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,2,128,1,float16,fp8,15,0.012375999987125397
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,2,128,1,float16,fp8,31,0.01218239963054657
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,2,128,1,float16,float16,63,0.011521600186824799
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,2,128,1,float16,fp8,63,0.012206400185823441
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,2,128,1,float16,float16,127,0.011686400324106217
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,2,128,1,float16,fp8,127,0.012366399914026261
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,2,128,1,float16,float16,255,0.01327040046453476
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,2,128,1,float16,fp8,255,0.01417279988527298
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,1,128,1,float16,fp8,1,0.01215519979596138
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,2,128,1,float16,float16,511,0.016176000237464905
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,2,128,1,float16,fp8,511,0.01812320053577423
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,2,128,1,float16,float16,1023,0.015932799875736238
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,2,128,1,float16,fp8,1023,0.017092800140380858
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,2,128,1,float16,float16,2047,0.016441600024700166
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,1,128,1,float16,fp8,31,0.01242239996790886
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,2,128,1,float16,float16,4095,0.01847359985113144
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,2,128,1,float16,fp8,4095,0.018192000687122345
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,2,128,1,float16,fp8,8191,0.02189760059118271
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,4,128,1,float16,float16,1,0.011540800333023071
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,4,128,1,float16,fp8,1,0.012200000137090683
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,4,128,1,float16,float16,3,0.01154239997267723
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,4,128,1,float16,fp8,3,0.012272000312805176
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,4,128,1,float16,float16,7,0.011494400352239609
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,4,128,1,float16,fp8,7,0.0123648002743721
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,4,128,1,float16,float16,15,0.011695999652147293
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,4,128,1,float16,fp8,15,0.012390399724245072
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,4,128,1,float16,float16,31,0.01165120005607605
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,4,128,1,float16,fp8,31,0.012425599992275238
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,4,128,1,float16,float16,63,0.011684799939393998
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,4,128,1,float16,fp8,63,0.012348800152540206
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,4,128,1,float16,float16,127,0.01167839989066124
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,4,128,1,float16,fp8,127,0.012379200011491776
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,4,128,1,float16,float16,255,0.01329279989004135
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,4,128,1,float16,fp8,255,0.014347200095653535
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,4,128,1,float16,float16,511,0.016467200219631196
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,2,128,1,float16,float16,31,0.011662399768829346
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,4,128,1,float16,fp8,511,0.018059200048446654
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,4,128,1,float16,float16,4095,0.07604640126228332
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,4,128,1,float16,float16,1023,0.016187199950218202
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,4,128,1,float16,fp8,1023,0.017027199268341064
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,4,128,1,float16,float16,2047,0.018140800297260284
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,4,128,1,float16,fp8,4095,0.02136639952659607
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,4,128,1,float16,float16,8191,0.031083199381828307
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,4,128,1,float16,fp8,8191,0.02740640044212341
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,8,128,1,float16,float16,1,0.011984000355005265
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,1,128,1,float16,float16,15,0.011659199744462967
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,8,128,1,float16,float16,3,0.011638399958610535
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,2,128,1,float16,fp8,2047,0.017579199373722078
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,8,128,1,float16,fp8,3,0.012464000284671784
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,8,128,1,float16,float16,7,0.011937599629163742
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,8,128,1,float16,fp8,7,0.012467200309038163
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,8,128,1,float16,float16,15,0.011956799775362015
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,8,128,1,float16,fp8,15,0.012492799758911132
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,8,128,1,float16,float16,31,0.011777599900960922
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,8,128,1,float16,fp8,31,0.012488000094890594
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,8,128,1,float16,float16,63,0.011952000111341477
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,8,128,1,float16,float16,127,0.011841599643230439
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,8,128,1,float16,fp8,127,0.012680000066757202
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,8,128,1,float16,float16,255,0.013609600067138673
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,8,128,1,float16,fp8,255,0.014646400511264802
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,8,128,1,float16,float16,511,0.016808000206947327
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,8,128,1,float16,fp8,511,0.01810719966888428
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,8,128,1,float16,float16,1023,0.01767839938402176
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,8,128,1,float16,fp8,1023,0.017627200484275816
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,8,128,1,float16,float16,2047,0.02001280039548874
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,8,128,1,float16,fp8,2047,0.021617600321769716
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,8,128,1,float16,float16,4095,0.03020800054073334
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,8,128,1,float16,fp8,4095,0.027447998523712158
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,8,128,1,float16,float16,8191,0.04495840072631836
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,8,128,1,float16,fp8,8191,0.04188799858093262
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,1,128,1,float16,float16,1,0.01523520052433014
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,1,128,1,float16,fp8,1,0.015644800662994385
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,1,128,1,float16,float16,3,0.014420799911022186
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,1,128,1,float16,fp8,3,0.016070400178432465
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,1,128,1,float16,float16,7,0.014407999813556671
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,1,128,1,float16,fp8,7,0.016011199355125426
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,1,128,1,float16,float16,15,0.014608000218868256
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,1,128,1,float16,float16,31,0.014679999649524688
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,1,128,1,float16,fp8,31,0.015561600029468537
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,1,128,1,float16,float16,63,0.014604799449443817
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,1,128,1,float16,fp8,63,0.015297600626945495
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,1,128,1,float16,float16,127,0.01464959979057312
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,4,128,1,float16,fp8,2047,0.01751679927110672
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,1,128,1,float16,fp8,127,0.015379199385643005
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,1,128,1,float16,float16,255,0.01674239933490753
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,1,128,1,float16,float16,511,0.02027360051870346
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,8,128,1,float16,fp8,1,0.012774400413036346
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,1,128,1,float16,fp8,511,0.02136320024728775
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,2,128,1,float16,float16,8191,0.022092799842357635
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,1,128,1,float16,fp8,1023,0.029790401458740234
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,8,128,1,float16,fp8,63,0.012675200402736665
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,1,128,1,float16,float16,2047,0.049000000953674315
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,1,128,1,float16,fp8,2047,0.04267520010471344
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,1,128,1,float16,float16,4095,0.07886880040168762
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,2,128,1,float16,float16,1,0.01873439997434616
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,2,128,1,float16,fp8,1,0.020110400021076204
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,2,128,1,float16,float16,3,0.018481600284576415
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,2,128,1,float16,fp8,3,0.020177599787712098
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,2,128,1,float16,float16,7,0.01880960017442703
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,2,128,1,float16,fp8,7,0.02044160068035126
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,2,128,1,float16,float16,15,0.01889919936656952
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,1,128,1,float16,fp8,15,0.01605599969625473
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,2,128,1,float16,float16,31,0.018718400597572328
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,2,128,1,float16,fp8,31,0.02025119960308075
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,2,128,1,float16,float16,63,0.018862399458885192
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,2,128,1,float16,fp8,63,0.020388799905776977
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,2,128,1,float16,float16,127,0.018838399648666383
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,32,4,128,1,float16,float16,4095,0.021121600270271303
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,2,128,1,float16,fp8,127,0.02048799991607666
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,1,128,1,float16,fp8,255,0.017190399765968322
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,2,128,1,float16,float16,255,0.02223680019378662
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,2,128,1,float16,float16,511,0.03368479907512665
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,2,128,1,float16,fp8,511,0.03172160089015961
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,2,128,1,float16,float16,1023,0.05098239779472351
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,2,128,1,float16,fp8,1023,0.04747360050678253
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,2,128,1,float16,float16,2047,0.08180639743804932
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,1,128,1,float16,float16,1023,0.032892799377441405
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,2,128,1,float16,float16,4095,0.14203039407730103
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,4,128,1,float16,float16,1,0.025835201144218445
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,2,128,1,float16,fp8,4095,0.12016799449920654
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,4,128,1,float16,fp8,1,0.02858720123767853
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,4,128,1,float16,float16,3,0.026286399364471434
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,32,8,128,1,float16,fp8,127,0.018024000525474548
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,4,128,1,float16,float16,7,0.0259552001953125
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,4,128,1,float16,fp8,7,0.029028800129890443
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,1,128,1,float16,fp8,4095,0.06783999800682068
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,4,128,1,float16,fp8,15,0.029041600227355958
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,4,128,1,float16,float16,31,0.026443201303482055
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,4,128,1,float16,float16,63,0.02646239995956421
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,4,128,1,float16,fp8,63,0.029254400730133058
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,4,128,1,float16,float16,127,0.02624959945678711
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,4,128,1,float16,fp8,127,0.029216000437736513
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,2,128,1,float16,fp8,15,0.02028000056743622
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,4,128,1,float16,float16,255,0.03967840075492859
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,4,128,1,float16,fp8,255,0.03760640025138855
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,4,128,1,float16,float16,511,0.05333120226860046
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,4,128,1,float16,fp8,511,0.05389279723167419
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,4,128,1,float16,float16,1023,0.08618400096893311
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,4,128,1,float16,fp8,1023,0.07913600206375122
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,4,128,1,float16,float16,2047,0.144924795627594
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,2,128,1,float16,fp8,255,0.02448800057172775
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,4,128,1,float16,fp8,2047,0.1255344033241272
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,8,128,1,float16,float16,1,0.04082559943199158
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,4,128,1,float16,float16,4095,0.26199679374694823
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,4,128,1,float16,fp8,4095,0.2251823902130127
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,2,128,1,float16,fp8,2047,0.07160320281982421
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,8,128,1,float16,float16,3,0.04101920127868652
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,8,128,1,float16,float16,7,0.0407696008682251
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,8,128,1,float16,fp8,7,0.04686239957809448
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,8,128,1,float16,float16,15,0.04092159867286682
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,8,128,1,float16,fp8,15,0.046649599075317384
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,8,128,1,float16,fp8,31,0.046768000721931456
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,8,128,1,float16,float16,63,0.041150400042533876
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,8,128,1,float16,fp8,63,0.046828800439834596
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,4,128,1,float16,fp8,3,0.02857759892940521
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,8,128,1,float16,float16,255,0.06238399744033814
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,4,128,1,float16,float16,15,0.026159998774528504
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,8,128,1,float16,fp8,255,0.06487039923667907
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,4,128,1,float16,fp8,31,0.02903519868850708
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,8,128,1,float16,float16,511,0.08950560092926026
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,8,128,1,float16,fp8,511,0.09420639872550965
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,8,128,1,float16,float16,1023,0.15389440059661866
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,8,128,1,float16,float16,2047,0.27493441104888916
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,8,128,1,float16,fp8,2047,0.2362015962600708
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,8,128,1,float16,float16,4095,0.5216944217681885
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,8,128,1,float16,fp8,4095,0.4340240001678467
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,1,128,1,float16,float16,1,0.020868800580501556
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,1,128,1,float16,fp8,1,0.022867199778556824
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,8,128,1,float16,fp8,1,0.04662719964981079
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,1,128,1,float16,float16,7,0.0213359996676445
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,1,128,1,float16,fp8,7,0.02319999933242798
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,8,128,1,float16,fp8,3,0.04697119891643524
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,1,128,1,float16,float16,15,0.02143840044736862
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,1,128,1,float16,fp8,15,0.023259200155735016
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,8,128,1,float16,float16,31,0.04089600145816803
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,1,128,1,float16,fp8,31,0.023160000145435334
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,1,128,1,float16,float16,63,0.0210207998752594
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,1,128,1,float16,float16,127,0.02109919935464859
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,8,128,1,float16,fp8,127,0.04742879867553711
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,1,128,1,float16,fp8,127,0.022601599991321563
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,1,128,1,float16,float16,255,0.02466080039739609
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,1,128,1,float16,fp8,255,0.026612800359725953
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,1,128,1,float16,float16,511,0.036750400066375734
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,1,128,1,float16,fp8,511,0.03424000144004822
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,1,128,1,float16,float16,1023,0.055961602926254274
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,1,128,1,float16,fp8,1023,0.05076640248298645
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,2,128,1,float16,float16,1,0.028366398811340333
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,2,128,1,float16,fp8,1,0.03118560016155243
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,2,128,1,float16,float16,3,0.02821600139141083
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,2,128,1,float16,fp8,3,0.031193599104881287
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,2,128,1,float16,float16,7,0.028204798698425293
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,2,128,1,float16,fp8,7,0.03129599988460541
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,2,128,1,float16,float16,15,0.028321599960327147
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,2,128,1,float16,fp8,15,0.03131200075149536
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,2,128,1,float16,float16,31,0.028251200914382935
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,2,128,1,float16,fp8,31,0.03127999901771546
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,2,128,1,float16,float16,63,0.02824159860610962
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,2,128,1,float16,fp8,63,0.03130559921264649
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,2,128,1,float16,float16,127,0.02863200008869171
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,2,128,1,float16,fp8,127,0.031323200464248656
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,2,128,1,float16,float16,255,0.04239520132541656
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,2,128,1,float16,fp8,255,0.039657598733901976
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,2,128,1,float16,float16,511,0.056739199161529544
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,2,128,1,float16,fp8,511,0.05641440153121948
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,1,128,1,float16,fp8,63,0.0226160004734993
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,2,128,1,float16,float16,1023,0.09116320013999939
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,8,128,1,float16,float16,127,0.04701760113239288
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,2,128,1,float16,fp8,1023,0.08310719728469848
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,4,128,1,float16,float16,1,0.04307360053062439
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,4,128,1,float16,fp8,1,0.049423998594284056
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,4,128,1,float16,fp8,3,0.04892480075359344
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,4,128,1,float16,float16,7,0.04325439929962158
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,4,128,1,float16,fp8,7,0.04911360144615173
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,4,128,1,float16,float16,15,0.043300798535346983
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,4,128,1,float16,fp8,15,0.04921280145645142
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,4,128,1,float16,float16,31,0.043300798535346983
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,4,128,1,float16,fp8,31,0.04927999973297119
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,32,8,128,1,float16,fp8,1023,0.1434831976890564
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,4,128,1,float16,float16,63,0.04397599995136261
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,4,128,1,float16,float16,127,0.04916160106658936
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,4,128,1,float16,fp8,127,0.050783997774124144
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,4,128,1,float16,float16,255,0.06557279825210571
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,4,128,1,float16,fp8,255,0.06732159852981567
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,4,128,1,float16,float16,511,0.09388480186462403
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,4,128,1,float16,fp8,511,0.09672639966011047
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,4,128,1,float16,float16,1023,0.15791200399398803
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,4,128,1,float16,fp8,1023,0.14658880233764648
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,8,128,1,float16,float16,1,0.07135519981384278
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,8,128,1,float16,fp8,1,0.0846560001373291
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,8,128,1,float16,float16,3,0.07120000123977661
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,8,128,1,float16,fp8,3,0.08486080169677734
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,8,128,1,float16,float16,7,0.0711135983467102
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,8,128,1,float16,fp8,7,0.08483840227127075
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,1,128,1,float16,float16,3,0.020751999318599702
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,8,128,1,float16,float16,15,0.07131360173225403
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,8,128,1,float16,fp8,15,0.08505120277404785
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,8,128,1,float16,float16,31,0.07125920057296753
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,8,128,1,float16,fp8,31,0.08447999954223633
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,8,128,1,float16,float16,63,0.07786399722099305
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,8,128,1,float16,float16,127,0.08113279938697815
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,1,128,1,float16,float16,31,0.021158400177955627
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,8,128,1,float16,fp8,127,0.09314879775047302
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,8,128,1,float16,float16,255,0.10871679782867431
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,8,128,1,float16,fp8,255,0.11900800466537476
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,8,128,1,float16,float16,511,0.16571680307388306
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,8,128,1,float16,fp8,511,0.17675039768218995
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,4,128,1,float16,float16,3,0.04309279918670654
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,1,128,1,float16,float16,1,0.03434880077838898
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,8,128,1,float16,float16,1023,0.2915679931640625
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,1,128,1,float16,fp8,1,0.037625598907470706
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,1,128,1,float16,float16,3,0.03442400097846985
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,8,128,1,float16,fp8,1023,0.2751136064529419
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,1,128,1,float16,float16,7,0.03448159992694855
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,1,128,1,float16,fp8,7,0.03754239976406097
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,1,128,1,float16,fp8,3,0.03750239908695221
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,1,128,1,float16,fp8,15,0.03752320110797882
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,1,128,1,float16,float16,31,0.03435679972171783
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,1,128,1,float16,float16,63,0.03431519865989685
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,4,128,1,float16,fp8,63,0.049200001358985904
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,1,128,1,float16,float16,127,0.03552800118923187
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,1,128,1,float16,fp8,127,0.037596800923347475
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,1,128,1,float16,float16,255,0.04812479913234711
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,1,128,1,float16,fp8,255,0.04589920043945313
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,1,128,1,float16,float16,511,0.06308959722518921
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,1,128,1,float16,fp8,511,0.06276800036430359
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,2,128,1,float16,float16,1,0.048814401030540466
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,2,128,1,float16,float16,3,0.0486735999584198
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,2,128,1,float16,fp8,3,0.054612797498703
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,2,128,1,float16,float16,7,0.04865919947624207
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,2,128,1,float16,fp8,7,0.054708802700042726
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,2,128,1,float16,float16,15,0.04880799949169159
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,2,128,1,float16,fp8,15,0.054657602310180665
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,2,128,1,float16,float16,31,0.04890879988670349
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,2,128,1,float16,fp8,31,0.05460799932479858
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,2,128,1,float16,float16,63,0.04967679977416992
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,2,128,1,float16,fp8,63,0.05482720136642456
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,2,128,1,float16,float16,127,0.05535039901733398
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,1,128,1,float16,fp8,3,0.022275200486183165
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,2,128,1,float16,float16,255,0.07227839827537537
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,2,128,1,float16,fp8,255,0.07154560089111328
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,32,8,128,1,float16,fp8,63,0.08493120074272156
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,2,128,1,float16,float16,511,0.10028640031814576
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,2,128,1,float16,fp8,511,0.10293920040130615
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,4,128,1,float16,float16,1,0.07688000202178955
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,4,128,1,float16,fp8,1,0.08963999748229981
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,4,128,1,float16,fp8,3,0.08964160084724426
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,4,128,1,float16,float16,7,0.0767952024936676
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,4,128,1,float16,fp8,7,0.08972799777984619
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,4,128,1,float16,float16,15,0.07679359912872315
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,4,128,1,float16,fp8,15,0.08977280259132385
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,4,128,1,float16,float16,31,0.07717120051383972
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,1,128,1,float16,float16,15,0.03442240059375763
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,1,128,1,float16,fp8,31,0.03767519891262054
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,4,128,1,float16,fp8,31,0.08990560173988342
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,4,128,1,float16,float16,63,0.08239679932594299
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,4,128,1,float16,fp8,63,0.09191039800643921
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,4,128,1,float16,float16,127,0.08602880239486695
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,4,128,1,float16,float16,255,0.11603519916534424
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,4,128,1,float16,fp8,255,0.12324479818344117
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,2,128,1,float16,fp8,1,0.054527997970581055
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,4,128,1,float16,float16,511,0.17060960531234742
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,4,128,1,float16,fp8,511,0.18294399976730347
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,8,128,1,float16,fp8,1,0.15763839483261108
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,8,128,1,float16,float16,3,0.13074879646301268
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,8,128,1,float16,fp8,3,0.1573791980743408
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,8,128,1,float16,float16,7,0.1309615969657898
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,8,128,1,float16,fp8,7,0.15818239450454713
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,2,128,1,float16,fp8,127,0.05605120062828064
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,8,128,1,float16,fp8,15,0.15792160034179686
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,8,128,1,float16,float16,31,0.13799359798431396
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,8,128,1,float16,fp8,31,0.15782560110092164
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,8,128,1,float16,float16,63,0.140665602684021
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,4,128,1,float16,float16,3,0.07700799703598023
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,8,128,1,float16,fp8,63,0.16787840127944947
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,8,128,1,float16,float16,127,0.14575519561767578
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,8,128,1,float16,fp8,127,0.1729375958442688
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,8,128,1,float16,float16,255,0.20236001014709473
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,8,128,1,float16,fp8,255,0.22413599491119385
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,1,128,1,float16,float16,1,0.011772800236940384
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,8,128,1,float16,float16,511,0.30980958938598635
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,1,128,1,float16,fp8,1,0.01252799928188324
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,1,128,1,float16,fp8,3,0.012675200402736665
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,1,128,1,float16,fp8,63,0.037441599369049075
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,1,128,1,float16,float16,7,0.011872000247240066
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,1,128,1,float16,float16,15,0.011876799911260606
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,4,128,1,float16,fp8,127,0.09606239795684815
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,1,128,1,float16,fp8,15,0.012596799433231354
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,1,128,1,float16,fp8,31,0.012699200212955475
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,1,128,1,float16,float16,63,0.01180960014462471
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,1,128,1,float16,fp8,63,0.012411200255155564
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,1,128,1,float16,float16,127,0.01175839975476265
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,8,128,1,float16,float16,1,0.13072160482406617
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,1,128,1,float16,float16,255,0.013566400110721587
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,1,128,1,float16,fp8,255,0.014606399834156037
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,1,128,1,float16,float16,511,0.016598400473594666
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,1,128,1,float16,fp8,511,0.01820639967918396
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,1,128,1,float16,float16,1023,0.016087999939918517
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,1,128,1,float16,fp8,1023,0.017124800384044646
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,1,128,1,float16,float16,2047,0.016624000668525696
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,1,128,1,float16,fp8,2047,0.017606399953365326
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,1,128,1,float16,float16,4095,0.018532800674438476
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,1,128,1,float16,fp8,4095,0.018268799781799315
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,1,128,1,float16,float16,8191,0.022019200026988983
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,1,128,1,float16,fp8,8191,0.02188960015773773
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,2,128,1,float16,float16,1,0.011875200271606445
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,2,128,1,float16,fp8,1,0.012347199767827988
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,2,128,1,float16,float16,3,0.01178399994969368
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,2,128,1,float16,fp8,3,0.012399999797344208
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,2,128,1,float16,fp8,7,0.012468799948692322
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,2,128,1,float16,float16,15,0.011823999881744384
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,2,128,1,float16,fp8,15,0.012630400061607362
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,2,128,1,float16,float16,31,0.011950399726629257
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,2,128,1,float16,fp8,31,0.012577599287033081
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,2,128,1,float16,float16,63,0.011908800154924393
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,2,128,1,float16,fp8,63,0.012587200105190276
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,2,128,1,float16,float16,127,0.011711999773979187
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,8,128,1,float16,fp8,511,0.342576003074646
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,1,128,1,float16,float16,3,0.011737599968910217
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,2,128,1,float16,fp8,255,0.014720000326633453
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,2,128,1,float16,float16,511,0.016652800142765045
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,1,128,1,float16,float16,31,0.011832000315189361
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,2,128,1,float16,float16,1023,0.01637759953737259
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,2,128,1,float16,fp8,1023,0.01735360026359558
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,2,128,1,float16,float16,2047,0.018313600122928618
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,1,128,1,float16,fp8,127,0.01252799928188324
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,2,128,1,float16,fp8,2047,0.01775040030479431
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,2,128,1,float16,float16,4095,0.02154400050640106
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,2,128,1,float16,fp8,4095,0.021388800442218782
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,2,128,1,float16,float16,8191,0.031462401151657104
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,2,128,1,float16,fp8,8191,0.027460798621177673
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,4,128,1,float16,float16,1,0.012151999771595002
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,4,128,1,float16,fp8,1,0.012787200510501862
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,4,128,1,float16,float16,3,0.012169600278139115
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,4,128,1,float16,fp8,3,0.012929600477218629
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,32,8,128,1,float16,float16,15,0.13106880187988282
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,4,128,1,float16,fp8,7,0.012948800623416901
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,4,128,1,float16,float16,15,0.01204800009727478
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,4,128,1,float16,float16,31,0.012062399834394454
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,4,128,1,float16,fp8,31,0.012915199995040894
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,4,128,1,float16,float16,63,0.012220799922943115
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,4,128,1,float16,fp8,63,0.012948800623416901
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,4,128,1,float16,float16,127,0.012223999947309494
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,2,128,1,float16,float16,7,0.01167680025100708
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,4,128,1,float16,float16,255,0.013777600228786468
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,4,128,1,float16,fp8,255,0.014892800152301789
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,4,128,1,float16,float16,511,0.016774399578571318
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,2,128,1,float16,fp8,127,0.012383999675512314
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,4,128,1,float16,fp8,511,0.018343999981880188
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,2,128,1,float16,float16,255,0.01388320028781891
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,4,128,1,float16,float16,1023,0.017697599530220032
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,4,128,1,float16,fp8,1023,0.017679999768733978
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,4,128,1,float16,float16,2047,0.01982239931821823
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,4,128,1,float16,fp8,2047,0.021320000290870667
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,4,128,1,float16,float16,4095,0.03022879958152771
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,4,128,1,float16,fp8,4095,0.027217599749565124
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,4,128,1,float16,float16,8191,0.04541119933128357
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,4,128,1,float16,fp8,8191,0.041519999504089355
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,8,128,1,float16,float16,1,0.012097600102424621
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,8,128,1,float16,fp8,1,0.012923200428485871
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,8,128,1,float16,float16,3,0.012067200243473053
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,8,128,1,float16,fp8,3,0.012919999659061432
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,8,128,1,float16,float16,7,0.011977600306272507
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,8,128,1,float16,fp8,7,0.012918399274349212
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,8,128,1,float16,float16,15,0.011886399984359742
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,8,128,1,float16,fp8,15,0.012915199995040894
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,8,128,1,float16,float16,31,0.012031999975442886
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,8,128,1,float16,fp8,31,0.012929600477218629
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,8,128,1,float16,float16,63,0.012004800140857697
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,8,128,1,float16,fp8,63,0.013012799620628356
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,8,128,1,float16,float16,127,0.012100800126791
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,8,128,1,float16,fp8,127,0.012919999659061432
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,8,128,1,float16,float16,255,0.014033600687980652
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,4,128,1,float16,fp8,15,0.012943999469280243
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,8,128,1,float16,float16,511,0.017294399440288544
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,8,128,1,float16,fp8,511,0.018428799510002137
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,8,128,1,float16,fp8,1023,0.021001599729061127
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,8,128,1,float16,float16,2047,0.030374398827552794
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,8,128,1,float16,fp8,2047,0.02741599977016449
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,4,128,1,float16,fp8,127,0.012948800623416901
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,8,128,1,float16,float16,4095,0.04478879868984222
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,8,128,1,float16,fp8,4095,0.041319999098777774
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,8,128,1,float16,float16,8191,0.07604159712791443
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,8,128,1,float16,fp8,8191,0.06673120260238648
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,32,1,128,1,float16,float16,1,0.05539360046386719
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,32,1,128,1,float16,fp8,1,0.06130239963531494
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,32,1,128,1,float16,float16,3,0.055276799201965335
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,32,1,128,1,float16,fp8,3,0.06114720106124878
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,32,1,128,1,float16,float16,7,0.05511999726295471
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,32,1,128,1,float16,float16,15,0.05527520179748535
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,1,128,1,float16,fp8,7,0.01252640038728714
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,32,1,128,1,float16,float16,31,0.05538719892501831
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,2,128,1,float16,fp8,511,0.018139199912548067
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,32,1,128,1,float16,fp8,31,0.061684799194335935
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,32,1,128,1,float16,float16,63,0.05891519784927368
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,32,1,128,1,float16,fp8,63,0.061883199214935306
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,32,1,128,1,float16,float16,127,0.06297119855880737
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,32,1,128,1,float16,fp8,127,0.06500160098075866
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,32,1,128,1,float16,float16,255,0.07903519868850709
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,32,1,128,1,float16,fp8,255,0.07911520004272461
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,32,2,128,1,float16,float16,1,0.08333600163459778
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,32,2,128,1,float16,fp8,1,0.09534720182418824
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,32,2,128,1,float16,float16,3,0.08351680040359497
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,4,128,1,float16,float16,7,0.012078399956226348
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,32,2,128,1,float16,fp8,3,0.09514080286026001
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,32,2,128,1,float16,float16,7,0.0833407998085022
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,8,128,1,float16,fp8,255,0.014758400619029999
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,32,2,128,1,float16,fp8,7,0.09529439806938171
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,32,2,128,1,float16,float16,15,0.08315359950065612
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,32,8,128,1,float16,float16,1023,0.019760000705718993
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,32,2,128,1,float16,float16,31,0.08328160047531127
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,32,2,128,1,float16,float16,63,0.08863360285758973
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,32,2,128,1,float16,fp8,63,0.10041760206222534
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,32,2,128,1,float16,float16,127,0.0924351990222931
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,32,2,128,1,float16,fp8,127,0.10304479598999024
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,32,2,128,1,float16,float16,255,0.12342079877853393
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,32,2,128,1,float16,fp8,255,0.12912000417709352
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,32,4,128,1,float16,float16,1,0.13893120288848876
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,32,4,128,1,float16,fp8,1,0.16267999410629272
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,32,4,128,1,float16,float16,3,0.13884320259094238
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,32,4,128,1,float16,fp8,3,0.16289440393447877
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,32,4,128,1,float16,float16,7,0.13878079652786254
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,32,4,128,1,float16,float16,15,0.13864799737930297
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,32,4,128,1,float16,fp8,7,0.16310080289840698
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,32,4,128,1,float16,fp8,15,0.1633407950401306
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,32,4,128,1,float16,float16,31,0.1442911982536316
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,32,4,128,1,float16,fp8,31,0.16483999490737916
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,32,1,128,1,float16,fp8,7,0.061289602518081666
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,32,4,128,1,float16,fp8,63,0.17323999404907225
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,32,4,128,1,float16,float16,127,0.1518064022064209
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,32,4,128,1,float16,fp8,127,0.17567520141601561
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,32,1,128,1,float16,fp8,15,0.06145439743995666
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,32,4,128,1,float16,float16,255,0.20644640922546387
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,32,4,128,1,float16,fp8,255,0.22800159454345703
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,32,8,128,1,float16,fp8,1,0.29529919624328616
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,32,8,128,1,float16,fp8,3,0.2944799900054932
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,32,8,128,1,float16,float16,7,0.24473440647125244
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,32,8,128,1,float16,fp8,7,0.2953279972076416
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,32,8,128,1,float16,float16,15,0.2530639886856079
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,32,8,128,1,float16,fp8,15,0.2944000005722046
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,32,8,128,1,float16,float16,31,0.26121280193328855
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,32,8,128,1,float16,fp8,31,0.3142927885055542
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,32,8,128,1,float16,float16,63,0.26352319717407224
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,32,8,128,1,float16,fp8,63,0.3175983905792236
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,32,8,128,1,float16,float16,127,0.2707616090774536
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,32,2,128,1,float16,fp8,15,0.09593120217323303
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,32,8,128,1,float16,fp8,127,0.3195807933807373
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,32,2,128,1,float16,fp8,31,0.09565920233726502
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,32,1,128,1,float16,float16,1,0.0956816017627716
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,32,1,128,1,float16,fp8,1,0.10761760473251343
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,32,8,128,1,float16,float16,255,0.3816303968429565
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,32,1,128,1,float16,float16,3,0.09627040028572083
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,32,1,128,1,float16,fp8,3,0.10761439800262451
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,32,8,128,1,float16,fp8,255,0.4256735801696777
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,32,1,128,1,float16,fp8,7,0.10776959657669068
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,32,1,128,1,float16,float16,15,0.09637439846992493
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,32,1,128,1,float16,fp8,15,0.10905439853668213
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,32,1,128,1,float16,fp8,31,0.10956000089645386
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,32,1,128,1,float16,float16,63,0.10452159643173217
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,32,1,128,1,float16,fp8,63,0.11425280570983887
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,32,1,128,1,float16,float16,127,0.10901119709014892
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,32,4,128,1,float16,float16,63,0.1462399959564209
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,32,1,128,1,float16,fp8,127,0.11709599494934082
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,32,2,128,1,float16,float16,1,0.1497007966041565
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,32,2,128,1,float16,fp8,1,0.17320319414138793
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,32,2,128,1,float16,float16,3,0.14860960245132446
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,32,2,128,1,float16,fp8,3,0.1735103964805603
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,32,2,128,1,float16,float16,7,0.14970719814300537
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,32,8,128,1,float16,float16,1,0.24426560401916503
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,32,2,128,1,float16,fp8,7,0.17395520210266113
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,32,8,128,1,float16,float16,3,0.24434878826141357
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,32,2,128,1,float16,float16,15,0.15367519855499268
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,32,2,128,1,float16,fp8,15,0.1741328001022339
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,32,2,128,1,float16,float16,31,0.15575519800186158
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,32,2,128,1,float16,fp8,31,0.1802240014076233
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,32,2,128,1,float16,float16,63,0.15985759496688842
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,32,2,128,1,float16,fp8,63,0.18597439527511597
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,32,2,128,1,float16,fp8,127,0.18712320327758789
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,32,4,128,1,float16,float16,1,0.2611151933670044
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,32,4,128,1,float16,float16,3,0.26226880550384524
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,32,4,128,1,float16,fp8,3,0.30834240913391114
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,32,4,128,1,float16,float16,7,0.2645967960357666
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,32,4,128,1,float16,fp8,7,0.30951519012451173
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,32,4,128,1,float16,float16,15,0.2675951957702637
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,32,4,128,1,float16,fp8,15,0.31382720470428466
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,32,4,128,1,float16,float16,31,0.2714560031890869
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,32,4,128,1,float16,fp8,31,0.32403199672698973
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,32,4,128,1,float16,float16,63,0.27453439235687255
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,32,4,128,1,float16,fp8,63,0.3290447950363159
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,32,1,128,1,float16,float16,7,0.09611200094223023
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,32,4,128,1,float16,float16,127,0.2806112051010132
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,32,1,128,1,float16,float16,31,0.10045759677886963
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,32,4,128,1,float16,fp8,127,0.3312319993972778
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,32,8,128,1,float16,float16,1,0.49185919761657715
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,32,8,128,1,float16,float16,3,0.4922192096710205
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,32,8,128,1,float16,fp8,3,0.5740464210510254
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,32,8,128,1,float16,float16,7,0.4938159942626953
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,32,8,128,1,float16,fp8,7,0.5780240058898926
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,32,8,128,1,float16,float16,15,0.49585599899291993
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,32,8,128,1,float16,float16,31,0.4987504005432129
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,32,8,128,1,float16,fp8,15,0.605401611328125
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,32,2,128,1,float16,float16,127,0.16527199745178223
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,32,8,128,1,float16,float16,63,0.5045008182525634
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,1,128,1,float16,float16,1,0.01202400028705597
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,1,128,1,float16,fp8,1,0.012371200323104858
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,32,4,128,1,float16,fp8,1,0.3080319881439209
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,32,8,128,1,float16,fp8,63,0.6089600086212158
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,1,128,1,float16,fp8,3,0.01252799928188324
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,1,128,1,float16,fp8,7,0.012510399520397186
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,1,128,1,float16,float16,15,0.01173119992017746
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,1,128,1,float16,fp8,15,0.012547199428081513
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,1,128,1,float16,float16,31,0.011671999841928482
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,32,8,128,1,float16,fp8,127,0.6170303821563721
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,1,128,1,float16,fp8,31,0.012544000148773193
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,1,128,1,float16,float16,63,0.011827199906110763
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,1,128,1,float16,float16,127,0.011932799965143204
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,1,128,1,float16,fp8,63,0.012577599287033081
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,1,128,1,float16,float16,255,0.013654400408267976
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,1,128,1,float16,fp8,255,0.014553600549697876
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,1,128,1,float16,float16,511,0.01685439944267273
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,1,128,1,float16,fp8,511,0.018303999304771425
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,1,128,1,float16,fp8,1023,0.01736160069704056
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,1,128,1,float16,float16,2047,0.018199999630451203
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,1,128,1,float16,fp8,2047,0.017875200510025023
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,1,128,1,float16,float16,4095,0.021566399931907655
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,1,128,1,float16,fp8,4095,0.021712000668048858
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,1,128,1,float16,float16,8191,0.03191039860248566
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,32,8,128,1,float16,fp8,1,0.569532823562622
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,1,128,1,float16,fp8,8191,0.02747200131416321
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,2,128,1,float16,float16,1,0.01188800036907196
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,2,128,1,float16,float16,3,0.011952000111341477
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,2,128,1,float16,fp8,3,0.012811200320720672
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,2,128,1,float16,float16,7,0.012060800194740295
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,2,128,1,float16,fp8,7,0.012681600451469422
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,2,128,1,float16,float16,15,0.01183359995484352
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,2,128,1,float16,fp8,15,0.012703999876976013
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,2,128,1,float16,float16,31,0.012055999785661697
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,2,128,1,float16,fp8,31,0.012806400656700134
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,32,8,128,1,float16,fp8,31,0.6096127986907959
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,2,128,1,float16,fp8,63,0.012708799540996551
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,2,128,1,float16,float16,127,0.01188800036907196
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,2,128,1,float16,fp8,127,0.012731200456619263
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,1,128,1,float16,float16,3,0.011747200042009354
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,1,128,1,float16,float16,7,0.012089599668979645
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,2,128,1,float16,fp8,255,0.014803199470043183
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,2,128,1,float16,float16,511,0.016675199568271636
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,2,128,1,float16,float16,1023,0.01798879951238632
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,1,128,1,float16,fp8,127,0.012625600397586822
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,2,128,1,float16,fp8,1023,0.017591999471187593
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,2,128,1,float16,float16,2047,0.019764800369739533
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,2,128,1,float16,fp8,2047,0.021166400611400606
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,2,128,1,float16,float16,4095,0.030953601002693176
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,2,128,1,float16,fp8,4095,0.027054399251937866
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,2,128,1,float16,float16,8191,0.04614399969577789
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,2,128,1,float16,fp8,8191,0.04148640036582947
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,4,128,1,float16,float16,1,0.012111999839544297
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,4,128,1,float16,fp8,1,0.012839999794960023
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,2,128,1,float16,fp8,1,0.012627199292182922
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,4,128,1,float16,float16,3,0.01196800023317337
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,4,128,1,float16,fp8,3,0.012740799784660339
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,4,128,1,float16,fp8,7,0.01279360055923462
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,4,128,1,float16,float16,15,0.012080000340938568
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,4,128,1,float16,fp8,15,0.012963199615478515
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,4,128,1,float16,float16,31,0.012145599722862244
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,4,128,1,float16,fp8,31,0.012889599800109864
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,4,128,1,float16,float16,63,0.012219200283288956
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,2,128,1,float16,float16,63,0.012030400335788727
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,4,128,1,float16,float16,127,0.01226079985499382
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,4,128,1,float16,fp8,127,0.012910400331020356
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,4,128,1,float16,float16,255,0.013809600472450256
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,4,128,1,float16,fp8,255,0.014820800721645355
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,4,128,1,float16,float16,511,0.01695840060710907
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,2,128,1,float16,fp8,511,0.01847359985113144
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,4,128,1,float16,fp8,511,0.01844799965620041
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,4,128,1,float16,float16,1023,0.01964319944381714
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,4,128,1,float16,fp8,1023,0.020942400395870208
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,4,128,1,float16,fp8,2047,0.027115198969841003
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,4,128,1,float16,float16,4095,0.04601120054721832
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,1,128,1,float16,float16,1023,0.016700799763202667
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,4,128,1,float16,fp8,4095,0.04154239892959595
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,4,128,1,float16,float16,8191,0.07647519707679748
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,4,128,1,float16,fp8,8191,0.06632959842681885
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,8,128,1,float16,float16,1,0.012324800342321396
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,8,128,1,float16,fp8,1,0.012870399653911591
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,8,128,1,float16,float16,3,0.012121599912643433
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,8,128,1,float16,fp8,3,0.012838399410247803
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,8,128,1,float16,float16,7,0.012121599912643433
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,8,128,1,float16,fp8,7,0.012960000336170197
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,8,128,1,float16,float16,15,0.012120000272989272
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,8,128,1,float16,fp8,15,0.01297439932823181
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,8,128,1,float16,float16,31,0.012247999757528305
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,8,128,1,float16,fp8,31,0.01297599971294403
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,8,128,1,float16,float16,63,0.012251199781894683
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,8,128,1,float16,fp8,63,0.013105599582195282
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,8,128,1,float16,float16,127,0.012392000108957291
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,8,128,1,float16,float16,255,0.014140799641609192
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,8,128,1,float16,fp8,255,0.01515360027551651
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,8,128,1,float16,float16,511,0.01741439998149872
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,8,128,1,float16,fp8,511,0.018852800130844116
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,8,128,1,float16,float16,1023,0.028860801458358766
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,8,128,1,float16,fp8,1023,0.026183998584747313
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,8,128,1,float16,float16,2047,0.04419200122356415
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,8,128,1,float16,fp8,2047,0.04056000113487244
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,8,128,1,float16,float16,4095,0.07438079714775085
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,8,128,1,float16,fp8,4095,0.06580960154533386
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,8,128,1,float16,float16,8191,0.13382560014724731
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,8,128,1,float16,fp8,8191,0.11552159786224366
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,4,128,1,float16,fp8,63,0.012803199887275695
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,1,128,1,float16,float16,1,0.013675199449062347
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,1,128,1,float16,float16,3,0.013142399489879608
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,1,128,1,float16,fp8,3,0.014144000411033631
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,1,128,1,float16,fp8,7,0.014129599928855896
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,32,8,128,1,float16,float16,127,0.5208447933197021
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,1,128,1,float16,float16,15,0.01340479999780655
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,2,128,1,float16,float16,255,0.01377280056476593
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,1,128,1,float16,float16,31,0.013387200236320496
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,1,128,1,float16,fp8,31,0.014153599739074707
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,1,128,1,float16,float16,63,0.01326880007982254
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,1,128,1,float16,fp8,63,0.014115199446678162
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,1,128,1,float16,float16,127,0.013312000036239623
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,1,128,1,float16,fp8,127,0.014142400026321411
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,1,128,1,float16,float16,255,0.01510239988565445
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,1,128,1,float16,fp8,255,0.01623679995536804
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,4,128,1,float16,float16,2047,0.030137598514556885
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,1,128,1,float16,fp8,511,0.019841599464416503
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,1,128,1,float16,float16,1023,0.02096640020608902
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,1,128,1,float16,fp8,1023,0.02234559953212738
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,1,128,1,float16,float16,2047,0.032332798838615416
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,1,128,1,float16,fp8,2047,0.028439998626708984
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,1,128,1,float16,float16,4095,0.04719679951667786
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,1,128,1,float16,fp8,4095,0.042691200971603394
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,1,128,1,float16,float16,8191,0.07930560111999511
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,1,128,1,float16,float16,16383,0.13909920454025268
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,8,128,1,float16,fp8,127,0.013121600449085235
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,1,128,1,float16,fp8,16383,0.11822240352630616
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,2,128,1,float16,fp8,1,0.01417279988527298
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,32,4,128,1,float16,float16,7,0.01194560006260872
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,2,128,1,float16,float16,3,0.01313920021057129
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,2,128,1,float16,fp8,3,0.014254400134086609
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,2,128,1,float16,float16,7,0.013148799538612366
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,2,128,1,float16,fp8,7,0.014251199364662171
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,2,128,1,float16,float16,15,0.013331200182437896
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,2,128,1,float16,fp8,15,0.014097599685192109
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,2,128,1,float16,float16,31,0.013380800187587739
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,2,128,1,float16,fp8,31,0.014291200041770934
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,2,128,1,float16,float16,63,0.013476799428462981
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,2,128,1,float16,fp8,63,0.014302399754524232
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,2,128,1,float16,float16,127,0.013580800592899322
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,2,128,1,float16,fp8,127,0.014233599603176116
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,1,128,1,float16,fp8,1,0.014094400405883788
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,1,128,1,float16,float16,7,0.01311360001564026
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,2,128,1,float16,float16,511,0.018223999440670012
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,2,128,1,float16,fp8,511,0.01964160054922104
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,1,128,1,float16,fp8,15,0.014233599603176116
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,2,128,1,float16,fp8,1023,0.02766079902648926
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,2,128,1,float16,float16,2047,0.04565440118312836
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,1,128,1,float16,float16,511,0.018113599717617036
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,2,128,1,float16,fp8,2047,0.04117920100688934
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,2,128,1,float16,float16,4095,0.077019202709198
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,2,128,1,float16,fp8,4095,0.06659200191497802
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,2,128,1,float16,float16,8191,0.13685599565505982
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,2,128,1,float16,fp8,8191,0.11646239757537842
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,1,128,1,float16,fp8,8191,0.06778720021247864
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,4,128,1,float16,float16,1,0.017156800627708434
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,4,128,1,float16,fp8,1,0.018699200451374055
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,2,128,1,float16,float16,16383,0.2552175998687744
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,2,128,1,float16,fp8,16383,0.21377120018005372
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,2,128,1,float16,float16,1,0.01363680064678192
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,4,128,1,float16,fp8,3,0.01868959963321686
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,4,128,1,float16,float16,7,0.017470400035381316
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,4,128,1,float16,fp8,7,0.01892800033092499
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,4,128,1,float16,float16,15,0.017584000527858735
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,4,128,1,float16,float16,31,0.01743839979171753
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,4,128,1,float16,fp8,31,0.018939200043678283
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,4,128,1,float16,float16,63,0.01759520024061203
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,4,128,1,float16,float16,127,0.017590400576591492
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,4,128,1,float16,fp8,127,0.01897760033607483
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,4,128,1,float16,float16,255,0.021031999588012697
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,4,128,1,float16,fp8,255,0.022945599257946016
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,4,128,1,float16,float16,511,0.0314767986536026
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,2,128,1,float16,float16,255,0.015081599354743958
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,4,128,1,float16,fp8,511,0.030212798714637758
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,2,128,1,float16,fp8,255,0.01613440066576004
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,4,128,1,float16,fp8,1023,0.0460207998752594
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,4,128,1,float16,float16,2047,0.07901120185852051
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,4,128,1,float16,fp8,2047,0.06910719871520996
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,2,128,1,float16,float16,1023,0.030895999073982237
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,4,128,1,float16,float16,4095,0.13825759887695313
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,4,128,1,float16,fp8,4095,0.1185487985610962
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,4,128,1,float16,float16,8191,0.25548639297485354
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,4,128,1,float16,fp8,8191,0.21834878921508788
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,8,128,1,float16,float16,1,0.02499520033597946
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,8,128,1,float16,fp8,1,0.027585598826408386
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,4,128,1,float16,float16,16383,0.5003136157989502
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,8,128,1,float16,float16,3,0.024398399889469145
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,8,128,1,float16,float16,7,0.024435199797153473
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,8,128,1,float16,fp8,3,0.027507200837135315
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,8,128,1,float16,fp8,7,0.027454400062561037
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,8,128,1,float16,float16,15,0.024382400512695312
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,4,128,1,float16,float16,3,0.017284800112247468
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,8,128,1,float16,float16,31,0.024905599653720856
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,8,128,1,float16,fp8,31,0.027527999877929688
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,8,128,1,float16,float16,63,0.024916799366474153
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,8,128,1,float16,fp8,63,0.027480000257492067
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,8,128,1,float16,float16,127,0.025297600030899047
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,4,128,1,float16,fp8,15,0.019006399810314177
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,8,128,1,float16,float16,255,0.03698399960994721
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,4,128,1,float16,fp8,63,0.018799999356269838
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,8,128,1,float16,float16,511,0.050577598810195926
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,8,128,1,float16,fp8,511,0.05252640247344971
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,8,128,1,float16,float16,1023,0.08338239789009094
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,8,128,1,float16,fp8,1023,0.0780239999294281
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,8,128,1,float16,float16,2047,0.14217599630355834
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,8,128,1,float16,float16,4095,0.2611407995223999
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,8,128,1,float16,fp8,4095,0.22397758960723876
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,8,128,1,float16,float16,8191,0.5011680126190186
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,8,128,1,float16,fp8,8191,0.4198751926422119
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,4,128,1,float16,fp8,16383,0.41188478469848633
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,1,128,1,float16,float16,1,0.011300799995660782
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,1,128,1,float16,fp8,1,0.01207199990749359
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,1,128,1,float16,float16,3,0.011456000059843064
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,1,128,1,float16,fp8,3,0.01223680004477501
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,8,128,1,float16,fp8,127,0.027470400929450987
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,1,128,1,float16,fp8,7,0.012212800234556198
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,8,128,1,float16,fp8,255,0.03617919981479645
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,1,128,1,float16,float16,15,0.011512000113725662
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,1,128,1,float16,float16,31,0.01146719977259636
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,1,128,1,float16,fp8,31,0.012273599952459335
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,1,128,1,float16,float16,63,0.011529599875211715
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,1,128,1,float16,fp8,63,0.012246400117874146
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,1,128,1,float16,float16,127,0.011444800347089768
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,1,128,1,float16,fp8,127,0.012142399698495865
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,1,128,1,float16,float16,255,0.013033600151538849
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,1,128,1,float16,fp8,255,0.014108799397945404
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,4,128,1,float16,float16,1023,0.04812000095844269
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,1,128,1,float16,float16,511,0.016096000373363496
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,1,128,1,float16,float16,1023,0.01557600051164627
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,1,128,1,float16,fp8,1023,0.01674560010433197
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,1,128,1,float16,float16,2047,0.015775999426841734
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,1,128,1,float16,fp8,2047,0.016942399740219116
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,1,128,1,float16,float16,4095,0.01666239947080612
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,1,128,1,float16,fp8,4095,0.017292800545692443
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,1,128,1,float16,float16,8191,0.018086400628089905
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,1,128,1,float16,fp8,8191,0.018510399758815764
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,1,128,1,float16,float16,16383,0.020100800693035124
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,1,128,1,float16,fp8,16383,0.020468799769878386
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,2,128,1,float16,float16,1,0.011376000195741653
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,8,128,1,float16,fp8,15,0.027470400929450987
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,2,128,1,float16,fp8,1,0.01210559979081154
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,2,128,1,float16,float16,3,0.011449600011110306
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,8,128,1,float16,fp8,16383,0.8085344314575196
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,8,128,1,float16,float16,16383,0.9938688278198242
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,2,128,1,float16,float16,7,0.011348800361156463
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,2,128,1,float16,fp8,7,0.012268800288438797
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,2,128,1,float16,float16,15,0.011211200058460236
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,2,128,1,float16,fp8,15,0.012324800342321396
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,2,128,1,float16,fp8,31,0.012486399710178375
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,16,8,128,1,float16,fp8,2047,0.12461119890213013
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,2,128,1,float16,fp8,63,0.012246400117874146
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,2,128,1,float16,float16,127,0.011422400176525117
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,2,128,1,float16,fp8,127,0.01223199963569641
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,1,128,1,float16,fp8,511,0.017833599448204042
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,2,128,1,float16,fp8,255,0.013950400054454803
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,2,128,1,float16,float16,511,0.015934400260448456
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,2,128,1,float16,fp8,511,0.017873600125312805
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,2,128,1,float16,float16,1023,0.015380799770355225
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,2,128,1,float16,fp8,1023,0.016332800686359405
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,2,128,1,float16,float16,2047,0.015636800229549407
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,2,128,1,float16,fp8,2047,0.016979199647903443
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,2,128,1,float16,float16,4095,0.016921600699424742
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,2,128,1,float16,fp8,4095,0.017411200702190398
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,2,128,1,float16,float16,8191,0.01786080002784729
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,2,128,1,float16,fp8,8191,0.01873279958963394
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,2,128,1,float16,float16,16383,0.02064319998025894
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,2,128,1,float16,fp8,16383,0.020547200739383698
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,4,128,1,float16,float16,1,0.011579199880361556
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,4,128,1,float16,fp8,1,0.012044800072908401
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,2,128,1,float16,fp8,3,0.01228479966521263
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,1,128,1,float16,float16,7,0.011503999680280685
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,4,128,1,float16,float16,7,0.01146399974822998
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,4,128,1,float16,fp8,7,0.012323199957609176
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,1,128,1,float16,fp8,15,0.01228799968957901
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,2,128,1,float16,float16,31,0.01127839982509613
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,4,128,1,float16,float16,31,0.011476799845695496
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,4,128,1,float16,fp8,31,0.01225920021533966
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,2,128,1,float16,float16,63,0.011659199744462967
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,4,128,1,float16,float16,63,0.011548800021409988
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,4,128,1,float16,fp8,63,0.012409599870443344
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,4,128,1,float16,float16,127,0.011547199636697768
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,4,128,1,float16,fp8,127,0.01234079971909523
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,4,128,1,float16,float16,255,0.013232000172138214
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,4,128,1,float16,fp8,255,0.014419199526309967
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,4,128,1,float16,float16,511,0.016227200627326965
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,4,128,1,float16,fp8,511,0.01817599982023239
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,2,128,1,float16,float16,255,0.013104000687599182
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,4,128,1,float16,float16,1023,0.015471999347209931
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,4,128,1,float16,fp8,1023,0.01674720048904419
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,4,128,1,float16,fp8,2047,0.017190399765968322
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,4,128,1,float16,float16,4095,0.01714719980955124
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,4,128,1,float16,fp8,4095,0.017660799622535705
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,4,128,1,float16,float16,8191,0.01923840045928955
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,4,128,1,float16,fp8,8191,0.018859200179576874
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,4,128,1,float16,float16,16383,0.022780799865722658
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,8,128,1,float16,float16,1,0.011590400338172912
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,8,128,1,float16,fp8,1,0.012257599830627441
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,8,128,1,float16,float16,3,0.01157120019197464
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,8,128,1,float16,fp8,3,0.012243200093507767
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,8,128,1,float16,float16,7,0.011579199880361556
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,8,128,1,float16,fp8,7,0.012244799733161926
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,8,128,1,float16,float16,15,0.011633600294589996
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,8,128,1,float16,fp8,15,0.012212800234556198
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,8,128,1,float16,float16,31,0.011633600294589996
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,8,128,1,float16,fp8,31,0.012415999919176102
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,8,128,1,float16,float16,63,0.011699199676513672
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,8,128,1,float16,fp8,63,0.012412799894809723
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,8,128,1,float16,float16,127,0.011443199962377549
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,8,128,1,float16,fp8,127,0.012436799705028534
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,8,128,1,float16,float16,255,0.013075199723243714
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,8,128,1,float16,fp8,255,0.014502400159835815
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,8,128,1,float16,float16,511,0.016286399960517884
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,8,128,1,float16,fp8,511,0.018188799917697906
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,8,128,1,float16,float16,1023,0.015809600055217744
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,8,128,1,float16,fp8,1023,0.017097599804401398
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,8,128,1,float16,fp8,2047,0.017347200214862822
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,8,128,1,float16,float16,4095,0.018299199640750885
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,8,128,1,float16,fp8,4095,0.018063999712467194
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,8,128,1,float16,float16,8191,0.02192319929599762
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,4,128,1,float16,fp8,3,0.01231359988451004
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,8,128,1,float16,fp8,8191,0.021859200298786165
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,8,128,1,float16,float16,16383,0.03103039860725403
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,8,128,1,float16,fp8,16383,0.02784320116043091
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,1,128,1,float16,float16,1,0.011451199650764465
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,1,128,1,float16,fp8,1,0.012201599776744843
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,1,128,1,float16,float16,3,0.011537600308656693
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,1,128,1,float16,fp8,3,0.0122079998254776
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,1,128,1,float16,float16,7,0.011563199758529662
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,1,128,1,float16,fp8,7,0.012380799651145935
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,1,128,1,float16,float16,15,0.01162080019712448
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,1,128,1,float16,float16,31,0.01162559986114502
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,1,128,1,float16,fp8,31,0.01231039986014366
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,1,128,1,float16,float16,63,0.011566399782896041
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,1,128,1,float16,fp8,63,0.012241599708795547
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,1,128,1,float16,float16,127,0.011400000005960465
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,1,128,1,float16,fp8,127,0.012308800220489502
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,1,128,1,float16,float16,255,0.013321599364280701
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,1,128,1,float16,fp8,255,0.014297600090503692
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,1,128,1,float16,float16,511,0.016156800091266632
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,4,128,1,float16,fp8,16383,0.022881600260734557
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,1,128,1,float16,fp8,511,0.0182096004486084
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,1,128,1,float16,float16,1023,0.015622399747371674
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,1,128,1,float16,fp8,1023,0.016859200596809388
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,1,128,1,float16,float16,2047,0.016150400042533875
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,1,128,1,float16,fp8,2047,0.017233599722385407
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,1,128,1,float16,float16,4095,0.016777600347995757
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,1,128,1,float16,fp8,4095,0.017446400225162507
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,1,128,1,float16,float16,8191,0.018291200697422027
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,8,128,1,float16,float16,2047,0.016355200111865996
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,1,128,1,float16,fp8,8191,0.018702399730682374
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,1,128,1,float16,float16,16383,0.020817600190639496
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,2,128,1,float16,float16,1,0.011401599645614624
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,4,128,1,float16,float16,3,0.011608000099658965
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,2,128,1,float16,float16,3,0.011473599821329117
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,2,128,1,float16,fp8,3,0.012190400063991547
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,2,128,1,float16,float16,7,0.01144160032272339
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,4,128,1,float16,float16,15,0.011475200206041336
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,2,128,1,float16,fp8,7,0.012187200039625168
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,4,128,1,float16,fp8,15,0.012244799733161926
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,2,128,1,float16,float16,31,0.011454399675130844
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,2,128,1,float16,fp8,31,0.012267199903726577
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,2,128,1,float16,float16,63,0.011552000045776367
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,2,128,1,float16,fp8,63,0.01242239996790886
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,2,128,1,float16,float16,127,0.011584000289440155
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,2,128,1,float16,fp8,127,0.012387199699878693
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,2,128,1,float16,float16,255,0.013184000551700593
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,2,128,1,float16,fp8,255,0.014323200285434722
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,16,4,128,1,float16,float16,2047,0.016139200329780577
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,2,128,1,float16,fp8,511,0.018172800540924072
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,2,128,1,float16,float16,1023,0.015747199952602386
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,2,128,1,float16,fp8,1023,0.01693439930677414
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,2,128,1,float16,float16,2047,0.015907199680805208
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,2,128,1,float16,fp8,2047,0.017124800384044646
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,2,128,1,float16,float16,4095,0.016883200407028197
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,2,128,1,float16,fp8,4095,0.017731200158596038
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,2,128,1,float16,float16,8191,0.019083200395107268
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,2,128,1,float16,fp8,8191,0.01902880072593689
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,2,128,1,float16,float16,16383,0.022841599583625794
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,2,128,1,float16,fp8,16383,0.022435200214385987
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,1,128,1,float16,fp8,16383,0.021060800552368163
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,4,128,1,float16,float16,1,0.011425600200891495
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,4,128,1,float16,fp8,1,0.012291199713945388
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,2,128,1,float16,fp8,1,0.012127999961376191
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,4,128,1,float16,float16,7,0.01170239970088005
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,4,128,1,float16,fp8,7,0.01215519979596138
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,4,128,1,float16,float16,15,0.011687999963760376
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,4,128,1,float16,fp8,15,0.01234399974346161
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,4,128,1,float16,float16,31,0.011710400134325028
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,4,128,1,float16,fp8,31,0.012347199767827988
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,4,128,1,float16,float16,63,0.011668799817562104
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,4,128,1,float16,fp8,63,0.012326399981975555
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,4,128,1,float16,float16,127,0.011673600226640702
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,2,128,1,float16,fp8,15,0.01210239976644516
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,4,128,1,float16,fp8,127,0.012348800152540206
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,4,128,1,float16,float16,255,0.013193599879741669
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,4,128,1,float16,fp8,255,0.014025600254535675
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,4,128,1,float16,float16,511,0.01618400067090988
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,1,128,1,float16,fp8,15,0.012401600182056428
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,4,128,1,float16,float16,1023,0.01563999950885773
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,4,128,1,float16,fp8,1023,0.016868799924850464
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,4,128,1,float16,float16,2047,0.016499200463294984
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,4,128,1,float16,fp8,2047,0.01711200028657913
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,4,128,1,float16,float16,4095,0.018156799674034118
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,4,128,1,float16,fp8,4095,0.018110400438308714
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,4,128,1,float16,float16,8191,0.02162880003452301
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,4,128,1,float16,fp8,8191,0.02157119959592819
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,4,128,1,float16,float16,16383,0.03160960078239441
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,4,128,1,float16,fp8,16383,0.027823999524116516
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,8,128,1,float16,fp8,1,0.012428800016641617
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,8,128,1,float16,float16,3,0.011587200313806533
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,8,128,1,float16,fp8,3,0.012363199889659882
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,8,128,1,float16,float16,7,0.01168000027537346
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,8,128,1,float16,fp8,7,0.012299200147390365
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,8,128,1,float16,float16,15,0.01170400008559227
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,2,128,1,float16,float16,511,0.01605280041694641
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,8,128,1,float16,fp8,15,0.012432000041007996
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,8,128,1,float16,fp8,31,0.01242239996790886
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,8,128,1,float16,float16,63,0.011844799667596818
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,8,128,1,float16,fp8,63,0.012508800625801087
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,8,128,1,float16,float16,127,0.011972799897193909
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,8,128,1,float16,fp8,127,0.012569600343704223
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,8,128,1,float16,float16,255,0.013484799861907959
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,8,128,1,float16,fp8,255,0.01451520025730133
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,8,128,1,float16,float16,511,0.016806399822235106
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,8,128,1,float16,fp8,511,0.018262399733066557
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,8,128,1,float16,float16,1023,0.016201600432395935
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,8,128,1,float16,fp8,1023,0.01713760048151016
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,8,128,1,float16,float16,2047,0.01809920072555542
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,8,128,1,float16,fp8,2047,0.01759839951992035
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,8,128,1,float16,float16,4095,0.02146880030632019
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,8,128,1,float16,fp8,4095,0.02123039960861206
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,8,128,1,float16,float16,8191,0.03031199872493744
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,8,128,1,float16,fp8,8191,0.027260801196098326
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,8,128,1,float16,float16,16383,0.045151999592781066
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,8,128,1,float16,fp8,16383,0.04124000072479248
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,1,128,1,float16,float16,1,0.012580800056457519
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,1,128,1,float16,fp8,1,0.01363680064678192
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,4,128,1,float16,float16,3,0.01154400035738945
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,1,128,1,float16,float16,3,0.01271200031042099
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,4,128,1,float16,fp8,3,0.01223680004477501
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,1,128,1,float16,fp8,7,0.013184000551700593
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,1,128,1,float16,float16,15,0.012483199685811996
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,1,128,1,float16,fp8,15,0.013299199938774108
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,1,128,1,float16,float16,31,0.012600000202655792
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,2,128,1,float16,float16,15,0.011459200084209442
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,1,128,1,float16,float16,63,0.012374400347471236
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,4,128,1,float16,fp8,511,0.01786080002784729
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,1,128,1,float16,fp8,63,0.013009600341320038
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,1,128,1,float16,float16,127,0.012929600477218629
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,1,128,1,float16,fp8,127,0.013089600205421447
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,1,128,1,float16,float16,255,0.014537599682807923
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,1,128,1,float16,fp8,255,0.015041600167751312
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,1,128,1,float16,float16,511,0.017668800055980684
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,1,128,1,float16,fp8,511,0.018908800184726716
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,1,128,1,float16,float16,1023,0.018564799427986146
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,1,128,1,float16,fp8,1023,0.018223999440670012
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,1,128,1,float16,float16,2047,0.020737600326538087
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,1,128,1,float16,fp8,2047,0.021807999908924104
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,8,128,1,float16,float16,31,0.011636800318956374
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,1,128,1,float16,float16,8191,0.04676640033721924
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,1,128,1,float16,fp8,8191,0.041889598965644835
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,1,128,1,float16,float16,16383,0.07863199710845947
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,1,128,1,float16,fp8,16383,0.06681920289993286
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,2,128,1,float16,float16,1,0.012436799705028534
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,2,128,1,float16,fp8,1,0.013289600610733032
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,2,128,1,float16,float16,3,0.01231359988451004
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,2,128,1,float16,fp8,3,0.013067199289798737
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,2,128,1,float16,float16,7,0.012227199971675873
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,2,128,1,float16,fp8,7,0.013257600367069244
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,2,128,1,float16,float16,15,0.012571200728416443
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,2,128,1,float16,fp8,15,0.013369600474834441
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,1,128,1,float16,fp8,3,0.013651199638843536
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,1,128,1,float16,float16,7,0.012772800028324127
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,2,128,1,float16,fp8,31,0.01364160031080246
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,2,128,1,float16,float16,63,0.012803199887275695
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,2,128,1,float16,fp8,63,0.013592000305652618
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,2,128,1,float16,float16,127,0.012705600261688233
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,2,128,1,float16,fp8,127,0.013436800241470337
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,2,128,1,float16,float16,255,0.014542399346828461
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,2,128,1,float16,fp8,255,0.015563200414180755
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,2,128,1,float16,fp8,511,0.01918559968471527
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,2,128,1,float16,float16,1023,0.020321600139141083
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,2,128,1,float16,fp8,1023,0.021695999801158904
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,16,8,128,1,float16,float16,1,0.011393599957227708
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,2,128,1,float16,fp8,2047,0.027534401416778563
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,1,128,1,float16,float16,4095,0.03131360113620758
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,2,128,1,float16,fp8,4095,0.041684800386428834
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,2,128,1,float16,float16,8191,0.07799519896507263
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,2,128,1,float16,fp8,8191,0.06723840236663818
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,2,128,1,float16,float16,16383,0.13760160207748412
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,2,128,1,float16,fp8,16383,0.11661920547485352
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,4,128,1,float16,float16,1,0.012811200320720672
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,4,128,1,float16,fp8,1,0.013782399892807006
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,4,128,1,float16,fp8,3,0.013767999410629273
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,2,128,1,float16,float16,31,0.01255040019750595
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,4,128,1,float16,fp8,7,0.013513599336147309
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,4,128,1,float16,float16,15,0.012511999905109405
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,4,128,1,float16,fp8,15,0.013289600610733032
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,1,128,1,float16,fp8,31,0.013318400084972381
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,4,128,1,float16,fp8,31,0.013767999410629273
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,4,128,1,float16,float16,63,0.012865599989891053
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,4,128,1,float16,fp8,63,0.01372320055961609
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,4,128,1,float16,float16,127,0.012913599610328674
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,4,128,1,float16,fp8,127,0.013687999546527862
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,2,128,1,float16,float16,2047,0.03152799904346466
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,4,128,1,float16,float16,255,0.014873600006103516
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,4,128,1,float16,fp8,255,0.015854400396347047
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,4,128,1,float16,float16,511,0.01831520050764084
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,4,128,1,float16,fp8,511,0.01951040029525757
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,4,128,1,float16,float16,1023,0.029731199145317078
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,2,128,1,float16,float16,4095,0.046438398957252505
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,4,128,1,float16,fp8,1023,0.02677919864654541
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,4,128,1,float16,float16,2047,0.044761601090431216
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,4,128,1,float16,fp8,2047,0.039847999811172485
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,4,128,1,float16,float16,4095,0.07469599843025207
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,4,128,1,float16,float16,8191,0.13551199436187744
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,4,128,1,float16,fp8,8191,0.11572320461273193
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,4,128,1,float16,float16,16383,0.25419681072235106
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,4,128,1,float16,fp8,16383,0.2128079891204834
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,8,128,1,float16,float16,1,0.01659359931945801
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,8,128,1,float16,fp8,1,0.01775519996881485
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,8,128,1,float16,float16,3,0.01664319932460785
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,8,128,1,float16,fp8,3,0.017926399409770966
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,8,128,1,float16,float16,7,0.0165583997964859
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,8,128,1,float16,fp8,7,0.017947199940681457
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,8,128,1,float16,float16,15,0.01658719927072525
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,8,128,1,float16,fp8,15,0.01790879964828491
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,8,128,1,float16,float16,31,0.01669600009918213
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,8,128,1,float16,fp8,31,0.01791359931230545
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,8,128,1,float16,float16,63,0.01687040030956268
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,8,128,1,float16,fp8,63,0.018001599609851836
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,8,128,1,float16,float16,127,0.016998399794101716
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,8,128,1,float16,fp8,127,0.017950400710105896
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,8,128,1,float16,float16,255,0.020510399341583253
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,8,128,1,float16,fp8,255,0.021817600727081297
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,8,128,1,float16,float16,511,0.029889601469039916
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,8,128,1,float16,fp8,511,0.02940959930419922
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,2,128,1,float16,float16,511,0.01759999990463257
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,4,128,1,float16,float16,31,0.012507200241088867
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,8,128,1,float16,float16,1023,0.04716640114784241
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,8,128,1,float16,fp8,1023,0.04505920112133026
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,8,128,1,float16,float16,2047,0.07681279778480529
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,8,128,1,float16,fp8,2047,0.06831520199775695
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,8,128,1,float16,float16,4095,0.13694560527801514
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,8,128,1,float16,fp8,4095,0.11828800439834594
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,8,128,1,float16,float16,8191,0.255679988861084
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,8,128,1,float16,fp8,8191,0.218123197555542
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,1,128,1,float16,fp8,4095,0.02776640057563782
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,1,128,1,float16,fp8,1,0.012217599898576736
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,1,128,1,float16,float16,3,0.011444800347089768
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,8,128,1,float16,fp8,16383,0.40918397903442383
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,1,128,1,float16,float16,7,0.011347199976444244
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,1,128,1,float16,fp8,7,0.012244799733161926
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,1,128,1,float16,float16,15,0.011505600064992905
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,1,128,1,float16,fp8,15,0.012211199849843979
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,1,128,1,float16,float16,31,0.011390399932861329
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,1,128,1,float16,fp8,31,0.012051200121641159
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,1,128,1,float16,float16,63,0.011547199636697768
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,1,128,1,float16,fp8,63,0.012241599708795547
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,4,128,1,float16,float16,3,0.01289760023355484
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,1,128,1,float16,fp8,127,0.012238399684429168
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,4,128,1,float16,float16,7,0.012918399274349212
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,1,128,1,float16,fp8,255,0.014382399618625641
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,1,128,1,float16,float16,511,0.016190400719642638
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,1,128,1,float16,fp8,511,0.01809599995613098
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,1,128,1,float16,float16,1023,0.01587519943714142
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,1,128,1,float16,fp8,1023,0.016771200299263
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,1,128,1,float16,float16,2047,0.0161408007144928
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,1,128,1,float16,fp8,2047,0.017318400740623473
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,1,128,1,float16,float16,4095,0.017182399332523347
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,1,128,1,float16,fp8,4095,0.017772799730300902
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,1,128,1,float16,float16,8191,0.019382399320602418
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,1,128,1,float16,fp8,8191,0.01903039962053299
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,1,128,1,float16,float16,16383,0.02300799936056137
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,1,128,1,float16,float16,1,0.01159520000219345
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,1,128,1,float16,fp8,16383,0.023046399652957916
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,8,128,1,float16,float16,16383,0.4899456024169922
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,2,128,1,float16,fp8,1,0.012316799908876418
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,16,4,128,1,float16,fp8,4095,0.06598719954490662
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,2,128,1,float16,fp8,3,0.012220799922943115
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,2,128,1,float16,float16,3,0.011575999855995178
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,2,128,1,float16,fp8,7,0.012225600332021714
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,2,128,1,float16,fp8,15,0.012227199971675873
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,2,128,1,float16,float16,31,0.011552000045776367
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,2,128,1,float16,fp8,31,0.012214399874210358
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,2,128,1,float16,float16,63,0.01170239970088005
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,2,128,1,float16,fp8,63,0.012380799651145935
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,2,128,1,float16,float16,127,0.011691199988126755
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,2,128,1,float16,fp8,127,0.012414400279521943
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,2,128,1,float16,float16,255,0.013286399841308593
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,2,128,1,float16,fp8,255,0.014292800426483154
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,1,128,1,float16,float16,127,0.011523199826478958
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,2,128,1,float16,float16,511,0.016198399662971496
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,2,128,1,float16,fp8,511,0.01828960031270981
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,1,128,1,float16,float16,255,0.013132800161838532
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,2,128,1,float16,fp8,1023,0.017059199512004852
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,2,128,1,float16,float16,2047,0.016433599591255187
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,2,128,1,float16,fp8,2047,0.01725279986858368
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,2,128,1,float16,float16,4095,0.018275199830532073
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,2,128,1,float16,fp8,4095,0.017772799730300902
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,2,128,1,float16,float16,8191,0.021670399606227873
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,2,128,1,float16,fp8,8191,0.021646399796009064
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,2,128,1,float16,float16,16383,0.03144800066947937
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,4,128,1,float16,float16,1,0.011524800211191177
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,4,128,1,float16,fp8,1,0.012238399684429168
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,4,128,1,float16,float16,3,0.011660800129175187
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,4,128,1,float16,fp8,3,0.01239679977297783
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,4,128,1,float16,float16,7,0.011671999841928482
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,4,128,1,float16,fp8,7,0.01239520013332367
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,4,128,1,float16,float16,15,0.011710400134325028
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,4,128,1,float16,fp8,15,0.012403199821710587
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,4,128,1,float16,float16,31,0.011736000329256058
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,4,128,1,float16,fp8,31,0.01239359974861145
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,4,128,1,float16,float16,63,0.011788800358772278
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,4,128,1,float16,fp8,63,0.012427199631929398
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,4,128,1,float16,float16,127,0.011641599982976914
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,4,128,1,float16,fp8,127,0.012612800300121307
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,4,128,1,float16,float16,255,0.013100799918174744
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,4,128,1,float16,fp8,255,0.014337599277496338
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,4,128,1,float16,float16,511,0.016172799468040466
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,4,128,1,float16,fp8,511,0.01804639995098114
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,4,128,1,float16,float16,1023,0.016097599267959596
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,4,128,1,float16,fp8,1023,0.01693760007619858
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,4,128,1,float16,float16,2047,0.017924800515174866
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,4,128,1,float16,fp8,2047,0.017470400035381316
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,4,128,1,float16,float16,4095,0.021196800470352172
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,1,128,1,float16,fp8,3,0.01215519979596138
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,4,128,1,float16,fp8,4095,0.021240000426769257
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,4,128,1,float16,float16,8191,0.030215999484062193
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,4,128,1,float16,fp8,8191,0.027422401309013366
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,4,128,1,float16,fp8,16383,0.04064480066299438
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,8,128,1,float16,float16,1,0.011795199662446975
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,8,128,1,float16,fp8,1,0.012606400251388549
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,8,128,1,float16,float16,3,0.011913599818944931
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,8,128,1,float16,fp8,3,0.012535999715328216
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,8,128,1,float16,float16,7,0.01196959987282753
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,8,128,1,float16,float16,15,0.012009599804878235
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,8,128,1,float16,fp8,15,0.01276639997959137
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,8,128,1,float16,float16,31,0.011990399658679962
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,2,128,1,float16,float16,1023,0.01579039990901947
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,8,128,1,float16,fp8,31,0.012596799433231354
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,8,128,1,float16,float16,63,0.011753600090742111
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,8,128,1,float16,fp8,63,0.012614400684833526
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,8,128,1,float16,float16,127,0.01197120025753975
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,8,128,1,float16,fp8,127,0.01271200031042099
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,2,128,1,float16,fp8,16383,0.02752000093460083
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,8,128,1,float16,float16,255,0.01353919953107834
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,8,128,1,float16,float16,511,0.016708800196647645
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,8,128,1,float16,fp8,511,0.01823839992284775
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,8,128,1,float16,float16,1023,0.017895999550819396
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,8,128,1,float16,fp8,1023,0.01748799979686737
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,8,128,1,float16,float16,2047,0.01963520050048828
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,8,128,1,float16,fp8,2047,0.02088479995727539
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,8,128,1,float16,float16,4095,0.029783999919891356
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,2,128,1,float16,float16,1,0.011769600212574005
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,8,128,1,float16,fp8,4095,0.02686559855937958
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,8,128,1,float16,float16,8191,0.044740799069404605
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,4,128,1,float16,float16,16383,0.045161598920822145
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,8,128,1,float16,fp8,16383,0.06589279770851135
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,1,128,1,float16,float16,1,0.015222400426864624
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,1,128,1,float16,fp8,1,0.015500800311565399
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,1,128,1,float16,float16,3,0.014612799882888794
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,1,128,1,float16,fp8,3,0.015447999536991119
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,1,128,1,float16,float16,7,0.014731200039386749
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,1,128,1,float16,fp8,7,0.015591999888420105
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,1,128,1,float16,float16,15,0.014574399590492249
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,1,128,1,float16,fp8,15,0.015248000621795654
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,1,128,1,float16,float16,31,0.014614400267601014
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,1,128,1,float16,fp8,31,0.015403200685977936
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,1,128,1,float16,float16,63,0.01467680037021637
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,1,128,1,float16,fp8,63,0.015427200496196747
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,1,128,1,float16,float16,127,0.014955200254917145
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,1,128,1,float16,fp8,127,0.015617600083351136
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,8,128,1,float16,fp8,255,0.01435520052909851
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,1,128,1,float16,float16,255,0.016649599373340606
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,1,128,1,float16,fp8,255,0.017289599776268004
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,1,128,1,float16,fp8,511,0.021163199841976166
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,1,128,1,float16,float16,1023,0.03251039981842041
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,1,128,1,float16,fp8,1023,0.028683200478553772
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,1,128,1,float16,float16,2047,0.0471807986497879
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,1,128,1,float16,fp8,2047,0.04185439944267273
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,8,128,1,float16,fp8,8191,0.0414463996887207
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,1,128,1,float16,float16,4095,0.07886559963226318
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,2,128,1,float16,float16,15,0.01151840016245842
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,1,128,1,float16,fp8,4095,0.06824640035629273
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,1,128,1,float16,float16,8191,0.13959360122680664
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,1,128,1,float16,fp8,8191,0.1177024006843567
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,2,128,1,float16,fp8,1,0.02009280025959015
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,2,128,1,float16,float16,3,0.018464000523090364
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,2,128,1,float16,float16,7,0.01833759993314743
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,2,128,1,float16,float16,15,0.01823039948940277
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,2,128,1,float16,fp8,15,0.019728000462055206
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,2,128,1,float16,float16,31,0.018675200641155243
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,2,128,1,float16,fp8,31,0.020206399261951447
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,2,128,1,float16,float16,63,0.018675200641155243
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,2,128,1,float16,fp8,63,0.020371200144290925
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,2,128,1,float16,float16,127,0.018993599712848662
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,2,128,1,float16,fp8,127,0.020444799959659577
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,2,128,1,float16,float16,255,0.022196799516677856
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,2,128,1,float16,fp8,255,0.024400000274181367
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,2,128,1,float16,float16,511,0.033913600444793704
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,1,128,1,float16,float16,511,0.01980320066213608
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,2,128,1,float16,fp8,511,0.03172959983348846
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,2,128,1,float16,float16,1023,0.0506384015083313
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,2,128,1,float16,float16,2047,0.08015360236167908
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,2,128,1,float16,fp8,2047,0.07085440158843995
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,2,128,1,float16,float16,4095,0.13995840549468994
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,8,128,1,float16,float16,16383,0.07569760084152222
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,2,128,1,float16,fp8,4095,0.12066080570220947
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,2,128,1,float16,float16,1,0.018670399487018586
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,8,128,1,float16,fp8,7,0.01276479959487915
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,2,128,1,float16,float16,8191,0.25882079601287844
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,2,128,1,float16,fp8,3,0.019659200310707094
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,2,128,1,float16,fp8,7,0.01977120041847229
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,4,128,1,float16,fp8,3,0.02855679988861084
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,4,128,1,float16,float16,3,0.026102399826049803
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,4,128,1,float16,float16,7,0.02576479911804199
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,4,128,1,float16,fp8,7,0.028553599119186403
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,4,128,1,float16,float16,15,0.026280000805854797
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,4,128,1,float16,fp8,15,0.028863999247550964
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,4,128,1,float16,fp8,31,0.02943840026855469
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,4,128,1,float16,float16,63,0.025913599133491515
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,4,128,1,float16,fp8,63,0.029281601309776306
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,4,128,1,float16,float16,127,0.02595840096473694
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,4,128,1,float16,fp8,127,0.02922559976577759
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,4,128,1,float16,float16,255,0.03914879858493805
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,4,128,1,float16,float16,511,0.05258719921112061
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,4,128,1,float16,fp8,511,0.053502398729324344
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,4,128,1,float16,float16,1023,0.08469759821891784
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,4,128,1,float16,fp8,1023,0.07985759973526001
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,2,128,1,float16,fp8,1023,0.047513601183891294
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,4,128,1,float16,float16,2047,0.14384959936141967
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,4,128,1,float16,fp8,2047,0.1261136054992676
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,4,128,1,float16,float16,4095,0.2600640058517456
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,16,2,128,1,float16,float16,7,0.011606399714946748
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,4,128,1,float16,float16,1,0.026023998856544495
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,4,128,1,float16,float16,8191,0.49817600250244143
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,8,128,1,float16,float16,1,0.040454399585723874
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,8,128,1,float16,float16,3,0.04047040045261383
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,8,128,1,float16,fp8,1,0.046588799357414244
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,4,128,1,float16,float16,31,0.025668799877166748
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,8,128,1,float16,fp8,3,0.04640479981899261
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,8,128,1,float16,float16,7,0.040007999539375304
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,8,128,1,float16,fp8,7,0.0465503990650177
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,8,128,1,float16,float16,15,0.04024479985237121
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,8,128,1,float16,fp8,15,0.04662240147590637
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,8,128,1,float16,float16,31,0.04040479958057404
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,4,128,1,float16,fp8,255,0.03739520013332367
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,8,128,1,float16,fp8,31,0.04667200148105621
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,8,128,1,float16,float16,63,0.040910398960113524
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,8,128,1,float16,fp8,63,0.0468095988035202
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,8,128,1,float16,float16,127,0.04601120054721832
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,8,128,1,float16,float16,255,0.06235520243644714
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,8,128,1,float16,float16,511,0.08963680267333984
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,8,128,1,float16,fp8,511,0.09524319767951965
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,8,128,1,float16,float16,1023,0.15331679582595825
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,8,128,1,float16,fp8,1023,0.14473600387573243
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,4,128,1,float16,fp8,4095,0.22481279373168944
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,8,128,1,float16,float16,2047,0.2715679883956909
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,8,128,1,float16,fp8,2047,0.23712639808654784
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,4,128,1,float16,fp8,1,0.028958401083946227
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,4,128,1,float16,fp8,8191,0.42194399833679197
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,8,128,1,float16,float16,4095,0.5124320030212403
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,8,128,1,float16,fp8,4095,0.43553919792175294
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,1,128,1,float16,float16,1,0.020947200059890748
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,1,128,1,float16,float16,3,0.021031999588012697
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,8,128,1,float16,fp8,127,0.04723039865493774
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,1,128,1,float16,fp8,3,0.022676800191402436
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,8,128,1,float16,fp8,255,0.06484159827232361
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,1,128,1,float16,fp8,7,0.02259040027856827
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,1,128,1,float16,float16,15,0.02091359943151474
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,1,128,1,float16,fp8,15,0.02248000055551529
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,1,128,1,float16,float16,31,0.020924800634384157
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,1,128,1,float16,fp8,31,0.02239840030670166
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,1,128,1,float16,float16,63,0.020870399475097657
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,1,128,1,float16,fp8,63,0.022438399493694305
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,1,128,1,float16,fp8,127,0.02274399995803833
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,1,128,1,float16,float16,255,0.024803200364112855
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,1,128,1,float16,fp8,255,0.02650879919528961
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,1,128,1,float16,float16,511,0.03654879927635193
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,1,128,1,float16,fp8,511,0.03410399854183197
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,1,128,1,float16,float16,1023,0.0548192024230957
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,1,128,1,float16,fp8,1023,0.050639998912811277
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,1,128,1,float16,float16,2047,0.08393279910087585
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,1,128,1,float16,fp8,2047,0.07390400171279907
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,2,128,1,float16,float16,1,0.02844800055027008
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,2,128,1,float16,fp8,1,0.03115360140800476
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,2,128,1,float16,float16,3,0.028248000144958495
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,2,128,1,float16,fp8,3,0.03133760094642639
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,2,128,1,float16,float16,7,0.028279998898506166
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,2,128,1,float16,fp8,7,0.031376001238822934
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,1,128,1,float16,float16,7,0.02077919989824295
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,2,128,1,float16,float16,31,0.028064000606536865
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,2,128,1,float16,fp8,31,0.030969598889350893
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,2,128,1,float16,float16,63,0.028062400221824647
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,1,128,1,float16,float16,127,0.020868800580501556
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,2,128,1,float16,float16,127,0.028270399570465087
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,2,128,1,float16,fp8,127,0.031068798899650574
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,2,128,1,float16,float16,255,0.0416159987449646
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,2,128,1,float16,fp8,255,0.039750400185585025
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,2,128,1,float16,float16,511,0.05562719702720642
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,2,128,1,float16,fp8,511,0.05631999969482422
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,2,128,1,float16,float16,1023,0.0892687976360321
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,2,128,1,float16,fp8,1023,0.08243200182914734
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,2,128,1,float16,fp8,8191,0.22120959758758546
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,2,128,1,float16,float16,2047,0.14680800437927247
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,2,128,1,float16,fp8,2047,0.1296623945236206
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,4,128,1,float16,float16,1,0.043089601397514346
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,4,128,1,float16,fp8,1,0.049112001061439516
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,4,128,1,float16,float16,3,0.04303520023822784
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,4,128,1,float16,fp8,3,0.048814401030540466
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,4,128,1,float16,float16,7,0.04307680130004883
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,4,128,1,float16,fp8,7,0.048974400758743285
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,4,128,1,float16,float16,15,0.04307039976119995
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,4,128,1,float16,fp8,15,0.04907999932765961
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,4,128,1,float16,float16,31,0.0431088000535965
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,4,128,1,float16,fp8,31,0.04916160106658936
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,2,128,1,float16,float16,15,0.028411200642585753
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,4,128,1,float16,float16,63,0.04358560144901276
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,4,128,1,float16,fp8,63,0.049065598845481874
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,2,128,1,float16,fp8,15,0.03115519881248474
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,4,128,1,float16,float16,127,0.04860160052776337
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,4,128,1,float16,fp8,127,0.04981600046157837
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,4,128,1,float16,float16,255,0.06554239988327026
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,4,128,1,float16,fp8,255,0.06671839952468872
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,4,128,1,float16,float16,511,0.0925935983657837
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,4,128,1,float16,fp8,511,0.09661120176315308
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,4,128,1,float16,float16,1023,0.15567200183868407
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,2,128,1,float16,fp8,63,0.030983999371528625
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,4,128,1,float16,fp8,1023,0.14668320417404174
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,8,128,1,float16,float16,1,0.07137119770050049
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,8,128,1,float16,fp8,1,0.08395519852638245
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,4,128,1,float16,fp8,2047,0.23884959220886232
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,8,128,1,float16,fp8,3,0.08411679863929748
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,8,128,1,float16,float16,15,0.07153760194778443
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,8,128,1,float16,fp8,15,0.08448799848556518
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,8,128,1,float16,float16,31,0.07174400091171265
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,8,128,1,float16,fp8,31,0.08485280275344849
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,8,128,1,float16,float16,63,0.07677279710769654
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,1,128,1,float16,fp8,1,0.022358399629592896
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,8,128,1,float16,fp8,63,0.08501920104026794
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,8,128,1,float16,float16,127,0.08035039901733398
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,8,128,1,float16,fp8,127,0.09241120219230652
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,8,128,1,float16,float16,255,0.1081712007522583
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,8,128,1,float16,fp8,255,0.11958240270614624
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,4,128,1,float16,float16,2047,0.27544479370117186
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,8,128,1,float16,float16,7,0.07084640264511108
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,8,128,1,float16,fp8,7,0.08455520272254943
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,8,128,1,float16,fp8,511,0.17723519802093507
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,8,128,1,float16,float16,1023,0.2915136098861694
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,8,128,1,float16,fp8,1023,0.2737296104431152
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,1,128,1,float16,float16,1,0.034360000491142274
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,1,128,1,float16,fp8,1,0.0374208003282547
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,1,128,1,float16,float16,3,0.03447520136833191
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,1,128,1,float16,fp8,3,0.03728159964084625
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,1,128,1,float16,float16,7,0.03416639864444733
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,1,128,1,float16,fp8,7,0.03737919926643372
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,1,128,1,float16,float16,15,0.03438239991664886
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,1,128,1,float16,fp8,15,0.037380799651145935
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,1,128,1,float16,float16,31,0.03434880077838898
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,1,128,1,float16,fp8,31,0.03743039965629578
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,8,128,1,float16,float16,8191,0.9962431907653808
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,1,128,1,float16,float16,127,0.03447839915752411
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,1,128,1,float16,fp8,63,0.03733119964599609
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,1,128,1,float16,fp8,127,0.037467199563980105
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,1,128,1,float16,float16,255,0.048660799860954285
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,1,128,1,float16,float16,511,0.06218240261077881
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,8,128,1,float16,float16,3,0.07167199850082398
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,8,128,1,float16,float16,511,0.16383039951324463
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,1,128,1,float16,fp8,511,0.06259040236473083
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,1,128,1,float16,float16,1023,0.09714879989624023
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,2,128,1,float16,float16,1,0.04848639965057373
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,1,128,1,float16,fp8,1023,0.08931840062141419
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,2,128,1,float16,float16,3,0.04844639897346496
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,2,128,1,float16,fp8,3,0.054448002576828004
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,2,128,1,float16,float16,7,0.048449599742889406
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,2,128,1,float16,fp8,7,0.05431039929389954
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,2,128,1,float16,float16,15,0.048614400625228885
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,2,128,1,float16,fp8,15,0.05443680286407471
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,2,128,1,float16,float16,31,0.048630398511886594
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,2,128,1,float16,fp8,31,0.05444480180740356
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,8,128,1,float16,float16,2047,0.5305200099945069
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,2,128,1,float16,fp8,63,0.054548799991607666
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,16,8,128,1,float16,fp8,2047,0.45899357795715334
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,2,128,1,float16,fp8,127,0.05568479895591736
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,2,128,1,float16,float16,127,0.056353598833084106
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,2,128,1,float16,float16,255,0.07278559803962707
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,2,128,1,float16,float16,511,0.10022239685058594
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,2,128,1,float16,fp8,511,0.10265599489212036
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,2,128,1,float16,float16,1023,0.16419520378112792
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,2,128,1,float16,fp8,1023,0.15373920202255248
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,4,128,1,float16,float16,1,0.07681599855422974
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,4,128,1,float16,fp8,1,0.0893999993801117
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,4,128,1,float16,float16,3,0.07715839743614197
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,16,8,128,1,float16,fp8,8191,0.831390380859375
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,4,128,1,float16,float16,7,0.07709599733352661
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,4,128,1,float16,float16,15,0.07722240090370178
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,4,128,1,float16,fp8,7,0.08967519998550415
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,4,128,1,float16,fp8,15,0.08972799777984619
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,4,128,1,float16,fp8,31,0.09001759886741638
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,4,128,1,float16,float16,31,0.07723199725151061
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,4,128,1,float16,float16,63,0.081768000125885
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,4,128,1,float16,fp8,63,0.09108960032463073
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,4,128,1,float16,float16,127,0.08524159789085388
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,4,128,1,float16,fp8,127,0.09602400064468383
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,4,128,1,float16,float16,255,0.11519039869308471
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,2,128,1,float16,fp8,1,0.054467201232910156
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,4,128,1,float16,fp8,255,0.12276639938354492
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,4,128,1,float16,float16,511,0.16876800060272218
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,4,128,1,float16,fp8,511,0.18393440246582032
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,4,128,1,float16,float16,1023,0.2963376045227051
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,2,128,1,float16,float16,63,0.04900639951229095
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,8,128,1,float16,float16,1,0.13099520206451415
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,4,128,1,float16,fp8,1023,0.2818511962890625
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,8,128,1,float16,fp8,1,0.1570512056350708
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,8,128,1,float16,float16,7,0.13100320100784302
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,8,128,1,float16,fp8,7,0.1572160005569458
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,8,128,1,float16,float16,15,0.131167995929718
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,8,128,1,float16,fp8,15,0.15778399705886842
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,8,128,1,float16,float16,31,0.13722560405731202
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,8,128,1,float16,fp8,31,0.15761120319366456
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,8,128,1,float16,float16,63,0.14039360284805297
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,8,128,1,float16,fp8,63,0.16727839708328246
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,8,128,1,float16,float16,127,0.14486080408096313
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,8,128,1,float16,fp8,127,0.17080479860305786
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,8,128,1,float16,float16,255,0.20084960460662843
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,1,128,1,float16,fp8,255,0.0454367995262146
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,8,128,1,float16,float16,511,0.30749919414520266
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,8,128,1,float16,fp8,511,0.340065598487854
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,8,128,1,float16,float16,1023,0.5580063819885254
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,8,128,1,float16,fp8,1023,0.5333024024963379
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,8,128,1,float16,float16,3,0.13055039644241334
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,1,128,1,float16,fp8,1,0.012449599802494049
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,1,128,1,float16,float16,3,0.011582399904727935
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,2,128,1,float16,fp8,255,0.07154880166053772
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,1,128,1,float16,float16,7,0.011830399930477142
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,1,128,1,float16,fp8,7,0.01239520013332367
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,1,128,1,float16,fp8,15,0.01241919994354248
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,1,128,1,float16,float16,31,0.011630400270223617
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,4,128,1,float16,fp8,3,0.08926079869270324
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,1,128,1,float16,fp8,31,0.012415999919176102
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,1,128,1,float16,float16,63,0.011694400012493134
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,1,128,1,float16,fp8,63,0.012342400103807449
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,1,128,1,float16,float16,127,0.011556799709796905
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,1,128,1,float16,fp8,127,0.01223680004477501
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,1,128,1,float16,float16,255,0.013156799972057343
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,1,128,1,float16,fp8,255,0.014292800426483154
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,1,128,1,float16,float16,511,0.01618559956550598
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,1,128,1,float16,fp8,511,0.018427200615406036
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,1,128,1,float16,float16,1023,0.01605280041694641
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,1,128,1,float16,fp8,1023,0.017080000042915343
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,1,128,1,float16,float16,2047,0.01652639955282211
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,1,128,1,float16,fp8,2047,0.017286400496959686
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,1,128,1,float16,float16,4095,0.018406400084495546
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,1,128,1,float16,fp8,4095,0.01825760006904602
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,1,128,1,float16,float16,8191,0.022103999555110932
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,1,128,1,float16,float16,1,0.011790399998426437
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,1,128,1,float16,fp8,8191,0.021984000504016877
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,1,128,1,float16,float16,16383,0.03229599893093109
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,2,128,1,float16,float16,1,0.011740799993276596
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,2,128,1,float16,fp8,1,0.01249760016798973
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,1,128,1,float16,float16,15,0.011923199892044068
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,2,128,1,float16,fp8,3,0.012683199346065521
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,2,128,1,float16,float16,7,0.011915200203657151
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,2,128,1,float16,fp8,7,0.012656000256538392
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,1,128,1,float16,float16,63,0.03440960049629212
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,2,128,1,float16,float16,15,0.011953599750995636
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,2,128,1,float16,fp8,15,0.012520000338554382
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,2,128,1,float16,fp8,31,0.012747199833393097
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,2,128,1,float16,float16,63,0.011852800101041793
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,2,128,1,float16,fp8,63,0.012561599910259246
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,8,128,1,float16,fp8,255,0.22339839935302735
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,2,128,1,float16,float16,127,0.011851199716329575
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,2,128,1,float16,fp8,127,0.012587200105190276
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,2,128,1,float16,fp8,255,0.01454080045223236
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,2,128,1,float16,float16,511,0.016385599970817566
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,2,128,1,float16,fp8,511,0.018190400302410127
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,2,128,1,float16,float16,1023,0.016012799739837647
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,2,128,1,float16,fp8,1023,0.017259199917316437
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,2,128,1,float16,float16,2047,0.01809120029211044
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,2,128,1,float16,fp8,2047,0.01770399957895279
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,2,128,1,float16,float16,4095,0.021267199516296388
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,2,128,1,float16,fp8,4095,0.021376000344753267
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,2,128,1,float16,float16,8191,0.03127839863300323
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,2,128,1,float16,fp8,8191,0.0275983989238739
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,16,8,128,1,float16,fp8,3,0.15752320289611815
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,2,128,1,float16,float16,16383,0.04595040082931519
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,1,128,1,float16,fp8,3,0.012430399656295776
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,4,128,1,float16,float16,1,0.01191840022802353
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,2,128,1,float16,fp8,16383,0.04228320121765137
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,4,128,1,float16,fp8,1,0.01276639997959137
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,4,128,1,float16,float16,3,0.011795199662446975
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,2,128,1,float16,float16,3,0.01170559972524643
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,4,128,1,float16,fp8,7,0.012654399871826172
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,4,128,1,float16,float16,15,0.011787199974060058
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,4,128,1,float16,float16,31,0.012151999771595002
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,4,128,1,float16,fp8,31,0.012534399330615998
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,4,128,1,float16,float16,63,0.01218079999089241
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,4,128,1,float16,fp8,63,0.012665599584579468
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,4,128,1,float16,float16,127,0.012212800234556198
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,4,128,1,float16,fp8,127,0.012624000012874604
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,4,128,1,float16,fp8,255,0.014630399644374847
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,4,128,1,float16,float16,511,0.016657599806785585
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,4,128,1,float16,fp8,511,0.018310399353504182
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,2,128,1,float16,float16,31,0.011796800047159195
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,4,128,1,float16,float16,1023,0.017652800679206847
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,4,128,1,float16,fp8,1023,0.0172992005944252
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,4,128,1,float16,float16,2047,0.019468800723552705
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,4,128,1,float16,fp8,2047,0.0212336003780365
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,4,128,1,float16,float16,4095,0.03007520139217377
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,4,128,1,float16,fp8,4095,0.026958400011062623
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,4,128,1,float16,float16,8191,0.04514720141887665
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,2,128,1,float16,float16,255,0.013622400164604188
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,4,128,1,float16,fp8,8191,0.04089919924736023
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,4,128,1,float16,float16,16383,0.07612800002098083
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,8,128,1,float16,float16,1,0.012228800356388092
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,4,128,1,float16,fp8,16383,0.0657696008682251
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,8,128,1,float16,float16,3,0.012211199849843979
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,8,128,1,float16,fp8,3,0.012947200238704682
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,8,128,1,float16,float16,7,0.012222400307655335
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,8,128,1,float16,fp8,7,0.012899200618267059
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,8,128,1,float16,fp8,15,0.012892800569534301
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,8,128,1,float16,float16,31,0.012118399888277055
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,8,128,1,float16,fp8,31,0.012835200130939483
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,8,128,1,float16,float16,63,0.012118399888277055
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,8,128,1,float16,fp8,63,0.012839999794960023
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,8,128,1,float16,float16,127,0.012160000205039979
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,8,128,1,float16,fp8,127,0.012995199859142303
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,8,128,1,float16,float16,255,0.014033600687980652
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,8,128,1,float16,fp8,255,0.014895999431610107
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,8,128,1,float16,float16,511,0.017183999717235564
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,4,128,1,float16,fp8,3,0.012803199887275695
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,4,128,1,float16,fp8,15,0.012643200159072877
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,8,128,1,float16,fp8,1023,0.02095839977264404
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,8,128,1,float16,float16,2047,0.029948800802230835
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,4,128,1,float16,float16,255,0.013755199313163758
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,8,128,1,float16,fp8,2047,0.027115198969841003
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,8,128,1,float16,float16,4095,0.04519680142402649
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,8,128,1,float16,fp8,4095,0.04175040125846863
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,8,128,1,float16,float16,8191,0.07550879716873168
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,8,128,1,float16,fp8,8191,0.06695839762687683
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,8,128,1,float16,float16,16383,0.13485920429229736
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,8,128,1,float16,fp8,16383,0.11656479835510254
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,8,128,1,float16,fp8,1,0.01279360055923462
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,1,128,1,float16,fp8,1,0.06093599796295166
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,1,128,1,float16,float16,3,0.054763197898864746
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,1,128,1,float16,fp8,3,0.06057279706001282
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,1,128,1,float16,float16,7,0.054756802320480344
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,1,128,1,float16,fp8,7,0.06065120100975037
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,1,128,1,float16,float16,15,0.0550000011920929
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,1,128,1,float16,fp8,16383,0.02812960147857666
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,1,128,1,float16,fp8,15,0.06086080074310303
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,1,128,1,float16,float16,31,0.05475199818611145
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,8,128,1,float16,fp8,511,0.018598400056362152
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,1,128,1,float16,float16,63,0.05674399733543396
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,8,128,1,float16,float16,1023,0.019763199985027312
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,1,128,1,float16,fp8,63,0.06128479838371277
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,1,128,1,float16,float16,127,0.061504000425338747
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,1,128,1,float16,fp8,127,0.06305599808692933
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,1,128,1,float16,float16,255,0.07814080119132996
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,1,128,1,float16,fp8,255,0.07851359844207764
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,1,128,1,float16,float16,511,0.10737760066986084
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,2,128,1,float16,float16,1,0.0833519995212555
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,2,128,1,float16,fp8,1,0.0946175992488861
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,2,128,1,float16,float16,3,0.08327839970588684
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,2,128,1,float16,fp8,3,0.09465439915657044
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,2,128,1,float16,float16,7,0.08311039805412293
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,2,128,1,float16,fp8,7,0.09474400281906128
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,2,128,1,float16,fp8,15,0.09473919868469238
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,2,128,1,float16,float16,31,0.08371840119361877
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,2,128,1,float16,fp8,31,0.09505280256271362
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,1,128,1,float16,float16,1,0.05492479801177978
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,2,128,1,float16,float16,63,0.08827360272407532
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,2,128,1,float16,fp8,63,0.09703519940376282
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,2,128,1,float16,float16,127,0.0917743980884552
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,2,128,1,float16,float16,255,0.12331039905548095
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,2,128,1,float16,fp8,255,0.12882720232009887
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,2,128,1,float16,float16,511,0.17759040594100953
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,1,128,1,float16,fp8,31,0.06090400218963623
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,4,128,1,float16,float16,7,0.011846400052309036
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,2,128,1,float16,fp8,511,0.19162559509277344
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,4,128,1,float16,float16,3,0.1389680027961731
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,4,128,1,float16,fp8,3,0.16307519674301146
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,4,128,1,float16,float16,7,0.13885120153427125
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,1,128,1,float16,fp8,511,0.11054879426956177
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,4,128,1,float16,fp8,7,0.1634559988975525
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,4,128,1,float16,float16,15,0.13852479457855224
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,4,128,1,float16,fp8,15,0.16272640228271484
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,4,128,1,float16,fp8,31,0.16354559659957885
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,2,128,1,float16,float16,15,0.0829200029373169
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,4,128,1,float16,fp8,63,0.17267999649047852
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,4,128,1,float16,float16,127,0.15018559694290162
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,4,128,1,float16,fp8,127,0.1754240036010742
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,4,128,1,float16,float16,255,0.2046799898147583
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,16,8,128,1,float16,float16,15,0.01220960021018982
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,2,128,1,float16,fp8,127,0.10276639461517334
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,4,128,1,float16,float16,511,0.3160752058029175
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,4,128,1,float16,fp8,511,0.34794719219207765
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,8,128,1,float16,float16,1,0.24484798908233643
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,8,128,1,float16,fp8,1,0.2927680015563965
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,8,128,1,float16,float16,3,0.24419839382171632
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,8,128,1,float16,float16,7,0.24382879734039306
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,8,128,1,float16,fp8,7,0.29355359077453613
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,4,128,1,float16,float16,1,0.13870879411697387
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,4,128,1,float16,fp8,1,0.1622496008872986
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,8,128,1,float16,float16,15,0.24699039459228517
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,8,128,1,float16,fp8,15,0.29410240650177
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,8,128,1,float16,float16,31,0.25832960605621336
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,8,128,1,float16,float16,63,0.2619744062423706
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,8,128,1,float16,fp8,63,0.3153872013092041
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,8,128,1,float16,float16,127,0.2687407970428467
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,4,128,1,float16,float16,31,0.1426975965499878
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,8,128,1,float16,fp8,127,0.31653759479522703
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,8,128,1,float16,float16,255,0.37737600803375243
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,8,128,1,float16,fp8,255,0.42664318084716796
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,16,1,128,1,float16,float16,1,0.09475520253181458
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,8,128,1,float16,float16,511,0.5957744121551514
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,16,1,128,1,float16,fp8,1,0.10740799903869629
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,16,1,128,1,float16,float16,3,0.09467039704322815
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,4,128,1,float16,fp8,255,0.22743360996246337
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,8,128,1,float16,fp8,511,0.6587584018707275
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,16,1,128,1,float16,fp8,7,0.10714399814605713
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,16,1,128,1,float16,fp8,3,0.10699039697647095
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,16,1,128,1,float16,float16,31,0.095278400182724
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,16,1,128,1,float16,fp8,15,0.10665760040283204
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,16,1,128,1,float16,float16,63,0.1016271948814392
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,16,1,128,1,float16,fp8,63,0.1119488000869751
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,16,1,128,1,float16,float16,127,0.10588799715042115
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,8,128,1,float16,fp8,3,0.2947103977203369
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,16,1,128,1,float16,float16,255,0.13483200073242188
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,16,1,128,1,float16,fp8,255,0.14160000085830687
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,16,2,128,1,float16,float16,1,0.1492400050163269
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,16,2,128,1,float16,fp8,1,0.1722447991371155
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,16,2,128,1,float16,float16,3,0.14935679435729982
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,16,2,128,1,float16,fp8,3,0.1722815990447998
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,16,2,128,1,float16,float16,7,0.14958879947662354
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,8,128,1,float16,fp8,31,0.30763518810272217
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,16,2,128,1,float16,fp8,7,0.17216320037841798
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,16,2,128,1,float16,float16,15,0.1495584011077881
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,16,2,128,1,float16,fp8,15,0.17222880125045775
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,16,4,128,1,float16,float16,63,0.14451680183410645
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,16,2,128,1,float16,fp8,31,0.17473440170288085
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,16,2,128,1,float16,float16,63,0.15819840431213378
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,16,2,128,1,float16,fp8,63,0.18324639797210693
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,16,2,128,1,float16,float16,127,0.1635823965072632
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,16,2,128,1,float16,fp8,127,0.18609440326690674
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,16,2,128,1,float16,float16,255,0.2197727918624878
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,16,1,128,1,float16,float16,15,0.09565920233726502
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,16,2,128,1,float16,fp8,255,0.2393872022628784
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,16,1,128,1,float16,fp8,31,0.10700479745864869
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,16,4,128,1,float16,float16,1,0.26034400463104246
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,16,4,128,1,float16,fp8,1,0.307423996925354
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,16,4,128,1,float16,float16,3,0.26098880767822263
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,16,4,128,1,float16,float16,7,0.25985119342803953
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,16,1,128,1,float16,fp8,127,0.11489280462265014
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,16,4,128,1,float16,fp8,7,0.30730879306793213
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,16,4,128,1,float16,float16,15,0.2624736070632935
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,16,4,128,1,float16,fp8,15,0.30796959400177004
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,16,4,128,1,float16,float16,31,0.2678368091583252
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,16,4,128,1,float16,fp8,31,0.3221791982650757
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,16,4,128,1,float16,float16,63,0.2716223955154419
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,16,4,128,1,float16,fp8,63,0.3271440029144287
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,16,4,128,1,float16,float16,127,0.27727999687194826
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,16,4,128,1,float16,fp8,127,0.3303679943084717
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,16,4,128,1,float16,float16,255,0.38372800350189207
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,16,4,128,1,float16,fp8,255,0.43341917991638185
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,16,8,128,1,float16,float16,1,0.47042241096496584
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,16,2,128,1,float16,float16,31,0.15431360006332398
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,16,8,128,1,float16,fp8,1,0.5652912139892579
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,16,8,128,1,float16,float16,3,0.4725039958953857
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,16,8,128,1,float16,float16,7,0.48115038871765137
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,16,8,128,1,float16,fp8,3,0.5656367778778076
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,16,8,128,1,float16,fp8,7,0.5668928146362304
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,16,8,128,1,float16,fp8,15,0.5791120052337646
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,16,8,128,1,float16,float16,31,0.4954415798187256
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,16,1,128,1,float16,float16,7,0.0956719994544983
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,16,8,128,1,float16,float16,63,0.5029407978057862
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,16,8,128,1,float16,fp8,63,0.6081408023834228
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,16,8,128,1,float16,float16,127,0.5155456066131592
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,16,4,128,1,float16,fp8,3,0.30699520111083983
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,16,8,128,1,float16,fp8,127,0.6140528202056885
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,1,128,1,float16,float16,1,0.011742399632930755
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,1,128,1,float16,fp8,1,0.01255359947681427
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,1,128,1,float16,fp8,3,0.012399999797344208
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,1,128,1,float16,float16,7,0.011897599697113037
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,1,128,1,float16,float16,3,0.011788800358772278
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,16,8,128,1,float16,float16,255,0.7319744110107422
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,1,128,1,float16,float16,15,0.012036799639463424
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,1,128,1,float16,fp8,15,0.012694400548934937
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,1,128,1,float16,fp8,31,0.012556800246238708
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,1,128,1,float16,float16,31,0.012068799883127212
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,1,128,1,float16,float16,63,0.012113600224256515
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,1,128,1,float16,fp8,63,0.012478400021791458
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,1,128,1,float16,float16,127,0.012080000340938568
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,1,128,1,float16,fp8,127,0.012848000228404998
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,1,128,1,float16,fp8,255,0.014681600034236908
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,1,128,1,float16,float16,511,0.016782400012016297
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,1,128,1,float16,fp8,511,0.018409599363803864
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,1,128,1,float16,float16,1023,0.016358399391174318
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,1,128,1,float16,fp8,1023,0.017334400117397307
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,1,128,1,float16,float16,2047,0.0180976003408432
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,1,128,1,float16,fp8,2047,0.01783200055360794
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,1,128,1,float16,float16,4095,0.021611200273036958
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,1,128,1,float16,fp8,4095,0.021580800414085388
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,1,128,1,float16,float16,8191,0.03141599893569946
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,1,128,1,float16,float16,16383,0.046316799521446225
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,1,128,1,float16,fp8,16383,0.04261600077152252
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,2,128,1,float16,float16,1,0.011977600306272507
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,2,128,1,float16,fp8,1,0.012945599853992462
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,2,128,1,float16,float16,3,0.011972799897193909
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,2,128,1,float16,fp8,3,0.012827199697494508
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,2,128,1,float16,float16,7,0.011953599750995636
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,2,128,1,float16,fp8,7,0.01281919926404953
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,2,128,1,float16,float16,15,0.011956799775362015
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,2,128,1,float16,fp8,15,0.012780800461769104
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,2,128,1,float16,float16,31,0.012084800004959106
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,2,128,1,float16,fp8,31,0.012620800733566284
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,16,8,128,1,float16,float16,15,0.49286718368530275
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,2,128,1,float16,fp8,63,0.01271200031042099
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,2,128,1,float16,float16,127,0.012321600317955017
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,2,128,1,float16,float16,255,0.014060799777507783
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,2,128,1,float16,fp8,255,0.014982399344444276
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,2,128,1,float16,float16,511,0.017100800573825837
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,2,128,1,float16,fp8,511,0.018561600148677825
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,2,128,1,float16,float16,1023,0.017851200699806214
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,2,128,1,float16,fp8,1023,0.01775040030479431
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,2,128,1,float16,float16,2047,0.019944000244140624
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,2,128,1,float16,fp8,2047,0.02141759991645813
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,2,128,1,float16,float16,4095,0.03102239966392517
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,2,128,1,float16,fp8,4095,0.027296000719070436
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,2,128,1,float16,float16,8191,0.04555999934673309
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,1,128,1,float16,fp8,7,0.01239520013332367
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,2,128,1,float16,fp8,8191,0.042203199863433835
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,2,128,1,float16,float16,16383,0.07776479721069336
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,2,128,1,float16,fp8,16383,0.06672160029411316
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,4,128,1,float16,float16,1,0.012100800126791
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,4,128,1,float16,fp8,1,0.013124799728393555
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,4,128,1,float16,float16,3,0.012177599966526032
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,16,8,128,1,float16,fp8,255,0.826694393157959
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,4,128,1,float16,float16,7,0.012150400131940842
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,4,128,1,float16,fp8,7,0.012905600666999816
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,4,128,1,float16,float16,15,0.012177599966526032
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,4,128,1,float16,fp8,15,0.012873600423336028
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,1,128,1,float16,fp8,8191,0.027620801329612733
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,4,128,1,float16,float16,31,0.012174399942159653
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,4,128,1,float16,fp8,31,0.012809599936008453
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,4,128,1,float16,fp8,63,0.01284639984369278
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,4,128,1,float16,float16,127,0.012358400225639343
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,4,128,1,float16,fp8,127,0.013094399869441987
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,4,128,1,float16,float16,255,0.014110399782657624
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,4,128,1,float16,fp8,255,0.015184000134468079
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,2,128,1,float16,float16,63,0.012191999703645706
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,4,128,1,float16,fp8,511,0.018617600202560425
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,4,128,1,float16,float16,1023,0.01950239986181259
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,4,128,1,float16,fp8,1023,0.02102559953927994
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,16,8,128,1,float16,fp8,31,0.6072815895080567
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,4,128,1,float16,fp8,2047,0.02688640058040619
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,4,128,1,float16,float16,4095,0.04559679925441742
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,4,128,1,float16,fp8,4095,0.041308799386024476
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,4,128,1,float16,float16,8191,0.0763808012008667
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,4,128,1,float16,fp8,8191,0.06624320149421692
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,4,128,1,float16,float16,16383,0.13482719659805298
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,1,128,1,float16,float16,255,0.013663999736309052
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,4,128,1,float16,fp8,16383,0.11543680429458618
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,8,128,1,float16,float16,1,0.012291199713945388
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,8,128,1,float16,fp8,1,0.012868799269199371
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,8,128,1,float16,fp8,3,0.012839999794960023
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,8,128,1,float16,float16,7,0.012033600360155106
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,8,128,1,float16,fp8,7,0.012984000146389008
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,4,128,1,float16,float16,63,0.01231200024485588
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,8,128,1,float16,float16,15,0.012254399806261062
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,8,128,1,float16,fp8,15,0.013145600259304047
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,8,128,1,float16,float16,31,0.012251199781894683
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,8,128,1,float16,fp8,31,0.013184000551700593
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,8,128,1,float16,float16,63,0.01207360029220581
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,8,128,1,float16,fp8,63,0.013220800459384919
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,8,128,1,float16,float16,127,0.012167999893426895
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,8,128,1,float16,fp8,127,0.013228799402713775
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,8,128,1,float16,float16,255,0.013984000682830811
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,8,128,1,float16,fp8,255,0.015097600221633912
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,4,128,1,float16,float16,511,0.017241600155830383
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,8,128,1,float16,fp8,511,0.01876160055398941
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,2,128,1,float16,fp8,127,0.012918399274349212
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,8,128,1,float16,fp8,1023,0.026070401072502136
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,8,128,1,float16,float16,2047,0.04394719898700714
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,8,128,1,float16,fp8,2047,0.04057919979095459
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,8,128,1,float16,float16,4095,0.07458879947662353
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,4,128,1,float16,float16,2047,0.0302592009305954
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,8,128,1,float16,float16,8191,0.1342479944229126
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,8,128,1,float16,fp8,8191,0.11620639562606812
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,8,128,1,float16,fp8,16383,0.21364960670471192
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,1,128,1,float16,float16,1,0.013184000551700593
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,1,128,1,float16,float16,3,0.013219200074672699
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,1,128,1,float16,fp8,3,0.013955199718475341
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,1,128,1,float16,float16,7,0.013236799836158752
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,1,128,1,float16,fp8,7,0.014027200639247894
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,1,128,1,float16,float16,15,0.013246400654315949
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,1,128,1,float16,fp8,15,0.013843199610710144
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,1,128,1,float16,float16,31,0.013424000144004822
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,8,128,1,float16,float16,3,0.012321600317955017
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,1,128,1,float16,fp8,31,0.01406240016222
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,1,128,1,float16,float16,63,0.013308799266815186
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,1,128,1,float16,fp8,63,0.014259199798107147
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,1,128,1,float16,float16,127,0.013449600338935852
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,1,128,1,float16,float16,255,0.01510079950094223
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,1,128,1,float16,fp8,255,0.016038399934768677
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,1,128,1,float16,float16,511,0.01804320067167282
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,1,128,1,float16,fp8,511,0.019556799530982973
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,1,128,1,float16,float16,1023,0.020681600272655486
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,1,128,1,float16,fp8,1023,0.022256000339984892
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,8,128,1,float16,float16,511,0.017312000691890716
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,1,128,1,float16,float16,2047,0.032287999987602234
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,8,128,1,float16,float16,1023,0.029337599873542786
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,1,128,1,float16,fp8,2047,0.028630399703979494
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,1,128,1,float16,float16,4095,0.04693920016288757
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,1,128,1,float16,fp8,4095,0.04224640130996704
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,1,128,1,float16,float16,8191,0.07791360020637512
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,1,128,1,float16,fp8,8191,0.06729440093040466
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,8,128,1,float16,fp8,4095,0.06581119894981384
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,1,128,1,float16,float16,16383,0.13671200275421141
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,1,128,1,float16,fp8,16383,0.11765600442886352
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,2,128,1,float16,float16,1,0.01329600065946579
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,2,128,1,float16,fp8,1,0.01443679928779602
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,2,128,1,float16,float16,3,0.013335999846458436
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,2,128,1,float16,fp8,3,0.014387199282646179
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,1,128,1,float16,fp8,32767,0.2161184072494507
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,2,128,1,float16,fp8,7,0.014369599521160126
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,2,128,1,float16,float16,15,0.013411200046539307
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,2,128,1,float16,fp8,15,0.014260800182819366
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,2,128,1,float16,float16,31,0.013628800213336945
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,2,128,1,float16,fp8,31,0.014212800562381745
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,2,128,1,float16,float16,63,0.013315199315547943
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,2,128,1,float16,fp8,63,0.014302399754524232
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,1,128,1,float16,fp8,1,0.01420000046491623
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,2,128,1,float16,fp8,127,0.014416000247001648
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,2,128,1,float16,float16,255,0.01539359986782074
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,2,128,1,float16,fp8,255,0.0165120005607605
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,2,128,1,float16,float16,511,0.01865759938955307
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,2,128,1,float16,fp8,511,0.020137600600719452
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,2,128,1,float16,float16,1023,0.030318400263786315
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,4,128,1,float16,fp8,3,0.012996800243854523
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,2,128,1,float16,float16,2047,0.045351999998092654
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,2,128,1,float16,fp8,2047,0.04136320054531097
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,1,128,1,float16,fp8,127,0.01427839994430542
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,2,128,1,float16,float16,4095,0.07653120160102844
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,2,128,1,float16,fp8,4095,0.06645439863204956
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,2,128,1,float16,float16,8191,0.13633919954299928
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,2,128,1,float16,fp8,8191,0.11604959964752197
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,2,128,1,float16,float16,16383,0.25647358894348143
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,2,128,1,float16,fp8,16383,0.21309120655059816
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,2,128,1,float16,float16,32767,0.49805917739868166
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,2,128,1,float16,fp8,32767,0.40635042190551757
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,2,128,1,float16,float16,7,0.013467200100421906
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,4,128,1,float16,float16,1,0.018185600638389587
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,4,128,1,float16,fp8,3,0.019631999731063842
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,4,128,1,float16,float16,7,0.018137599527835845
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,2,128,1,float16,float16,127,0.0134320005774498
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,4,128,1,float16,float16,15,0.017195199429988862
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,4,128,1,float16,fp8,15,0.018654400110244752
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,4,128,1,float16,float16,31,0.017067199945449828
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,4,128,1,float16,fp8,31,0.018624000251293182
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,4,128,1,float16,float16,63,0.017127999663352968
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,4,128,1,float16,fp8,63,0.01855199933052063
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,4,128,1,float16,float16,127,0.017105600237846373
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,4,128,1,float16,fp8,127,0.018639999628067016
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,4,128,1,float16,float16,255,0.020683200657367708
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,4,128,1,float16,fp8,255,0.022555199265480042
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,4,128,1,float16,float16,511,0.031147199869155883
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,4,128,1,float16,fp8,511,0.030089598894119263
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,4,128,1,float16,float16,1023,0.04841119945049286
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,4,128,1,float16,fp8,1023,0.045824000239372255
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,4,128,1,float16,float16,2047,0.0793727993965149
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,4,128,1,float16,fp8,2047,0.06945760250091552
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,4,128,1,float16,float16,4095,0.1400864005088806
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,1,128,1,float16,float16,32767,0.25756800174713135
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,4,128,1,float16,fp8,4095,0.11924159526824951
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,4,128,1,float16,float16,3,0.017392000555992125
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,16,8,128,1,float16,float16,16383,0.2538464069366455
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,4,128,1,float16,fp8,8191,0.21897759437561035
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,4,128,1,float16,fp8,7,0.019407999515533448
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,1,128,1,float16,float16,1,0.011519999802112579
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,1,128,1,float16,fp8,1,0.012167999893426895
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,4,128,1,float16,fp8,16383,0.41073760986328123
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,2,128,1,float16,fp8,1023,0.02746880054473877
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,1,128,1,float16,float16,3,0.011310400068759918
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,1,128,1,float16,fp8,7,0.012027200311422348
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,1,128,1,float16,fp8,3,0.01207360029220581
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,1,128,1,float16,float16,15,0.011435200273990632
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,1,128,1,float16,fp8,15,0.012089599668979645
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,1,128,1,float16,float16,31,0.011563199758529662
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,1,128,1,float16,fp8,31,0.01213119998574257
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,1,128,1,float16,float16,63,0.011552000045776367
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,1,128,1,float16,fp8,63,0.012193600088357926
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,4,128,1,float16,float16,32767,1.0028656005859375
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,4,128,1,float16,fp8,32767,0.8028016090393066
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,1,128,1,float16,fp8,255,0.014372800290584565
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,1,128,1,float16,float16,511,0.01618559956550598
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,1,128,1,float16,fp8,511,0.018588800728321076
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,1,128,1,float16,float16,1023,0.01574240028858185
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,1,128,1,float16,fp8,1023,0.017030400037765504
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,1,128,1,float16,float16,2047,0.015831999480724335
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,1,128,1,float16,fp8,2047,0.01712159961462021
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,1,128,1,float16,float16,4095,0.016840000450611115
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,1,128,1,float16,fp8,4095,0.01758880019187927
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,1,128,1,float16,float16,8191,0.01813279986381531
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,1,128,1,float16,fp8,8191,0.018641600012779237
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,1,128,1,float16,float16,16383,0.0206496000289917
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,4,128,1,float16,fp8,1,0.01852000057697296
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,1,128,1,float16,fp8,16383,0.02024320065975189
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,1,128,1,float16,float16,32767,0.023652799427509308
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,4,128,1,float16,float16,8191,0.256606388092041
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,2,128,1,float16,float16,1,0.011395200341939925
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,2,128,1,float16,float16,3,0.011294399946928024
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,2,128,1,float16,fp8,1,0.012068799883127212
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,2,128,1,float16,fp8,3,0.012142399698495865
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,2,128,1,float16,float16,7,0.011257600039243698
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,2,128,1,float16,fp8,7,0.012212800234556198
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,2,128,1,float16,float16,15,0.011505600064992905
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,2,128,1,float16,fp8,15,0.01234399974346161
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,2,128,1,float16,float16,31,0.011444800347089768
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,2,128,1,float16,fp8,31,0.012316799908876418
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,2,128,1,float16,float16,63,0.01141119971871376
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,2,128,1,float16,fp8,63,0.012334399670362473
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,2,128,1,float16,float16,127,0.011343999952077865
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,2,128,1,float16,fp8,127,0.012296000123023986
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,2,128,1,float16,float16,255,0.012958399951457977
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,1,128,1,float16,float16,7,0.011275199800729751
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,2,128,1,float16,float16,511,0.015910400450229643
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,2,128,1,float16,fp8,511,0.018318399786949158
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,2,128,1,float16,float16,1023,0.015406399965286255
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,2,128,1,float16,fp8,1023,0.01680160015821457
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,2,128,1,float16,float16,2047,0.015905599296092986
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,1,128,1,float16,float16,127,0.011398400366306304
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,2,128,1,float16,fp8,2047,0.01712159961462021
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,1,128,1,float16,fp8,127,0.012240000069141388
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,2,128,1,float16,fp8,4095,0.017390400171279907
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,2,128,1,float16,float16,8191,0.017739200592041017
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,2,128,1,float16,fp8,8191,0.01867839992046356
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,2,128,1,float16,float16,16383,0.02024639993906021
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,2,128,1,float16,fp8,16383,0.020483200252056123
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,2,128,1,float16,float16,32767,0.024160000681877136
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,1,128,1,float16,float16,255,0.015240000188350677
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,2,128,1,float16,fp8,32767,0.024035200476646423
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,4,128,1,float16,float16,1,0.011468800157308579
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,4,128,1,float16,fp8,1,0.012303999811410903
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,4,128,1,float16,fp8,3,0.012147200107574464
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,4,128,1,float16,float16,7,0.011547199636697768
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,4,128,1,float16,fp8,7,0.012150400131940842
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,4,128,1,float16,float16,15,0.011459200084209442
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,4,128,1,float16,fp8,15,0.012246400117874146
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,4,128,1,float16,float16,31,0.011457599699497223
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,4,128,1,float16,fp8,31,0.012324800342321396
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,4,128,1,float16,float16,63,0.011444800347089768
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,4,128,1,float16,fp8,63,0.012303999811410903
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,4,128,1,float16,float16,127,0.011483199894428253
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,4,128,1,float16,fp8,127,0.012329600006341934
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,4,128,1,float16,float16,255,0.013076800107955932
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,4,128,1,float16,fp8,255,0.0143327996134758
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,4,128,1,float16,float16,511,0.0161408007144928
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,4,128,1,float16,fp8,511,0.018225599825382233
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,4,128,1,float16,float16,1023,0.01579039990901947
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,4,128,1,float16,fp8,1023,0.01666239947080612
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,4,128,1,float16,fp8,2047,0.017047999799251555
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,4,128,1,float16,float16,4095,0.01690399944782257
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,4,128,1,float16,fp8,4095,0.017628799378871917
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,4,128,1,float16,float16,8191,0.01897439956665039
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,1,128,1,float16,fp8,32767,0.024667200446128846
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,4,128,1,float16,fp8,8191,0.018665599822998046
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,4,128,1,float16,float16,16383,0.02266079932451248
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,4,128,1,float16,fp8,16383,0.02240000069141388
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,4,128,1,float16,float16,32767,0.03223040103912354
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,4,128,1,float16,fp8,32767,0.03067840039730072
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,1,128,1,float16,float16,1,0.011417599767446518
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,1,128,1,float16,fp8,1,0.012160000205039979
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,8,4,128,1,float16,float16,16383,0.4943664073944092
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,1,128,1,float16,fp8,3,0.01215519979596138
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,1,128,1,float16,fp8,7,0.01215839982032776
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,1,128,1,float16,float16,7,0.011519999802112579
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,1,128,1,float16,float16,15,0.011404799669981003
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,1,128,1,float16,float16,31,0.011448000371456147
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,1,128,1,float16,fp8,15,0.012353599816560746
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,1,128,1,float16,fp8,31,0.012137600034475327
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,1,128,1,float16,float16,63,0.011484800279140473
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,1,128,1,float16,fp8,63,0.011990399658679962
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,1,128,1,float16,float16,127,0.01149279996752739
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,1,128,1,float16,fp8,127,0.011955200135707856
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,1,128,1,float16,float16,255,0.013075199723243714
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,1,128,1,float16,fp8,255,0.014022399485111237
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,1,128,1,float16,float16,511,0.016126400232315062
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,1,128,1,float16,fp8,511,0.017953599989414214
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,1,128,1,float16,float16,1023,0.01560640037059784
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,1,128,1,float16,float16,2047,0.015995199978351592
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,1,128,1,float16,fp8,2047,0.01717599928379059
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,1,128,1,float16,float16,4095,0.016700799763202667
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,4,128,1,float16,float16,3,0.011308799684047698
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,1,128,1,float16,fp8,4095,0.01743520051240921
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,1,128,1,float16,float16,8191,0.018083199858665466
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,1,128,1,float16,fp8,8191,0.019152000546455383
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,1,128,1,float16,float16,16383,0.020526400208473204
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,1,128,1,float16,fp8,16383,0.0211776003241539
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,1,128,1,float16,float16,32767,0.024583999812602998
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,1,128,1,float16,fp8,32767,0.0253711998462677
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,2,128,1,float16,float16,1,0.011523199826478958
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,2,128,1,float16,float16,3,0.011556799709796905
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,2,128,1,float16,fp8,3,0.012324800342321396
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,2,128,1,float16,float16,7,0.011308799684047698
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,4,128,1,float16,float16,2047,0.01610559970140457
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,2,128,1,float16,fp8,7,0.012140800058841706
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,2,128,1,float16,float16,15,0.011548800021409988
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,2,128,1,float16,float16,31,0.011598400026559829
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,2,128,1,float16,float16,63,0.011528000235557556
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,2,128,1,float16,fp8,63,0.012176000326871873
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,2,128,1,float16,float16,127,0.011497599631547928
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,2,128,1,float16,fp8,127,0.012307199835777282
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,2,128,1,float16,float16,255,0.013187199831008911
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,2,128,1,float16,fp8,255,0.014270399510860444
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,1,128,1,float16,float16,3,0.011416000127792359
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,2,128,1,float16,fp8,511,0.01810240000486374
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,2,128,1,float16,float16,1023,0.01573439985513687
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,2,128,1,float16,fp8,1023,0.0168272003531456
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,2,128,1,float16,float16,2047,0.016128000617027283
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,2,128,1,float16,fp8,2047,0.01725119948387146
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,2,128,1,float16,float16,4095,0.017041599750518797
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,2,128,1,float16,fp8,4095,0.01761920005083084
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,2,128,1,float16,float16,4095,0.016777600347995757
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,2,128,1,float16,float16,8191,0.019097599387168884
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,1,128,1,float16,fp8,1023,0.016841599345207216
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,2,128,1,float16,float16,16383,0.022750400006771088
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,2,128,1,float16,fp8,16383,0.02264160066843033
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,2,128,1,float16,float16,32767,0.03267999887466431
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,2,128,1,float16,fp8,32767,0.02890079915523529
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,4,128,1,float16,float16,1,0.01168000027537346
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,4,128,1,float16,float16,3,0.011681599915027619
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,4,128,1,float16,fp8,3,0.012392000108957291
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,4,128,1,float16,float16,7,0.011612799763679505
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,4,128,1,float16,fp8,7,0.01231039986014366
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,4,128,1,float16,float16,15,0.01159999966621399
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,2,128,1,float16,fp8,1,0.01231359988451004
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,4,128,1,float16,fp8,15,0.012068799883127212
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,4,128,1,float16,float16,31,0.011535999923944473
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,4,128,1,float16,fp8,31,0.01218239963054657
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,4,128,1,float16,float16,63,0.01151840016245842
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,4,128,1,float16,fp8,63,0.012254399806261062
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,2,128,1,float16,fp8,15,0.012214399874210358
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,4,128,1,float16,float16,127,0.01165120005607605
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,4,128,1,float16,fp8,127,0.012115199863910676
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,4,128,1,float16,float16,255,0.01329279989004135
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,2,128,1,float16,fp8,31,0.012025599926710128
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,4,128,1,float16,float16,511,0.016235199570655823
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,4,128,1,float16,fp8,511,0.017723199725151063
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,4,128,1,float16,float16,1023,0.015771199762821198
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,4,128,1,float16,fp8,1023,0.016756799817085267
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,4,128,1,float16,fp8,2047,0.017321600019931792
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,4,128,1,float16,float16,4095,0.018136000633239745
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,4,128,1,float16,fp8,4095,0.01791999936103821
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,4,128,1,float16,float16,8191,0.021568000316619873
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,4,128,1,float16,fp8,8191,0.021457600593566894
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,8,2,128,1,float16,fp8,255,0.014227199554443359
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,4,128,1,float16,float16,16383,0.03102400004863739
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,4,128,1,float16,fp8,16383,0.027665600180625916
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,4,128,1,float16,float16,32767,0.047547200322151185
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,4,128,1,float16,fp8,32767,0.043329599499702456
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,1,128,1,float16,fp8,1,0.01308639943599701
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,1,128,1,float16,float16,3,0.01234399974346161
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,2,128,1,float16,fp8,8191,0.019152000546455383
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,1,128,1,float16,fp8,3,0.013230399787425995
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,1,128,1,float16,float16,7,0.012656000256538392
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,1,128,1,float16,float16,15,0.012627199292182922
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,1,128,1,float16,fp8,15,0.013246400654315949
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,4,128,1,float16,fp8,1,0.012268800288438797
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,1,128,1,float16,fp8,31,0.013329599797725678
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,1,128,1,float16,float16,63,0.01266240030527115
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,1,128,1,float16,fp8,63,0.013128000497817992
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,1,128,1,float16,float16,127,0.012779200077056884
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,1,128,1,float16,fp8,127,0.013332800567150116
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,1,128,1,float16,float16,255,0.014271999895572662
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,1,128,1,float16,fp8,255,0.015219199657440185
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,1,128,1,float16,float16,511,0.017523199319839478
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,1,128,1,float16,fp8,511,0.019223999977111817
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,1,128,1,float16,float16,1023,0.018492799997329713
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,1,128,1,float16,fp8,1023,0.01815200001001358
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,1,128,1,float16,float16,2047,0.020318399369716644
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,4,128,1,float16,float16,2047,0.016281600296497344
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,1,128,1,float16,fp8,2047,0.021700799465179443
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,1,128,1,float16,float16,4095,0.03152480125427246
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,1,128,1,float16,fp8,4095,0.02820959985256195
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,1,128,1,float16,float16,8191,0.0463375985622406
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,1,128,1,float16,fp8,8191,0.04163999855518341
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,1,128,1,float16,float16,16383,0.0769536018371582
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,1,128,1,float16,fp8,16383,0.06695680022239685
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,1,128,1,float16,float16,32767,0.1363360047340393
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,1,128,1,float16,fp8,32767,0.11716480255126953
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,2,128,1,float16,float16,1,0.012937599420547485
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,2,128,1,float16,fp8,1,0.0133775994181633
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,2,128,1,float16,float16,3,0.012750400602817536
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,2,128,1,float16,fp8,3,0.013462400436401368
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,2,128,1,float16,float16,7,0.012785600125789642
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,2,128,1,float16,fp8,7,0.013350400328636169
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,2,128,1,float16,float16,15,0.012732799351215362
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,2,128,1,float16,fp8,15,0.013281600177288055
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,2,128,1,float16,float16,31,0.012716799974441528
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,2,128,1,float16,fp8,31,0.013355199992656708
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,2,128,1,float16,float16,63,0.012724800407886505
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,2,128,1,float16,fp8,63,0.01356160044670105
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,2,128,1,float16,float16,127,0.01286720037460327
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,2,128,1,float16,fp8,127,0.013577599823474885
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,2,128,1,float16,float16,255,0.014628799259662628
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,1,128,1,float16,float16,31,0.012415999919176102
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,2,128,1,float16,fp8,511,0.01926559954881668
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,2,128,1,float16,fp8,1023,0.02181600034236908
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,2,128,1,float16,float16,2047,0.031358399987220766
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,2,128,1,float16,fp8,2047,0.027820798754692077
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,2,128,1,float16,float16,4095,0.046054399013519286
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,2,128,1,float16,fp8,4095,0.042454400658607484
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,2,128,1,float16,float16,8191,0.07627999782562256
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,4,128,1,float16,fp8,255,0.01406240016222
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,2,128,1,float16,float16,16383,0.13752959966659545
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,2,128,1,float16,fp8,16383,0.11634240150451661
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,8,2,128,1,float16,float16,511,0.016150400042533875
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,2,128,1,float16,float16,32767,0.25570080280303953
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,2,128,1,float16,fp8,32767,0.21481759548187257
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,4,128,1,float16,float16,1,0.01290079951286316
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,4,128,1,float16,fp8,1,0.013409599661827087
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,4,128,1,float16,float16,3,0.012670400738716125
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,4,128,1,float16,fp8,3,0.013374400138854981
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,4,128,1,float16,float16,7,0.0124208003282547
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,4,128,1,float16,fp8,7,0.013393600285053254
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,4,128,1,float16,fp8,15,0.01356319934129715
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,1,128,1,float16,fp8,7,0.01329279989004135
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,4,128,1,float16,fp8,31,0.013551999628543854
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,4,128,1,float16,float16,63,0.012721599638462066
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,2,128,1,float16,float16,511,0.017267200350761413
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,4,128,1,float16,fp8,63,0.013579200208187103
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,2,128,1,float16,float16,1023,0.020193600654602052
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,4,128,1,float16,fp8,127,0.013521599769592284
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,4,128,1,float16,float16,255,0.014697599411010741
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,4,128,1,float16,float16,511,0.017745600640773775
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,4,128,1,float16,fp8,511,0.01942239999771118
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,4,128,1,float16,float16,1023,0.02951520085334778
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,2,128,1,float16,fp8,8191,0.06685760021209716
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,4,128,1,float16,fp8,1023,0.026807999610900878
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,4,128,1,float16,float16,2047,0.04471360146999359
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,4,128,1,float16,fp8,2047,0.04042559862136841
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,4,128,1,float16,float16,4095,0.07497280240058898
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,4,128,1,float16,fp8,4095,0.06577919721603394
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,4,128,1,float16,float16,8191,0.13399200439453124
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,4,128,1,float16,fp8,8191,0.11588159799575806
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,4,128,1,float16,float16,15,0.012492799758911132
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,4,128,1,float16,float16,16383,0.25301599502563477
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,4,128,1,float16,fp8,16383,0.21291520595550537
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,2,128,1,float16,fp8,255,0.015596799552440643
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,1,128,1,float16,float16,1,0.011456000059843064
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,1,128,1,float16,fp8,1,0.012080000340938568
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,4,128,1,float16,fp8,255,0.015358400344848634
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,1,128,1,float16,float16,3,0.011363200098276138
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,1,128,1,float16,fp8,3,0.012216000258922577
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,1,128,1,float16,float16,7,0.011360000073909759
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,4,128,1,float16,float16,32767,0.4914207935333252
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,1,128,1,float16,fp8,7,0.012368000298738479
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,1,128,1,float16,float16,31,0.011606399714946748
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,4,128,1,float16,fp8,32767,0.4063680171966553
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,1,128,1,float16,fp8,31,0.012390399724245072
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,1,128,1,float16,fp8,63,0.012191999703645706
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,1,128,1,float16,float16,63,0.01162080019712448
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,1,128,1,float16,fp8,127,0.01225920021533966
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,1,128,1,float16,float16,255,0.013060800731182098
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,1,128,1,float16,fp8,255,0.014180800318717957
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,1,128,1,float16,float16,511,0.016100800037384032
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,1,128,1,float16,fp8,511,0.01812639981508255
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,1,128,1,float16,float16,1023,0.015851199626922607
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,1,128,1,float16,fp8,1023,0.017071999609470367
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,1,128,1,float16,float16,2047,0.01605760008096695
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,1,128,1,float16,fp8,2047,0.017312000691890716
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,1,128,1,float16,float16,4095,0.017003199458122252
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,1,128,1,float16,float16,1,0.012636800110340119
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,1,128,1,float16,fp8,4095,0.017612800002098083
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,1,128,1,float16,float16,8191,0.019156800210475923
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,1,128,1,float16,fp8,8191,0.01897599995136261
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,1,128,1,float16,float16,16383,0.022808000445365906
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,1,128,1,float16,fp8,16383,0.022708800435066224
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,4,128,1,float16,float16,31,0.01271360069513321
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,1,128,1,float16,float16,32767,0.03266400098800659
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,1,128,1,float16,fp8,32767,0.029104000329971312
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,2,128,1,float16,fp8,1,0.012169600278139115
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,2,128,1,float16,float16,3,0.011601600050926208
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,2,128,1,float16,float16,7,0.011462400108575821
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,8,4,128,1,float16,float16,127,0.012676799297332763
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,2,128,1,float16,fp8,7,0.012246400117874146
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,2,128,1,float16,float16,15,0.011462400108575821
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,2,128,1,float16,fp8,15,0.01223680004477501
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,2,128,1,float16,fp8,31,0.012452799826860428
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,1,128,1,float16,float16,15,0.011342400312423706
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,2,128,1,float16,float16,63,0.011615999788045884
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,2,128,1,float16,fp8,63,0.012366399914026261
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,2,128,1,float16,float16,127,0.011655999720096588
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,2,128,1,float16,fp8,127,0.012347199767827988
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,2,128,1,float16,float16,255,0.013286399841308593
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,2,128,1,float16,fp8,255,0.014321599900722504
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,1,128,1,float16,float16,127,0.011401599645614624
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,2,128,1,float16,fp8,511,0.018331199884414673
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,2,128,1,float16,float16,1023,0.015782399475574492
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,2,128,1,float16,fp8,1023,0.017051200568675994
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,2,128,1,float16,float16,2047,0.016225600242614747
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,2,128,1,float16,fp8,2047,0.0171984001994133
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,2,128,1,float16,fp8,4095,0.01810719966888428
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,2,128,1,float16,float16,8191,0.02146880030632019
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,2,128,1,float16,fp8,8191,0.021844799816608428
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,2,128,1,float16,float16,16383,0.03167519867420197
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,2,128,1,float16,fp8,16383,0.027968001365661622
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,2,128,1,float16,float16,32767,0.04734399914741516
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,2,128,1,float16,fp8,32767,0.04428800046443939
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,4,128,1,float16,float16,1,0.011684799939393998
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,4,128,1,float16,fp8,1,0.012377600371837615
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,4,128,1,float16,float16,3,0.011664000153541566
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,4,128,1,float16,fp8,3,0.01223360002040863
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,4,128,1,float16,float16,7,0.011667200177907944
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,4,128,1,float16,fp8,7,0.012251199781894683
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,4,128,1,float16,float16,15,0.011547199636697768
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,4,128,1,float16,fp8,15,0.01225920021533966
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,4,128,1,float16,float16,31,0.01154400035738945
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,4,128,1,float16,fp8,31,0.012247999757528305
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,4,128,1,float16,fp8,63,0.012249600142240524
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,4,128,1,float16,float16,127,0.01170559972524643
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,4,128,1,float16,fp8,127,0.012433599680662155
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,4,128,1,float16,float16,255,0.013279999792575835
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,4,128,1,float16,fp8,255,0.014483200013637542
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,2,128,1,float16,float16,1,0.011601600050926208
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,2,128,1,float16,fp8,3,0.012123200297355651
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,4,128,1,float16,float16,1023,0.016014400124549865
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,4,128,1,float16,fp8,1023,0.016888000071048737
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,2,128,1,float16,float16,31,0.011427199840545655
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,4,128,1,float16,fp8,2047,0.01770720034837723
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,4,128,1,float16,float16,4095,0.021331200003623964
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,1,128,1,float16,fp8,15,0.012368000298738479
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,4,128,1,float16,fp8,4095,0.02165919989347458
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,4,128,1,float16,float16,8191,0.03020159900188446
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,2,128,1,float16,float16,511,0.01634240001440048
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,4,128,1,float16,float16,16383,0.04514560103416443
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,4,128,1,float16,fp8,16383,0.04131839871406555
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,4,128,1,float16,float16,32767,0.07572960257530212
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,2,128,1,float16,float16,4095,0.018219199776649476
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,1,128,1,float16,float16,1,0.014584000408649444
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,1,128,1,float16,fp8,1,0.015548799932003022
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,1,128,1,float16,float16,3,0.014636799693107605
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,1,128,1,float16,fp8,3,0.01552799940109253
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,1,128,1,float16,float16,7,0.014660799503326416
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,1,128,1,float16,fp8,7,0.01552799940109253
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,1,128,1,float16,float16,15,0.014668799936771393
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,1,128,1,float16,fp8,15,0.015430399775505066
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,1,128,1,float16,float16,31,0.014519999921321868
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,4,128,1,float16,float16,63,0.011572799831628799
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,1,128,1,float16,fp8,31,0.015548799932003022
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,1,128,1,float16,float16,63,0.01510239988565445
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,1,128,1,float16,fp8,63,0.015627199411392213
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,1,128,1,float16,float16,127,0.015249599516391755
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,1,128,1,float16,fp8,127,0.015678399801254274
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,1,128,1,float16,float16,255,0.016804799437522888
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,1,128,1,float16,fp8,255,0.0175135999917984
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,1,128,1,float16,float16,511,0.02040639966726303
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,1,128,1,float16,fp8,511,0.021158400177955627
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,1,128,1,float16,float16,1023,0.03336319923400879
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,4,128,1,float16,float16,511,0.016129599511623384
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,1,128,1,float16,fp8,1023,0.028620800375938414
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,4,128,1,float16,fp8,511,0.018300800025463103
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,1,128,1,float16,float16,2047,0.04774560034275055
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,1,128,1,float16,float16,4095,0.07822399735450744
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,1,128,1,float16,fp8,4095,0.06724640130996704
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,1,128,1,float16,float16,8191,0.1375200033187866
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,4,128,1,float16,float16,2047,0.017712000012397765
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,1,128,1,float16,fp8,8191,0.1175887942314148
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,2,128,1,float16,float16,1,0.01858399957418442
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,2,128,1,float16,fp8,1,0.01987680047750473
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,2,128,1,float16,float16,3,0.018731200695037843
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,2,128,1,float16,fp8,3,0.019843199849128725
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,2,128,1,float16,float16,7,0.018727999925613404
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,2,128,1,float16,fp8,7,0.019940799474716185
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,2,128,1,float16,float16,15,0.018544000387191773
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,2,128,1,float16,fp8,15,0.019974400103092194
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,2,128,1,float16,float16,31,0.01899999976158142
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,2,128,1,float16,fp8,31,0.020044800639152528
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,2,128,1,float16,float16,63,0.018721599876880646
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,2,128,1,float16,fp8,63,0.019974400103092194
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,2,128,1,float16,float16,127,0.01870400011539459
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,2,128,1,float16,fp8,127,0.020020799338817598
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,2,128,1,float16,float16,255,0.02202560007572174
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,4,128,1,float16,fp8,32767,0.06661760210990905
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,2,128,1,float16,float16,511,0.03291040062904358
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,2,128,1,float16,fp8,511,0.031809601187705996
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,2,128,1,float16,float16,1023,0.05050560235977173
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,2,128,1,float16,fp8,1023,0.04797919988632202
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,2,128,1,float16,fp8,2047,0.07093440294265747
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,2,128,1,float16,float16,4095,0.13921120166778564
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,2,128,1,float16,fp8,4095,0.12106879949569702
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,2,128,1,float16,float16,8191,0.2569535970687866
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,1,128,1,float16,fp8,2047,0.042368000745773314
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,2,128,1,float16,fp8,8191,0.22058401107788086
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,1,128,1,float16,float16,16383,0.26066880226135253
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,4,128,1,float16,float16,1,0.026081600785255434
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,2,128,1,float16,float16,16383,0.49421119689941406
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,2,128,1,float16,fp8,16383,0.41461758613586425
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,8,4,128,1,float16,fp8,8191,0.02760159969329834
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,4,128,1,float16,fp8,3,0.029204800724983215
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,4,128,1,float16,float16,7,0.026316800713539125
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,4,128,1,float16,float16,15,0.02600800096988678
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,4,128,1,float16,fp8,15,0.029142400622367857
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,4,128,1,float16,float16,31,0.026009601354599
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,4,128,1,float16,fp8,31,0.029104000329971312
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,4,128,1,float16,float16,63,0.025886398553848267
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,2,128,1,float16,fp8,255,0.02390560060739517
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,4,128,1,float16,fp8,63,0.029094401001930236
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,4,128,1,float16,float16,127,0.026096001267433167
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,4,128,1,float16,fp8,127,0.02892799973487854
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,4,128,1,float16,fp8,255,0.03718880116939545
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,4,128,1,float16,float16,511,0.05243200063705444
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,4,128,1,float16,fp8,511,0.053438401222229
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,2,128,1,float16,float16,2047,0.0802191972732544
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,4,128,1,float16,float16,1023,0.08521119952201843
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,4,128,1,float16,fp8,1023,0.07851999998092651
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,4,128,1,float16,float16,2047,0.14346400499343873
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,4,128,1,float16,fp8,2047,0.12499200105667115
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,4,128,1,float16,float16,4095,0.26140799522399905
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,4,128,1,float16,fp8,4095,0.224289608001709
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,4,128,1,float16,float16,8191,0.5055952072143555
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,4,128,1,float16,fp8,8191,0.4210927963256836
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,1,128,1,float16,float16,1,0.021328000724315642
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,1,128,1,float16,fp8,1,0.022944000363349915
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,1,128,1,float16,float16,3,0.02101919949054718
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,1,128,1,float16,fp8,3,0.02268799990415573
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,1,128,1,float16,fp8,16383,0.21568799018859863
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,1,128,1,float16,float16,7,0.021009600162506102
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,4,128,1,float16,float16,3,0.02609440088272095
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,1,128,1,float16,float16,15,0.021214400231838227
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,1,128,1,float16,fp8,7,0.02229759991168976
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,1,128,1,float16,fp8,15,0.022385600209236144
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,4,128,1,float16,fp8,16383,0.8069456100463868
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,4,128,1,float16,fp8,7,0.029017600417137145
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,1,128,1,float16,float16,127,0.020742399990558623
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,1,128,1,float16,fp8,127,0.02231519967317581
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,1,128,1,float16,float16,255,0.024038399755954742
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,4,128,1,float16,float16,255,0.03882879912853241
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,1,128,1,float16,fp8,255,0.026790401339530943
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,1,128,1,float16,float16,511,0.03627679944038391
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,1,128,1,float16,fp8,511,0.03452160060405731
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,1,128,1,float16,float16,1023,0.054718399047851564
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,1,128,1,float16,fp8,1023,0.050134402513504026
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,1,128,1,float16,float16,2047,0.08353760242462158
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,1,128,1,float16,fp8,2047,0.07426559925079346
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,1,128,1,float16,float16,4095,0.1419584035873413
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,1,128,1,float16,fp8,4095,0.12369600534439087
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,2,128,1,float16,float16,1,0.02831520140171051
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,2,128,1,float16,fp8,1,0.03171040117740631
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,2,128,1,float16,float16,3,0.028324800729751586
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,2,128,1,float16,fp8,3,0.03145120143890381
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,4,128,1,float16,float16,16383,0.9707807540893555
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,8,4,128,1,float16,fp8,1,0.02871359884738922
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,1,128,1,float16,float16,31,0.02142080068588257
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,2,128,1,float16,float16,15,0.02834239900112152
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,2,128,1,float16,fp8,7,0.031832000613212584
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,1,128,1,float16,fp8,31,0.022193600237369538
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,2,128,1,float16,fp8,15,0.03165279924869537
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,2,128,1,float16,float16,31,0.02805120050907135
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,2,128,1,float16,float16,63,0.028201600909233092
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,2,128,1,float16,fp8,63,0.031600001454353335
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,2,128,1,float16,fp8,127,0.031172800064086913
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,2,128,1,float16,float16,255,0.04098399877548218
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,2,128,1,float16,fp8,255,0.03955360054969788
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,2,128,1,float16,float16,511,0.055550402402877806
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,2,128,1,float16,fp8,511,0.056411200761795045
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,2,128,1,float16,float16,1023,0.08882240056991578
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,2,128,1,float16,fp8,1023,0.08185120224952698
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,2,128,1,float16,float16,2047,0.14695039987564087
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,2,128,1,float16,fp8,2047,0.12872159481048584
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,2,128,1,float16,float16,4095,0.26421279907226564
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,2,128,1,float16,fp8,4095,0.226910400390625
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,4,128,1,float16,float16,1,0.04312799870967865
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,4,128,1,float16,fp8,1,0.048835200071334836
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,4,128,1,float16,float16,3,0.043119999766349795
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,4,128,1,float16,fp8,3,0.048923200368881224
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,4,128,1,float16,float16,7,0.042894399166107176
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,4,128,1,float16,fp8,7,0.048785600066185
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,4,128,1,float16,float16,15,0.042894399166107176
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,4,128,1,float16,fp8,15,0.049112001061439516
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,4,128,1,float16,float16,31,0.0430400013923645
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,4,128,1,float16,fp8,31,0.04922400116920471
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,4,128,1,float16,float16,63,0.04328800141811371
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,4,128,1,float16,fp8,63,0.04933919906616211
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,4,128,1,float16,float16,127,0.04828960001468659
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,4,128,1,float16,fp8,127,0.04984480142593384
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,4,128,1,float16,float16,255,0.06514080166816712
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,4,128,1,float16,fp8,255,0.06709759831428527
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,2,128,1,float16,float16,7,0.02821600139141083
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,4,128,1,float16,float16,511,0.09189599752426147
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,4,128,1,float16,fp8,511,0.09749439954757691
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,4,128,1,float16,float16,1023,0.15502560138702393
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,1,128,1,float16,float16,63,0.02074880003929138
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,4,128,1,float16,fp8,1023,0.14702880382537842
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,2,128,1,float16,fp8,31,0.03157120048999786
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,1,128,1,float16,float16,1,0.03438239991664886
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,4,128,1,float16,float16,2047,0.2737312078475952
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,1,128,1,float16,fp8,1,0.03763040006160736
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,1,128,1,float16,float16,3,0.03430559933185577
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,1,128,1,float16,float16,7,0.03458079993724823
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,1,128,1,float16,fp8,7,0.0372079998254776
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,1,128,1,float16,float16,15,0.034651198983192445
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,4,128,1,float16,fp8,4095,0.43558878898620607
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,1,128,1,float16,float16,31,0.03468959927558899
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,1,128,1,float16,float16,63,0.034539198875427245
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,1,128,1,float16,fp8,31,0.037308800220489505
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,1,128,1,float16,fp8,63,0.03765600025653839
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,1,128,1,float16,float16,127,0.0344895988702774
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,1,128,1,float16,fp8,127,0.03766559958457947
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,1,128,1,float16,float16,255,0.04711039960384369
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,1,128,1,float16,fp8,255,0.046184000372886655
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,1,128,1,float16,float16,511,0.062136000394821166
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,1,128,1,float16,fp8,511,0.06337440013885498
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,1,128,1,float16,float16,1023,0.09573760032653808
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,1,128,1,float16,fp8,1023,0.09026719927787781
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,1,128,1,float16,float16,2047,0.15475039482116698
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,2,128,1,float16,float16,1,0.04861119985580444
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,2,128,1,float16,fp8,1,0.05443840026855469
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,2,128,1,float16,float16,3,0.04863199889659882
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,2,128,1,float16,fp8,3,0.05436639785766602
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,2,128,1,float16,float16,7,0.048547199368476866
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,2,128,1,float16,fp8,7,0.05429440140724182
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,2,128,1,float16,float16,15,0.04843519926071167
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,2,128,1,float16,fp8,15,0.054318398237228394
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,1,128,1,float16,fp8,63,0.022763200104236603
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,2,128,1,float16,fp8,31,0.05453439950942993
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,4,128,1,float16,fp8,2047,0.23853919506072999
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,2,128,1,float16,float16,63,0.048875200748443606
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,1,128,1,float16,fp8,3,0.037248000502586365
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,2,128,1,float16,fp8,63,0.05465120077133179
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,1,128,1,float16,fp8,15,0.03724960088729858
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,2,128,1,float16,fp8,127,0.055320000648498534
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,4,128,1,float16,float16,4095,0.5148384094238281
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,2,128,1,float16,fp8,255,0.0718384027481079
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,2,128,1,float16,float16,511,0.09893119931221009
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,2,128,1,float16,fp8,511,0.10378719568252563
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,2,128,1,float16,float16,1023,0.16238880157470703
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,2,128,1,float16,fp8,1023,0.15442559719085694
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,2,128,1,float16,float16,2047,0.2804703950881958
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,2,128,1,float16,fp8,2047,0.24695041179656982
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,4,128,1,float16,float16,1,0.07688800096511841
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,4,128,1,float16,fp8,1,0.08912799954414367
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,4,128,1,float16,float16,3,0.07698879837989807
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,1,128,1,float16,fp8,2047,0.13818399906158446
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,4,128,1,float16,fp8,3,0.08917440176010132
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,4,128,1,float16,float16,15,0.07687199711799622
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,4,128,1,float16,fp8,15,0.08950240015983582
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,4,128,1,float16,float16,31,0.07703359723091126
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,4,128,1,float16,fp8,31,0.08969600200653076
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,4,128,1,float16,float16,63,0.08096799850463868
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,4,128,1,float16,fp8,63,0.09013280272483826
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,4,128,1,float16,float16,127,0.08312159776687622
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,2,128,1,float16,float16,31,0.04854080080986023
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,4,128,1,float16,fp8,127,0.09517279863357545
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,4,128,1,float16,float16,255,0.11378560066223145
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,4,128,1,float16,fp8,255,0.12254879474639893
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,2,128,1,float16,float16,127,0.05424479842185974
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,4,128,1,float16,float16,511,0.1684399962425232
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,2,128,1,float16,float16,255,0.0713375985622406
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,4,128,1,float16,fp8,511,0.18274879455566406
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,4,128,1,float16,float16,1023,0.2941135883331299
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,1,128,1,float16,float16,1,0.011950399726629257
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,1,128,1,float16,fp8,1,0.012673600018024445
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,1,128,1,float16,float16,3,0.011819200217723846
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,1,128,1,float16,fp8,3,0.012555199861526489
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,1,128,1,float16,float16,7,0.011767999827861786
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,1,128,1,float16,fp8,7,0.012569600343704223
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,1,128,1,float16,float16,15,0.011788800358772278
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,4,128,1,float16,float16,7,0.07676479816436768
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,4,128,1,float16,fp8,7,0.08944960236549378
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,1,128,1,float16,fp8,31,0.01249760016798973
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,1,128,1,float16,float16,63,0.011638399958610535
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,1,128,1,float16,fp8,63,0.012308800220489502
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,1,128,1,float16,float16,127,0.011559999734163284
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,1,128,1,float16,fp8,127,0.012444800138473511
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,1,128,1,float16,float16,255,0.013390399515628815
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,1,128,1,float16,fp8,255,0.014401599764823914
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,1,128,1,float16,float16,511,0.016195200383663177
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,1,128,1,float16,fp8,511,0.01786080002784729
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,8,2,128,1,float16,float16,127,0.028369599580764772
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,1,128,1,float16,float16,1023,0.015913599729537965
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,1,128,1,float16,fp8,1023,0.01695519983768463
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,1,128,1,float16,fp8,2047,0.0173552006483078
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,1,128,1,float16,fp8,4095,0.01807039976119995
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,1,128,1,float16,float16,8191,0.021715199947357176
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,1,128,1,float16,fp8,8191,0.021793599426746368
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,1,128,1,float16,float16,16383,0.032502400875091556
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,4,128,1,float16,fp8,1023,0.28166561126708983
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,1,128,1,float16,float16,32767,0.048291200399398805
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,4,128,1,float16,float16,2047,0.5325695991516113
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,1,128,1,float16,fp8,32767,0.043966400623321536
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,8,4,128,1,float16,fp8,2047,0.46479358673095705
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,2,128,1,float16,fp8,1,0.012291199713945388
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,2,128,1,float16,float16,1,0.011844799667596818
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,2,128,1,float16,fp8,3,0.012329600006341934
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,2,128,1,float16,fp8,7,0.01247360035777092
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,2,128,1,float16,float16,15,0.011817599833011627
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,2,128,1,float16,fp8,15,0.012544000148773193
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,1,128,1,float16,fp8,15,0.012615999579429627
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,2,128,1,float16,float16,31,0.011707200109958649
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,2,128,1,float16,fp8,31,0.012697599828243256
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,2,128,1,float16,fp8,63,0.012708799540996551
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,2,128,1,float16,float16,127,0.011972799897193909
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,2,128,1,float16,fp8,127,0.012630400061607362
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,2,128,1,float16,float16,255,0.01361120045185089
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,2,128,1,float16,fp8,255,0.014531199634075165
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,2,128,1,float16,float16,511,0.01640480011701584
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,2,128,1,float16,fp8,511,0.018214400112628936
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,2,128,1,float16,float16,1023,0.016103999316692354
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,2,128,1,float16,fp8,1023,0.017076799273490907
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,2,128,1,float16,float16,2047,0.018016000092029572
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,2,128,1,float16,fp8,2047,0.017734399437904357
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,1,128,1,float16,float16,2047,0.016288000345230102
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,2,128,1,float16,float16,4095,0.02152640074491501
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,2,128,1,float16,fp8,4095,0.021406400203704833
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,2,128,1,float16,float16,8191,0.030820798873901368
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,1,128,1,float16,float16,4095,0.01823199987411499
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,2,128,1,float16,float16,16383,0.04547680020332336
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,2,128,1,float16,fp8,16383,0.04191679954528808
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,2,128,1,float16,float16,32767,0.07593439817428589
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,2,128,1,float16,fp8,32767,0.0667967975139618
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,1,128,1,float16,fp8,16383,0.028046399354934692
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,4,128,1,float16,fp8,1,0.012811200320720672
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,4,128,1,float16,float16,3,0.01197120025753975
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,4,128,1,float16,float16,7,0.012043199688196182
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,4,128,1,float16,fp8,7,0.012585599720478059
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,4,128,1,float16,float16,15,0.011905600130558015
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,4,128,1,float16,fp8,15,0.012548799812793731
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,2,128,1,float16,float16,3,0.011608000099658965
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,2,128,1,float16,float16,7,0.012055999785661697
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,4,128,1,float16,float16,63,0.011961600184440613
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,4,128,1,float16,fp8,63,0.012665599584579468
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,4,128,1,float16,float16,127,0.012054400146007537
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,1,128,1,float16,float16,31,0.01154400035738945
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,4,128,1,float16,fp8,127,0.012726399302482604
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,4,128,1,float16,float16,255,0.013841600716114044
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,4,128,1,float16,float16,511,0.017028799653053282
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,2,128,1,float16,fp8,8191,0.02744159996509552
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,4,128,1,float16,fp8,511,0.018268799781799315
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,4,128,1,float16,float16,1023,0.017867200076580048
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,4,128,1,float16,fp8,1023,0.017407999932765962
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,4,128,1,float16,float16,1,0.01191679984331131
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,4,128,1,float16,fp8,3,0.012916800379753113
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,4,128,1,float16,float16,4095,0.03009119927883148
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,4,128,1,float16,fp8,4095,0.027225598692893982
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,4,128,1,float16,float16,8191,0.0451231986284256
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,4,128,1,float16,fp8,31,0.012678399682044983
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,4,128,1,float16,float16,16383,0.07633919715881347
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,4,128,1,float16,fp8,16383,0.06562079787254334
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,4,128,1,float16,fp8,255,0.014686399698257446
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,4,128,1,float16,float16,32767,0.13508000373840331
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,4,128,1,float16,fp8,32767,0.11671520471572876
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,8,1,128,1,float16,float16,1,0.05491200089454651
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,8,1,128,1,float16,fp8,1,0.06079840064048767
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,8,1,128,1,float16,fp8,3,0.06047840118408203
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,8,1,128,1,float16,float16,7,0.05464000105857849
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,8,1,128,1,float16,fp8,7,0.0606656014919281
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,8,1,128,1,float16,float16,15,0.05496320128440857
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,8,1,128,1,float16,fp8,15,0.06088799834251404
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,8,1,128,1,float16,float16,31,0.054915201663970944
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,8,1,128,1,float16,fp8,31,0.06069120168685913
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,8,1,128,1,float16,float16,63,0.05506399869918823
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,8,1,128,1,float16,fp8,63,0.06084160208702087
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,4,128,1,float16,float16,2047,0.019593599438667297
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,8,1,128,1,float16,float16,127,0.06075040102005005
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,4,128,1,float16,fp8,2047,0.021190400421619415
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,8,1,128,1,float16,fp8,127,0.06191520094871521
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,8,1,128,1,float16,float16,255,0.07720479965209961
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,8,1,128,1,float16,fp8,255,0.07833439707756043
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,8,1,128,1,float16,float16,511,0.105840003490448
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,8,1,128,1,float16,fp8,511,0.11023520231246949
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,4,128,1,float16,float16,31,0.01188960000872612
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,8,1,128,1,float16,float16,1023,0.17100800275802613
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,8,2,128,1,float16,float16,1,0.08337759971618652
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,8,1,128,1,float16,fp8,1023,0.16084799766540528
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,8,2,128,1,float16,float16,3,0.08361920118331909
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,8,2,128,1,float16,float16,7,0.08357120156288148
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,8,2,128,1,float16,fp8,3,0.09453759789466858
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,8,2,128,1,float16,fp8,7,0.09507200121879578
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,8,2,128,1,float16,fp8,15,0.09480800032615662
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,2,128,1,float16,float16,63,0.011622399836778641
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,8,2,128,1,float16,fp8,31,0.09480479955673218
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,8,2,128,1,float16,float16,63,0.08686720132827759
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,8,2,128,1,float16,fp8,63,0.09597759842872619
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,8,2,128,1,float16,float16,127,0.09038400053977966
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,8,2,128,1,float16,fp8,127,0.10197440385818482
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,8,2,128,1,float16,float16,255,0.12108319997787476
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,8,2,128,1,float16,fp8,255,0.1284351944923401
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,8,1,128,1,float16,float16,3,0.05494719743728638
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,8,2,128,1,float16,float16,511,0.17619359493255615
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,8,2,128,1,float16,fp8,511,0.19028480052948
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,8,4,128,1,float16,float16,1,0.138644802570343
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,8,2,128,1,float16,float16,1023,0.2983583927154541
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,8,4,128,1,float16,fp8,1,0.16279679536819458
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,8,2,128,1,float16,fp8,1023,0.28940639495849607
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,8,4,128,1,float16,float16,3,0.1386911988258362
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,8,4,128,1,float16,float16,7,0.13867679834365845
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,8,4,128,1,float16,fp8,3,0.16234079599380494
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,8,4,128,1,float16,float16,15,0.138372802734375
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,8,4,128,1,float16,fp8,15,0.16278719902038574
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,8,4,128,1,float16,float16,31,0.14134080410003663
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,8,4,128,1,float16,fp8,31,0.1628783941268921
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,8,4,128,1,float16,float16,63,0.1445024013519287
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,8,4,128,1,float16,fp8,63,0.17163519859313964
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,8,4,128,1,float16,float16,127,0.14952800273895264
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,8,4,128,1,float16,fp8,127,0.17436319589614868
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,8,4,128,1,float16,float16,255,0.20388638973236084
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,8,4,128,1,float16,fp8,255,0.22840800285339355
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,8,4,128,1,float16,float16,511,0.31283040046691896
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,8,4,128,1,float16,fp8,511,0.3495568037033081
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,8,4,128,1,float16,fp8,8191,0.0405023992061615
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,8,2,128,1,float16,fp8,1,0.09427520036697387
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,8,1,128,1,float16,float16,1,0.09513919949531555
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,8,4,128,1,float16,float16,1023,0.5576064109802246
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,8,2,128,1,float16,float16,15,0.08315200209617615
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,8,1,128,1,float16,fp8,3,0.10678880214691162
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,8,1,128,1,float16,fp8,7,0.10686559677124023
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,8,1,128,1,float16,float16,7,0.09537439942359924
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,8,1,128,1,float16,float16,15,0.09466080069541931
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,8,1,128,1,float16,fp8,15,0.1067247986793518
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,8,1,128,1,float16,float16,31,0.09548959732055665
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,8,1,128,1,float16,float16,63,0.10076639652252198
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,8,1,128,1,float16,fp8,63,0.10861279964447021
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,8,1,128,1,float16,float16,127,0.1047760009765625
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,8,1,128,1,float16,fp8,127,0.11433119773864746
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,8,1,128,1,float16,float16,255,0.13441599607467652
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,8,1,128,1,float16,fp8,255,0.14154239892959594
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,8,1,128,1,float16,float16,511,0.19049919843673707
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,8,1,128,1,float16,fp8,511,0.2042464017868042
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,8,2,128,1,float16,float16,1,0.14962079524993896
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,8,2,128,1,float16,fp8,1,0.17254400253295898
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,8,2,128,1,float16,float16,3,0.149344003200531
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,8,2,128,1,float16,fp8,3,0.17210719585418702
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,8,2,128,1,float16,float16,7,0.14898240566253662
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,8,2,128,1,float16,fp8,7,0.1722591996192932
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,8,2,128,1,float16,float16,15,0.14908000230789184
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,8,2,128,1,float16,fp8,15,0.1724544048309326
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,8,2,128,1,float16,float16,31,0.15257439613342286
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,8,2,128,1,float16,fp8,31,0.17269279956817626
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,8,1,128,1,float16,fp8,1,0.10610719919204711
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,8,1,128,1,float16,float16,3,0.09463840126991271
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,8,2,128,1,float16,float16,31,0.08280479907989502
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,8,4,128,1,float16,fp8,1023,0.5425536155700683
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,8,2,128,1,float16,fp8,63,0.18339840173721314
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,8,2,128,1,float16,float16,127,0.16236640214920045
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,8,2,128,1,float16,fp8,127,0.18626879453659057
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,8,1,128,1,float16,fp8,31,0.1071120023727417
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,8,2,128,1,float16,float16,255,0.2197648048400879
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,8,2,128,1,float16,fp8,255,0.23701601028442382
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,8,2,128,1,float16,float16,511,0.3273888111114502
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,8,2,128,1,float16,fp8,511,0.36103999614715576
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,8,4,128,1,float16,float16,3,0.26024160385131834
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,8,4,128,1,float16,fp8,3,0.3063584089279175
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,8,4,128,1,float16,float16,7,0.2606368064880371
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,8,4,128,1,float16,fp8,7,0.1626863956451416
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,8,4,128,1,float16,float16,15,0.26026880741119385
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,8,4,128,1,float16,fp8,15,0.30760319232940675
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,8,4,128,1,float16,float16,31,0.26713759899139405
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,8,4,128,1,float16,fp8,31,0.31961760520935056
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,8,4,128,1,float16,float16,63,0.2698767900466919
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,8,4,128,1,float16,fp8,63,0.32661759853363037
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,8,4,128,1,float16,float16,127,0.2758624076843262
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,8,4,128,1,float16,fp8,127,0.3294064044952393
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,8,4,128,1,float16,float16,255,0.38218719959259034
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,8,4,128,1,float16,fp8,255,0.43260798454284666
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,1,128,1,float16,float16,1,0.01191679984331131
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,1,128,1,float16,fp8,1,0.012611199915409089
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,8,2,128,1,float16,float16,63,0.15741440057754516
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,8,4,128,1,float16,float16,511,0.6008096218109131
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,1,128,1,float16,fp8,3,0.01284160017967224
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,1,128,1,float16,fp8,7,0.012889599800109864
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,1,128,1,float16,float16,15,0.011979199945926666
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,1,128,1,float16,fp8,15,0.012683199346065521
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,1,128,1,float16,float16,31,0.011840000003576278
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,8,4,128,1,float16,float16,1,0.2602528095245361
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,1,128,1,float16,float16,63,0.011617600172758102
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,8,4,128,1,float16,fp8,1,0.3075439929962158
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,1,128,1,float16,fp8,63,0.012337599694728852
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,1,128,1,float16,float16,127,0.011980800330638886
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,1,128,1,float16,fp8,127,0.012305600196123123
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,1,128,1,float16,float16,255,0.013302400708198547
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,1,128,1,float16,fp8,255,0.014267200231552124
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,8,4,128,1,float16,fp8,7,0.3071631908416748
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,1,128,1,float16,float16,511,0.016590400040149687
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,1,128,1,float16,fp8,511,0.01800000071525574
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,1,128,1,float16,float16,1023,0.016259199380874632
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,1,128,1,float16,float16,2047,0.01788959950208664
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,1,128,1,float16,fp8,2047,0.017774400115013123
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,1,128,1,float16,float16,4095,0.021374399960041045
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,1,128,1,float16,fp8,4095,0.021359999477863312
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,1,128,1,float16,float16,8191,0.031112000346183777
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,1,128,1,float16,fp8,8191,0.027399998903274537
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,1,128,1,float16,float16,16383,0.045956799387931825
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,1,128,1,float16,fp8,16383,0.04132960140705109
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,1,128,1,float16,float16,32767,0.07654880285263062
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,1,128,1,float16,fp8,32767,0.06621119976043702
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,2,128,1,float16,float16,1,0.011753600090742111
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,2,128,1,float16,fp8,1,0.012510399520397186
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,2,128,1,float16,float16,7,0.011764799803495407
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,2,128,1,float16,fp8,7,0.01252480000257492
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,2,128,1,float16,float16,15,0.011908800154924393
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,2,128,1,float16,fp8,15,0.012656000256538392
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,2,128,1,float16,float16,31,0.01188960000872612
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,2,128,1,float16,fp8,31,0.012665599584579468
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,2,128,1,float16,float16,63,0.011939200013875962
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,1,128,1,float16,float16,3,0.011881600320339202
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,2,128,1,float16,float16,127,0.01226079985499382
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,1,128,1,float16,float16,7,0.011822400242090225
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,2,128,1,float16,fp8,127,0.013067199289798737
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,8,4,128,1,float16,fp8,511,0.668991994857788
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,2,128,1,float16,float16,255,0.013867199420928955
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,2,128,1,float16,fp8,511,0.018639999628067016
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,2,128,1,float16,float16,511,0.01668799966573715
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,2,128,1,float16,float16,1023,0.017867200076580048
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,2,128,1,float16,fp8,1023,0.01757120043039322
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,1,128,1,float16,fp8,31,0.012619200348854064
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,2,128,1,float16,fp8,2047,0.021305599808692934
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,2,128,1,float16,float16,4095,0.03065440058708191
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,2,128,1,float16,fp8,4095,0.027459201216697694
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,2,128,1,float16,float16,8191,0.04541279971599579
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,2,128,1,float16,float16,16383,0.07629119753837585
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,1,128,1,float16,fp8,1023,0.017083199322223665
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,2,128,1,float16,float16,32767,0.13734560012817382
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,2,128,1,float16,fp8,32767,0.11681920289993286
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,4,128,1,float16,float16,1,0.012411200255155564
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,4,128,1,float16,fp8,1,0.012806400656700134
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,2,128,1,float16,float16,3,0.01213119998574257
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,4,128,1,float16,fp8,3,0.01302880048751831
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,4,128,1,float16,float16,7,0.012241599708795547
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,4,128,1,float16,fp8,7,0.013009600341320038
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,4,128,1,float16,float16,15,0.012383999675512314
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,4,128,1,float16,fp8,15,0.013148799538612366
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,4,128,1,float16,float16,31,0.01239520013332367
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,4,128,1,float16,fp8,31,0.012836800515651703
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,4,128,1,float16,float16,63,0.012142399698495865
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,4,128,1,float16,fp8,63,0.012740799784660339
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,4,128,1,float16,float16,127,0.012092799693346024
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,2,128,1,float16,fp8,255,0.015107199549674988
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,4,128,1,float16,fp8,127,0.013214400410652161
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,4,128,1,float16,float16,255,0.013926400244235993
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,4,128,1,float16,fp8,255,0.01517920047044754
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,2,128,1,float16,float16,2047,0.01973759979009628
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,4,128,1,float16,float16,511,0.016921600699424742
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,4,128,1,float16,fp8,511,0.018628799915313722
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,4,128,1,float16,float16,1023,0.01969760060310364
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,4,128,1,float16,fp8,1023,0.02081120014190674
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,4,128,1,float16,float16,2047,0.030110400915145875
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,4,128,1,float16,fp8,2047,0.02672159969806671
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,4,128,1,float16,float16,4095,0.04535680115222931
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,4,128,1,float16,fp8,4095,0.041145598888397215
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,4,128,1,float16,float16,8191,0.07684800028800964
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,2,128,1,float16,fp8,16383,0.06633279919624328
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,4,128,1,float16,float16,16383,0.1382848024368286
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,4,128,1,float16,fp8,16383,0.11612319946289062
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,4,128,1,float16,float16,32767,0.2601583957672119
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,1,0.013236799836158752
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,4,128,1,float16,float16,3,0.011950399726629257
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,1,128,1,float16,fp8,1,0.014201599359512328
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,3,0.01329440027475357
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,1,128,1,float16,fp8,3,0.014163200557231904
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,7,0.013247999548912048
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,1,128,1,float16,fp8,7,0.014103999733924866
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,15,0.013249599933624267
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,1,128,1,float16,fp8,15,0.014059199392795563
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,31,0.013318400084972381
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,1,128,1,float16,fp8,31,0.013812799751758576
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,63,0.013515199720859527
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,1,128,1,float16,fp8,63,0.013955199718475341
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,127,0.013505600392818451
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,1,128,1,float16,fp8,127,0.013963200151920319
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,255,0.015462400019168853
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,1,128,1,float16,fp8,255,0.016156800091266632
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,511,0.018139199912548067
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,1,128,1,float16,fp8,511,0.019654400646686554
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,1023,0.022310400009155275
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,1,128,1,float16,fp8,1023,0.023454399406909944
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,2047,0.0329008013010025
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,1,128,1,float16,fp8,2047,0.029532799124717714
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,4095,0.04767520129680634
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,1,128,1,float16,fp8,4095,0.04347679913043976
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,8191,0.08055359721183777
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,1,128,1,float16,fp8,8191,0.07032480239868164
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,16383,0.1434048056602478
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,2,128,1,float16,fp8,8191,0.041971200704574586
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,4,128,1,float16,fp8,8191,0.06567040085792542
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,1,128,1,float16,fp8,32767,0.22100160121917725
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,1,0.013310399651527405
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,2,128,1,float16,fp8,1,0.01414719969034195
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,3,0.013273599743843078
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,2,128,1,float16,fp8,3,0.014318400621414184
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,65535,0.4949215888977051
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,1,128,1,float16,fp8,65535,0.41519999504089355
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,2,128,1,float16,fp8,7,0.014054399728775025
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,15,0.013417600095272065
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,2,128,1,float16,fp8,15,0.013991999626159667
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,31,0.01377439945936203
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,2,128,1,float16,fp8,31,0.014054399728775025
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,2,128,1,float16,fp8,63,0.014163200557231904
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,127,0.013843199610710144
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,2,128,1,float16,fp8,127,0.014416000247001648
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,255,0.015537600219249725
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,2,128,1,float16,fp8,255,0.016499200463294984
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,511,0.018751999735832213
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,2,128,1,float16,fp8,511,0.020032000541687012
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,1023,0.030374398827552794
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,2,128,1,float16,fp8,1023,0.027433601021766663
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,2047,0.04510239958763122
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,2,128,1,float16,fp8,2047,0.04116480052471161
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,4095,0.07736639976501465
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,2,128,1,float16,fp8,4095,0.06613280177116394
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,8191,0.13663359880447387
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,32767,0.26293120384216306
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,2,128,1,float16,fp8,8191,0.11643680334091186
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,16383,0.256662392616272
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,2,128,1,float16,fp8,16383,0.21303679943084716
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,7,0.013406400382518769
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,2,128,1,float16,fp8,3,0.012494400143623352
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,32767,0.4977888107299805
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,2,128,1,float16,fp8,32767,0.40544958114624025
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,1,0.011488000303506852
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,3,0.011369600147008895
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,1,128,1,float16,fp8,3,0.012108799815177918
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,7,0.011334399878978729
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,1,128,1,float16,fp8,7,0.012076800316572189
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,65535,0.9535584449768066
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,15,0.01130400002002716
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,2,128,1,float16,fp8,65535,0.7936255931854248
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,1,128,1,float16,fp8,15,0.012291199713945388
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,63,0.011433599889278412
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,1,128,1,float16,fp8,63,0.014094400405883788
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,127,0.013172799348831176
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,1,128,1,float16,fp8,16383,0.12443840503692627
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,1,128,1,float16,fp8,127,0.013171200454235078
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,255,0.012788799405097962
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,1,128,1,float16,fp8,255,0.015396800637245179
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,511,0.015851199626922607
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,1,128,1,float16,fp8,511,0.019729599356651306
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,1023,0.016502399742603303
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,1,128,1,float16,fp8,1023,0.017659200727939604
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,2047,0.016502399742603303
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,1,128,1,float16,fp8,2047,0.017774400115013123
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,4095,0.016892799735069276
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,1,128,1,float16,fp8,4095,0.017873600125312805
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,8191,0.018638400733470915
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,1,128,1,float16,fp8,8191,0.019707199931144715
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,16383,0.02176479995250702
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,1,128,1,float16,fp8,16383,0.022655999660491942
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,32767,0.023636800050735474
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,1,128,1,float16,fp8,32767,0.02372799962759018
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,63,0.013486400246620178
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,1,128,1,float16,fp8,65535,0.02772960066795349
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,1,0.011327999830245971
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,2,128,1,float16,fp8,1,0.01204639971256256
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,1,128,1,float16,fp8,1,0.012137600034475327
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,3,0.011324799805879592
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,2,128,1,float16,fp8,63,0.012787200510501862
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,7,0.011404799669981003
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,2,128,1,float16,fp8,7,0.012092799693346024
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,2,128,1,float16,fp8,15,0.012057600170373916
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,31,0.011422400176525117
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,31,0.011428800225257874
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,2,128,1,float16,fp8,31,0.012281599640846252
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,63,0.011473599821329117
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,1,128,1,float16,fp8,31,0.012065599858760833
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,2,128,1,float16,fp8,127,0.012244799733161926
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,255,0.012993599474430084
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,2,128,1,float16,fp8,255,0.014169600605964661
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,511,0.015993599593639374
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,2,128,1,float16,fp8,511,0.018160000443458557
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,1023,0.0164000004529953
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,2,128,1,float16,fp8,1023,0.01763039976358414
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,2047,0.016646400094032288
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,2,128,1,float16,fp8,2047,0.01796479970216751
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,4095,0.016740800440311433
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,2,128,1,float16,fp8,4095,0.0179407998919487
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,8,4,128,1,float16,fp8,32767,0.21321120262145996
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,8191,0.018592000007629395
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,16383,0.020343999564647674
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,2,128,1,float16,fp8,8191,0.01956000030040741
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,2,128,1,float16,fp8,16383,0.020732800662517547
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,32767,0.02441119998693466
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,2,128,1,float16,fp8,32767,0.024027200043201448
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,65535,0.03391200006008148
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,65535,0.02757439911365509
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,2,128,1,float16,fp8,65535,0.03274720013141632
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,1,128,1,float16,fp8,1,0.0121568001806736
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,3,0.011340799927711486
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,7,0.011470399796962738
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,1,128,1,float16,fp8,7,0.01212640032172203
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,15,0.011497599631547928
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,1,128,1,float16,fp8,15,0.012134400010108948
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,31,0.01133119985461235
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,1,128,1,float16,fp8,31,0.012055999785661697
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,2,128,1,float16,fp8,3,0.01210239976644516
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,1,128,1,float16,fp8,63,0.01194079965353012
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,127,0.011294399946928024
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,255,0.012884800136089326
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,1,128,1,float16,fp8,255,0.013982400298118591
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,511,0.015863999724388123
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,2,128,1,float16,fp8,63,0.01228640004992485
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,127,0.011486399918794632
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,1023,0.01660960018634796
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,1,128,1,float16,fp8,1023,0.017744000256061553
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,1,128,1,float16,fp8,2047,0.018038399517536163
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,4095,0.016952000558376312
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,1,128,1,float16,fp8,4095,0.017847999930381775
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,8191,0.01889439970254898
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,1,128,1,float16,fp8,8191,0.019918400049209594
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,16383,0.023332799971103668
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,1,128,1,float16,fp8,16383,0.022932800650596618
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,32767,0.02667360007762909
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,1,128,1,float16,fp8,32767,0.026814401149749756
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,65535,0.03625600039958954
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,1,128,1,float16,fp8,65535,0.03300159871578216
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,1,0.01146399974822998
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,2,128,1,float16,fp8,1,0.012283200025558471
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,3,0.011508800089359283
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,2,128,1,float16,fp8,3,0.012147200107574464
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,1,0.011484800279140473
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,7,0.011236800253391266
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,2,128,1,float16,fp8,7,0.012257599830627441
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,1,128,1,float16,fp8,3,0.01234079971909523
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,2,128,1,float16,fp8,15,0.012247999757528305
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,2,128,1,float16,fp8,31,0.012086399644613267
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,63,0.011379200220108032
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,63,0.011284799873828888
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,2,128,1,float16,fp8,63,0.01212640032172203
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,15,0.011526399850845337
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,2,128,1,float16,fp8,127,0.012110400199890136
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,255,0.013193599879741669
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,511,0.016169600188732147
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,2,128,1,float16,fp8,511,0.018111999332904815
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,1,128,1,float16,fp8,511,0.01797119975090027
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,1023,0.016857600212097167
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,2,128,1,float16,fp8,1023,0.017844800651073457
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,2047,0.01648160070180893
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,4095,0.01712000072002411
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,15,0.011396799981594086
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,31,0.011665599793195725
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,8191,0.020252799987792967
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,2,128,1,float16,fp8,8191,0.020084799826145174
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,127,0.011376000195741653
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,2,128,1,float16,fp8,255,0.014000000059604644
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,32767,0.03342880010604858
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,2,128,1,float16,fp8,32767,0.02969760000705719
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,2047,0.016595199704170227
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,65535,0.04943679869174957
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,2,128,1,float16,fp8,2047,0.01809599995613098
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,1,0.012278400361537933
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,1,128,1,float16,fp8,1,0.013352000713348388
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,3,0.012324800342321396
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,1,128,1,float16,fp8,3,0.013299199938774108
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,1,128,1,float16,fp8,7,0.013308799266815186
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,15,0.012275200337171555
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,1,128,1,float16,fp8,15,0.013150399923324585
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,31,0.012263999879360199
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,1,128,1,float16,fp8,31,0.013143999874591828
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,63,0.012300799787044524
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,1,128,1,float16,fp8,63,0.013176000118255616
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,127,0.012566399574279786
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,1,128,1,float16,fp8,127,0.013156799972057343
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,255,0.01435839980840683
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,1,128,1,float16,fp8,255,0.015278400480747223
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,511,0.01727519929409027
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,1,128,1,float16,fp8,511,0.01884319931268692
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,1023,0.01929119974374771
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,1,128,1,float16,fp8,1023,0.01894879937171936
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,2047,0.02187200039625168
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,1,128,1,float16,fp8,2047,0.022619199752807618
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,4095,0.03248960077762604
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,1,128,1,float16,fp8,4095,0.028489598631858827
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,8191,0.049142399430274965
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,1,128,1,float16,fp8,8191,0.0440416008234024
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,16383,0.08283519744873047
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,1,128,1,float16,fp8,16383,0.0720848023891449
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,1,128,1,float16,fp8,127,0.011913599818944931
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,32767,0.14201279878616332
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,2,128,1,float16,fp8,16383,0.023588800430297853
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,1,128,1,float16,fp8,32767,0.12259999513626099
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,1,0.012537600100040435
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,2,128,1,float16,fp8,1,0.013396799564361572
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,3,0.012455999851226807
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,2,128,1,float16,fp8,3,0.013220800459384919
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,7,0.012518399953842163
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,1,128,1,float16,fp8,65535,0.22011520862579345
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,2,128,1,float16,fp8,7,0.0131632000207901
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,31,0.012366399914026261
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,2,128,1,float16,fp8,15,0.01345440000295639
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,2,128,1,float16,fp8,31,0.01340319961309433
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,63,0.012411200255155564
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,7,0.012428800016641617
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,2,128,1,float16,fp8,63,0.013391999900341034
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,2,128,1,float16,fp8,127,0.013475200533866883
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,2,128,1,float16,fp8,255,0.015279999375343323
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,511,0.01751199960708618
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,2,128,1,float16,fp8,511,0.018952000141143798
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,1023,0.021217599511146545
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,2,128,1,float16,fp8,1023,0.022487999498844148
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,2047,0.03223839998245239
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,2,128,1,float16,fp8,2047,0.028638398647308348
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,4095,0.04720160067081451
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,2,128,1,float16,fp8,4095,0.042508798837661746
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,8191,0.08012160062789916
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,2,128,1,float16,fp8,8191,0.06916319727897643
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,2,128,1,float16,fp8,4095,0.01791519969701767
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,16383,0.1387055993080139
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,2,128,1,float16,fp8,16383,0.11987999677658082
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,16383,0.023763200640678404
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,32767,0.25651519298553466
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,65535,0.2649264097213745
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,15,0.012567999958992004
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,1,128,1,float16,fp8,1,0.012110400199890136
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,2,128,1,float16,fp8,32767,0.21696000099182128
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,3,0.011316800117492675
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,7,0.011231999844312668
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,127,0.012724800407886505
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,1,128,1,float16,fp8,7,0.012099199742078782
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,255,0.014336000382900237
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,1,128,1,float16,fp8,15,0.012120000272989272
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,31,0.011343999952077865
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,1,128,1,float16,fp8,31,0.012223999947309494
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,63,0.011350400000810622
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,1,128,1,float16,fp8,63,0.012228800356388092
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,127,0.011371199786663056
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,255,0.012894399464130402
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,1,128,1,float16,fp8,255,0.013940800726413728
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,511,0.016209599375724793
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,1,128,1,float16,fp8,511,0.018275199830532073
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,1023,0.01674399971961975
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,1,128,1,float16,fp8,1023,0.017918400466442108
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,2047,0.016782400012016297
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,1,128,1,float16,fp8,2047,0.017870399355888366
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,4095,0.017051200568675994
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,1,128,1,float16,fp8,4095,0.018196800351142885
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,8191,0.020347200334072113
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,1,128,1,float16,fp8,8191,0.02003680020570755
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,16383,0.026443201303482055
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,4,2,128,1,float16,fp8,65535,0.045875200629234315
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,1,128,1,float16,fp8,16383,0.026532799005508423
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,1,128,1,float16,fp8,3,0.012267199903726577
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,65535,0.49580960273742675
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,32767,0.036057600378990175
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,4,2,128,1,float16,fp8,65535,0.40906558036804197
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,1,128,1,float16,fp8,32767,0.03252480030059814
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,1,128,1,float16,fp8,65535,0.048604801297187805
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,65535,0.05474079847335815
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,1,0.011617600172758102
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,3,0.011502400040626526
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,1,128,1,float16,fp8,127,0.012254399806261062
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,2,128,1,float16,fp8,7,0.012303999811410903
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,15,0.011532799899578094
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,2,128,1,float16,fp8,15,0.012193600088357926
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,31,0.011604800075292587
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,2,128,1,float16,fp8,31,0.012123200297355651
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,63,0.011470399796962738
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,127,0.011334399878978729
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,2,128,1,float16,fp8,127,0.012113600224256515
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,255,0.01313759982585907
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,2,128,1,float16,fp8,255,0.014248000085353851
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,511,0.0161408007144928
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,2,128,1,float16,fp8,511,0.017688000202178956
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,1023,0.016641600430011748
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,2,128,1,float16,fp8,1023,0.01770080029964447
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,2047,0.017192000150680543
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,2,128,1,float16,fp8,2047,0.01794400066137314
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,4095,0.018883199989795686
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,2,128,1,float16,fp8,4095,0.018318399786949158
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,8191,0.023427200317382813
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,2,128,1,float16,fp8,8191,0.023340800404548646
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,16383,0.032739201188087465
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,2,128,1,float16,fp8,16383,0.029334399104118346
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,32767,0.04849919974803925
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,2,128,1,float16,fp8,32767,0.045510399341583255
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,1,0.011495999991893768
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,65535,0.08002560138702393
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,1,128,1,float16,float16,1,0.01499519944190979
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,15,0.011566399782896041
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,1,128,1,float16,float16,3,0.015083199739456177
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,2,128,1,float16,fp8,1,0.012041600048542022
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,1,128,1,float16,fp8,3,0.01584800034761429
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,1,128,1,float16,float16,7,0.014660799503326416
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,1,128,1,float16,fp8,7,0.015806399285793304
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,1,128,1,float16,fp8,15,0.015724800527095795
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,1,128,1,float16,float16,31,0.015137599408626556
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,1,128,1,float16,fp8,31,0.015990400314331056
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,1,128,1,float16,float16,63,0.015131199359893798
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,1,128,1,float16,fp8,63,0.01592160016298294
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,1,128,1,float16,float16,127,0.015238399803638458
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,1,128,1,float16,fp8,127,0.015836800634860992
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,1,128,1,float16,float16,255,0.017159999907016756
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,1,128,1,float16,fp8,255,0.01794400066137314
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,1,128,1,float16,float16,511,0.019868800044059755
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,1,128,1,float16,fp8,511,0.021558399498462676
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,1,128,1,float16,float16,1023,0.03257119953632355
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,1,128,1,float16,fp8,1023,0.029148799180984498
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,1,128,1,float16,float16,2047,0.04707199931144714
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,1,128,1,float16,fp8,2047,0.04256480038166046
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,1,128,1,float16,float16,4095,0.07824320197105408
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,1,128,1,float16,fp8,4095,0.0678160011768341
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,1,128,1,float16,float16,8191,0.1378383994102478
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,1,128,1,float16,fp8,8191,0.11744320392608643
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,1,128,1,float16,float16,16383,0.254804801940918
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,1,128,1,float16,fp8,16383,0.21433279514312745
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,2,128,1,float16,fp8,65535,0.07244480252265931
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,1,128,1,float16,fp8,1,0.015263999998569488
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,2,128,1,float16,float16,1,0.018323199450969697
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,2,128,1,float16,fp8,1,0.01968960016965866
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,2,128,1,float16,float16,3,0.018486399948596955
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,1,128,1,float16,fp8,32767,0.40735678672790526
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,2,128,1,float16,fp8,3,0.019844800233840942
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,2,128,1,float16,float16,7,0.01852799952030182
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,2,128,1,float16,float16,15,0.018387199938297273
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,2,128,1,float16,fp8,7,0.020156799256801604
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,1,128,1,float16,float16,15,0.014766399562358857
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,2,128,1,float16,float16,31,0.018350400030612946
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,2,128,1,float16,fp8,31,0.02003999948501587
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,2,128,1,float16,fp8,63,0.01987680047750473
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,2,128,1,float16,float16,127,0.01886560022830963
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,2,128,1,float16,fp8,127,0.019950400292873382
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,2,128,1,float16,float16,255,0.022230400145053862
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,2,128,1,float16,fp8,255,0.02375520020723343
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,2,128,1,float16,float16,511,0.033308801054954526
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,2,128,1,float16,fp8,511,0.03125919997692108
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,2,128,1,float16,float16,1023,0.050281602144241336
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,2,128,1,float16,fp8,1023,0.047007998824119566
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,2,128,1,float16,float16,2047,0.08078879714012147
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,2,128,1,float16,fp8,2047,0.07010719776153565
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,2,128,1,float16,float16,4095,0.14004960060119628
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,2,128,1,float16,fp8,4095,0.11959840059280395
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,2,128,1,float16,fp8,8191,0.2197376012802124
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,2,128,1,float16,float16,16383,0.49933757781982424
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,1,128,1,float16,float16,32767,0.4858384132385254
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,7,0.01143840029835701
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,2,128,1,float16,fp8,15,0.019926400482654573
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,4,1,128,1,float16,float16,1,0.02129279971122742
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,2,128,1,float16,fp8,63,0.012068799883127212
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,4,1,128,1,float16,float16,3,0.02064799964427948
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,2,128,1,float16,fp8,32767,0.7963168144226074
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,4,1,128,1,float16,fp8,3,0.022163200378417968
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,4,1,128,1,float16,float16,7,0.02062080055475235
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,2,128,1,float16,float16,8191,0.25828640460968016
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,4,1,128,1,float16,float16,15,0.020785599946975708
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,4,1,128,1,float16,fp8,15,0.022894400358200073
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,4,1,128,1,float16,float16,31,0.021376000344753267
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,4,1,128,1,float16,fp8,31,0.02327360063791275
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,4,1,128,1,float16,float16,63,0.02152640074491501
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,4,1,128,1,float16,fp8,63,0.02279199957847595
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,4,1,128,1,float16,float16,127,0.02147520035505295
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,4,1,128,1,float16,fp8,127,0.022787199914455415
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,4,1,128,1,float16,float16,255,0.024702399969100952
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,4,1,128,1,float16,fp8,255,0.026931199431419372
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,4,2,128,1,float16,fp8,3,0.012368000298738479
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,4,1,128,1,float16,float16,511,0.03634400069713593
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,4,1,128,1,float16,fp8,511,0.034643200039863584
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,2,128,1,float16,float16,63,0.01844000071287155
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,4,1,128,1,float16,float16,1023,0.053724801540374754
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,4,1,128,1,float16,fp8,1,0.022889600694179536
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,4,1,128,1,float16,float16,2047,0.0839743971824646
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,4,1,128,1,float16,float16,4095,0.14421279430389405
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,4,1,128,1,float16,fp8,4095,0.12372959852218628
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,4,1,128,1,float16,float16,8191,0.26438078880310056
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,4,1,128,1,float16,fp8,8191,0.22311201095581054
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,2,128,1,float16,float16,32767,0.9626336097717285
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,4,2,128,1,float16,float16,1,0.028411200642585753
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,4,2,128,1,float16,fp8,1,0.031457599997520444
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,4,2,128,1,float16,fp8,3,0.03139840066432953
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,4,2,128,1,float16,float16,7,0.028126400709152222
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,4,2,128,1,float16,fp8,7,0.03144479990005493
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,4,2,128,1,float16,fp8,15,0.03142400085926056
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,4,1,128,1,float16,fp8,7,0.022257600724697114
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,4,2,128,1,float16,float16,31,0.028324800729751586
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,4,2,128,1,float16,fp8,31,0.03110719919204712
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,4,2,128,1,float16,float16,63,0.02847520112991333
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,4,2,128,1,float16,fp8,63,0.031353598833084105
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,4,2,128,1,float16,float16,127,0.02847839891910553
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,4,2,128,1,float16,fp8,127,0.031430399417877196
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,4,2,128,1,float16,float16,255,0.04087519943714142
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,4,2,128,1,float16,fp8,255,0.03948479890823364
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,4,2,128,1,float16,float16,511,0.05509600043296814
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,4,2,128,1,float16,fp8,511,0.056041598320007324
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,4,2,128,1,float16,float16,1023,0.08828480243682861
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,4,2,128,1,float16,fp8,1023,0.08222879767417908
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,4,2,128,1,float16,float16,2047,0.1476704001426697
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,4,2,128,1,float16,fp8,2047,0.12838239669799806
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,4,2,128,1,float16,fp8,16383,0.41368799209594725
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,4,2,128,1,float16,float16,4095,0.26911039352416993
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,4,2,128,1,float16,fp8,4095,0.22769439220428467
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,4,1,128,1,float16,fp8,1023,0.050414401292800906
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,4,1,128,1,float16,fp8,1,0.03708159923553467
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,4,1,128,1,float16,float16,3,0.034307199716567996
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,4,1,128,1,float16,fp8,3,0.037011200189590455
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,4,1,128,1,float16,float16,7,0.034324800968170165
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,4,2,128,1,float16,float16,8191,0.505241584777832
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,4,1,128,1,float16,fp8,7,0.03712159991264343
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,4,2,128,1,float16,fp8,8191,0.4248655796051025
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,4,1,128,1,float16,float16,31,0.03426080048084259
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,4,1,128,1,float16,fp8,15,0.03728800117969513
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,4,1,128,1,float16,fp8,31,0.03705439865589142
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,4,1,128,1,float16,float16,63,0.034272000193595886
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,4,1,128,1,float16,float16,127,0.03468480110168457
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,4,1,128,1,float16,fp8,127,0.03733119964599609
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,4,1,128,1,float16,float16,255,0.04669280052185058
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,4,1,128,1,float16,fp8,255,0.045371198654174806
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,4,1,128,1,float16,float16,511,0.061742401123046874
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,4,2,128,1,float16,float16,3,0.028088000416755677
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,4,2,128,1,float16,float16,15,0.02805280089378357
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,4,1,128,1,float16,float16,1023,0.09559199810028077
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,4,1,128,1,float16,fp8,1023,0.08883839845657349
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,4,1,128,1,float16,float16,2047,0.15421279668807983
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,4,1,128,1,float16,fp8,2047,0.1353935956954956
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,4,2,128,1,float16,float16,1,0.04874880015850067
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,4,1,128,1,float16,float16,4095,0.2697983980178833
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,4,2,128,1,float16,fp8,1,0.054313600063323975
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,4,2,128,1,float16,fp8,3,0.05436639785766602
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,4,2,128,1,float16,float16,7,0.04842720031738281
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,4,2,128,1,float16,fp8,7,0.05414720177650452
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,4,2,128,1,float16,float16,15,0.048390400409698484
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,4,2,128,1,float16,fp8,15,0.05421280264854431
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,4,2,128,1,float16,float16,31,0.048403200507164
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,4,2,128,1,float16,fp8,31,0.05431039929389954
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,4,2,128,1,float16,float16,63,0.05031520128250122
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,4,2,128,1,float16,fp8,63,0.05451520085334778
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,4,1,128,1,float16,fp8,2047,0.07346240282058716
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,4,2,128,1,float16,float16,127,0.05370879769325256
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,4,2,128,1,float16,fp8,127,0.05495679974555969
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,4,2,128,1,float16,float16,255,0.0709007978439331
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,4,1,128,1,float16,float16,15,0.03437759876251221
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,4,2,128,1,float16,fp8,255,0.07177600264549255
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,4,2,128,1,float16,float16,511,0.09872320294380188
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,4,2,128,1,float16,fp8,511,0.10279840230941772
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,4,2,128,1,float16,float16,1023,0.1614575982093811
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,4,2,128,1,float16,fp8,1023,0.1528656005859375
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,4,2,128,1,float16,float16,2047,0.27891519069671633
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,4,1,128,1,float16,fp8,511,0.06201760172843933
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,4,2,128,1,float16,fp8,2047,0.24368960857391359
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,1,0.011444800347089768
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,1,128,1,float16,fp8,1,0.012464000284671784
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,3,0.011499200016260147
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,1,128,1,float16,fp8,3,0.012273599952459335
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,4,2,128,1,float16,float16,4095,0.5204127788543701
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,7,0.01146719977259636
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,15,0.011740799993276596
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,1,128,1,float16,fp8,7,0.01249760016798973
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,4,2,128,1,float16,fp8,4095,0.4427328109741211
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,31,0.011611200124025344
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,1,128,1,float16,fp8,31,0.012484800070524216
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,4,2,128,1,float16,float16,3,0.048977598547935486
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,1,128,1,float16,fp8,63,0.012368000298738479
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,127,0.011777599900960922
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,1,128,1,float16,fp8,127,0.01250240057706833
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,255,0.013340799510478974
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,1,128,1,float16,fp8,255,0.014548799395561219
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,511,0.01641920059919357
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,1,128,1,float16,fp8,511,0.018190400302410127
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,1023,0.016995200514793397
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,1,128,1,float16,fp8,1023,0.01823199987411499
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,2047,0.017263999581336974
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,1,128,1,float16,fp8,2047,0.018380799889564516
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,4095,0.018913599848747253
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,1,128,1,float16,fp8,4095,0.018723200261592864
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,8191,0.02356480062007904
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,1,128,1,float16,fp8,8191,0.023951999843120575
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,16383,0.0362991988658905
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,1,128,1,float16,fp8,16383,0.03273119926452637
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,32767,0.05189120173454285
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,1,128,1,float16,fp8,32767,0.04906879961490631
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,65535,0.08342400193214417
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,1,128,1,float16,fp8,65535,0.0766048014163971
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,1,0.011777599900960922
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,2,128,1,float16,fp8,1,0.012715199589729309
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,3,0.011761599779129028
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,2,128,1,float16,fp8,3,0.012545600533485413
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,7,0.011766400188207626
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,2,128,1,float16,fp8,7,0.012406399846076966
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,15,0.011657600104808808
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,2,128,1,float16,fp8,15,0.012441600114107132
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,4,1,128,1,float16,fp8,63,0.03724640011787415
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,31,0.011604800075292587
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,2,128,1,float16,fp8,31,0.012583999335765839
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,63,0.011769600212574005
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,127,0.011870399862527848
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,2,128,1,float16,fp8,127,0.012548799812793731
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,255,0.013729600608348847
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,2,128,1,float16,fp8,255,0.014510400593280792
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,511,0.016756799817085267
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,2,128,1,float16,fp8,511,0.0181536003947258
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,2,128,1,float16,fp8,1023,0.017948800325393678
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,2047,0.01865919977426529
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,2,128,1,float16,fp8,2047,0.018299199640750885
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,4095,0.02205439954996109
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,2,128,1,float16,fp8,4095,0.02242079973220825
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,1,128,1,float16,fp8,15,0.01244639977812767
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,8191,0.032846400141716005
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,2,128,1,float16,fp8,8191,0.029532799124717714
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,63,0.011710400134325028
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,16383,0.04743840098381043
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,2,128,1,float16,fp8,16383,0.04396319985389709
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,32767,0.07902399897575378
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,65535,0.13790880441665648
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,2,128,1,float16,fp8,65535,0.11984000205993653
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,4,1,128,1,float16,float16,1,0.05445280075073242
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,4,1,128,1,float16,fp8,1,0.06090559959411621
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,4,1,128,1,float16,float16,3,0.05448639988899231
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,4,1,128,1,float16,fp8,3,0.0611519992351532
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,4,1,128,1,float16,float16,7,0.05434880256652832
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,4,1,128,1,float16,fp8,7,0.061187201738357545
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,4,1,128,1,float16,float16,15,0.054688000679016115
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,4,1,128,1,float16,fp8,15,0.061457598209381105
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,4,1,128,1,float16,float16,31,0.0546783983707428
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,4,1,128,1,float16,fp8,31,0.061459201574325564
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,4,1,128,1,float16,fp8,63,0.0616703987121582
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,4,1,128,1,float16,float16,127,0.060787200927734375
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,4,1,128,1,float16,fp8,127,0.0619983971118927
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,4,1,128,1,float16,float16,255,0.07644000053405761
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,2,128,1,float16,fp8,63,0.012342400103807449
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,4,1,128,1,float16,float16,511,0.10462080240249634
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,1023,0.0171984001994133
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,4,1,128,1,float16,float16,1023,0.16953760385513306
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,4,1,128,1,float16,fp8,1023,0.16054400205612182
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,4,1,128,1,float16,float16,2047,0.28664000034332277
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,4,1,128,1,float16,fp8,2047,0.2548192024230957
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,4,2,128,1,float16,float16,1,0.08304799795150757
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,4,2,128,1,float16,fp8,32767,0.06906560063362122
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,4,2,128,1,float16,float16,3,0.08285760283470153
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,4,2,128,1,float16,fp8,3,0.0949072003364563
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,4,2,128,1,float16,float16,7,0.08292959928512574
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,4,2,128,1,float16,fp8,7,0.09503520131111146
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,4,2,128,1,float16,float16,15,0.08326719999313355
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,4,2,128,1,float16,fp8,15,0.09520000219345093
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,4,2,128,1,float16,float16,31,0.08286399841308593
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,4,2,128,1,float16,fp8,31,0.09526399970054626
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,4,2,128,1,float16,float16,63,0.08720960021018982
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,4,2,128,1,float16,fp8,63,0.09745759963989258
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,4,2,128,1,float16,float16,127,0.09091839790344239
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,4,1,128,1,float16,float16,63,0.055272001028060916
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,4,2,128,1,float16,fp8,127,0.10162880420684814
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,4,2,128,1,float16,float16,255,0.12007999420166016
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,4,2,128,1,float16,fp8,255,0.1283903956413269
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,4,2,128,1,float16,float16,511,0.17598880529403688
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,4,2,128,1,float16,fp8,511,0.18987679481506348
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,4,1,128,1,float16,fp8,255,0.07940000295639038
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,4,2,128,1,float16,float16,1023,0.2989423990249634
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,4,1,128,1,float16,fp8,511,0.10991679430007935
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,4,2,128,1,float16,fp8,1023,0.2876255989074707
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,4,1,128,1,float16,float16,1,0.09541760087013244
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,4,1,128,1,float16,float16,3,0.09439359903335572
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,4,2,128,1,float16,float16,2047,0.5406896114349365
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,4,1,128,1,float16,fp8,3,0.10601760149002075
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,4,1,128,1,float16,fp8,4095,0.23509759902954103
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,4,1,128,1,float16,float16,15,0.09523360133171081
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,4,1,128,1,float16,fp8,15,0.10694080591201782
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,4,1,128,1,float16,float16,31,0.09503840208053589
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,4,1,128,1,float16,fp8,31,0.10660959482192993
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,4,1,128,1,float16,float16,63,0.10040479898452759
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,4,1,128,1,float16,float16,1,0.034248000383377074
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,4,1,128,1,float16,fp8,63,0.10726239681243896
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,4,1,128,1,float16,fp8,127,0.11429760456085206
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,4,1,128,1,float16,float16,255,0.13363200426101685
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,4,1,128,1,float16,fp8,255,0.14125920534133912
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,4,1,128,1,float16,float16,511,0.18860960006713867
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,4,1,128,1,float16,fp8,511,0.20371360778808595
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,4,1,128,1,float16,fp8,1,0.10650880336761474
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,4,2,128,1,float16,fp8,2047,0.4745151996612549
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,4,1,128,1,float16,float16,1023,0.3117232084274292
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,4,2,128,1,float16,float16,1,0.14977120161056517
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,4,1,128,1,float16,fp8,7,0.106331205368042
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,4,2,128,1,float16,fp8,3,0.17219680547714233
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,4,2,128,1,float16,float16,7,0.14890719652175904
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,4,2,128,1,float16,fp8,1,0.17203359603881835
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,4,2,128,1,float16,fp8,1,0.09464160203933716
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,4,2,128,1,float16,fp8,7,0.17195680141448974
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,4,2,128,1,float16,float16,15,0.14921599626541138
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,4,2,128,1,float16,fp8,15,0.17246559858322144
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,4,2,128,1,float16,float16,31,0.1510048031806946
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,4,2,128,1,float16,fp8,31,0.17278720140457154
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,4,2,128,1,float16,float16,63,0.15692960023880004
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,4,2,128,1,float16,fp8,63,0.18248800039291382
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,4,2,128,1,float16,float16,127,0.1615216016769409
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,4,2,128,1,float16,fp8,127,0.18511680364608765
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,4,2,128,1,float16,float16,255,0.22040960788726807
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,4,2,128,1,float16,fp8,255,0.2374943971633911
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,4,2,128,1,float16,float16,511,0.32371840476989744
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,4,1,128,1,float16,float16,127,0.10408960580825806
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,1,0.012129600346088409
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,3,0.012118399888277055
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,1,128,1,float16,fp8,3,0.012768000364303589
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,7,0.011964800208806992
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,4,2,128,1,float16,float16,1023,0.5672080039978027
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,1,128,1,float16,fp8,7,0.012731200456619263
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,4,2,128,1,float16,fp8,1023,0.5567279815673828
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,1,128,1,float16,fp8,15,0.012811200320720672
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,15,0.011875200271606445
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,1,128,1,float16,fp8,31,0.01281599998474121
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,31,0.011872000247240066
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,1,128,1,float16,fp8,63,0.01286720037460327
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,127,0.011982399970293045
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,1,128,1,float16,fp8,127,0.01287200003862381
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,255,0.013571199774742127
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,1,128,1,float16,fp8,255,0.014776000380516052
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,511,0.016543999314308167
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,1,128,1,float16,fp8,511,0.018411199748516082
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,1023,0.01720159947872162
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,1,128,1,float16,fp8,1023,0.018532800674438476
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,2047,0.018908800184726716
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,1,128,1,float16,fp8,2047,0.018918399512767792
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,4095,0.022342400252819063
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,1,128,1,float16,fp8,4095,0.022617599368095397
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,8191,0.033632001280784606
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,1,128,1,float16,fp8,8191,0.030132800340652466
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,4,1,128,1,float16,float16,7,0.09553599953651429
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,16383,0.050672000646591185
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,1,128,1,float16,fp8,16383,0.0474016010761261
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,32767,0.08301119804382324
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,65535,0.14149119853973388
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,1,128,1,float16,fp8,65535,0.12265599966049194
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,1,0.012368000298738479
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,2,128,1,float16,fp8,1,0.01252640038728714
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,2,128,1,float16,fp8,3,0.012694400548934937
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,7,0.011926399916410447
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,2,128,1,float16,fp8,7,0.012675200402736665
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,15,0.011913599818944931
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,2,128,1,float16,fp8,15,0.01266079992055893
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,31,0.011776000261306763
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,2,128,1,float16,fp8,31,0.012511999905109405
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,63,0.011790399998426437
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,2,128,1,float16,fp8,63,0.01260959953069687
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,127,0.011844799667596818
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,2,128,1,float16,fp8,127,0.012727999687194824
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,255,0.013795199990272521
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,2,128,1,float16,fp8,255,0.014617599546909332
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,511,0.01669279932975769
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,2,128,1,float16,fp8,511,0.0183119997382164
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,63,0.011827199906110763
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,2,128,1,float16,fp8,1023,0.018374399840831758
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,2047,0.020604799687862396
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,2,128,1,float16,fp8,2047,0.0223471999168396
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,4095,0.031523200869560244
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,4,1,128,1,float16,fp8,1023,0.3026560068130493
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,2,128,1,float16,fp8,4095,0.028177601099014283
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,4,2,128,1,float16,float16,3,0.14906079769134523
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,8191,0.04808639883995056
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,2,128,1,float16,fp8,8191,0.04388799965381622
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,16383,0.07922239899635315
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,3,0.012054400146007537
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,32767,0.13971359729766847
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,2,128,1,float16,fp8,32767,0.11988639831542969
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,2,128,1,float16,fp8,65535,0.21579999923706056
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,1,0.01358879953622818
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,4,2,128,1,float16,fp8,511,0.3594367980957031
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,2,1,128,1,float16,fp8,1,0.014110399782657624
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,1,128,1,float16,fp8,1,0.01252799928188324
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,2,1,128,1,float16,fp8,3,0.01419840008020401
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,7,0.013414399325847625
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,2,1,128,1,float16,fp8,7,0.01420000046491623
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,15,0.013339200615882873
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,2,1,128,1,float16,fp8,15,0.014201599359512328
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,31,0.01316000074148178
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,2,1,128,1,float16,fp8,31,0.013950400054454803
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,63,0.01310880035161972
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,2,1,128,1,float16,fp8,63,0.013964800536632538
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,127,0.013153600692749023
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,2,1,128,1,float16,fp8,127,0.013993600010871887
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,1023,0.018873600661754607
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,2,1,128,1,float16,fp8,255,0.015863999724388123
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,511,0.018265600502490997
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,2,1,128,1,float16,fp8,511,0.019555200636386872
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,1023,0.022460800409317017
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,2,1,128,1,float16,fp8,1023,0.02367199957370758
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,2047,0.033011201024055484
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,4095,0.04797439873218536
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,2,1,128,1,float16,fp8,4095,0.044064000248909
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,8191,0.08110719919204712
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,1,128,1,float16,fp8,32767,0.0722383975982666
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,2,1,128,1,float16,fp8,8191,0.07088959813117982
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,16383,0.14338879585266112
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,2,1,128,1,float16,fp8,16383,0.12486239671707153
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,2,128,1,float16,fp8,16383,0.06881759762763977
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,32767,0.262172794342041
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,65535,0.4943984031677246
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,65535,0.2587167978286743
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,1,0.011286400258541107
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,2,1,128,1,float16,fp8,1,0.012035199999809265
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,2,1,128,1,float16,fp8,65535,0.41590561866760256
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,3,0.013291199505329133
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,7,0.011443199962377549
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,2,1,128,1,float16,fp8,7,0.012268800288438797
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,15,0.011473599821329117
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,2,1,128,1,float16,fp8,15,0.012110400199890136
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,131071,0.959108829498291
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,2,1,128,1,float16,fp8,131071,0.8031920433044434
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,2,1,128,1,float16,fp8,31,0.013764800131320953
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,2,1,128,1,float16,fp8,2047,0.029702401161193846
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,63,0.011323200166225433
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,127,0.011574400216341018
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,2,1,128,1,float16,fp8,127,0.012137600034475327
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,2,1,128,1,float16,fp8,255,0.014113600552082061
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,511,0.0159183993935585
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,2,1,128,1,float16,fp8,511,0.018052799999713896
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,1023,0.016649599373340606
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,2,1,128,1,float16,fp8,1023,0.017828799784183502
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,2047,0.016672000288963318
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,2,1,128,1,float16,fp8,2047,0.018067200481891633
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,4095,0.0168272003531456
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,2,1,128,1,float16,fp8,4095,0.018361599743366243
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,8191,0.018908800184726716
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,2,1,128,1,float16,fp8,32767,0.22234559059143066
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,2,1,128,1,float16,fp8,8191,0.020080000162124634
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,2,1,128,1,float16,fp8,16383,0.023361599445343016
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,32767,0.02375359982252121
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,2,1,128,1,float16,fp8,32767,0.023979200422763823
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,65535,0.028046399354934692
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,2,1,128,1,float16,fp8,65535,0.02784799933433533
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,3,0.011457599699497223
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,2,1,128,1,float16,fp8,3,0.012011200189590454
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,1,0.011460799723863602
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,2,1,128,1,float16,fp8,1,0.012211199849843979
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,3,0.011519999802112579
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,255,0.015272000432014465
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,2,1,128,1,float16,fp8,3,0.012132800370454788
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,31,0.011284799873828888
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,2,1,128,1,float16,fp8,7,0.012055999785661697
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,15,0.011271999776363372
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,2,1,128,1,float16,fp8,15,0.012064000219106674
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,31,0.011432000249624253
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,2,1,128,1,float16,fp8,31,0.012292800098657608
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,63,0.011478400230407715
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,2,1,128,1,float16,fp8,63,0.012230399996042252
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,127,0.011383999884128571
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,2,1,128,1,float16,fp8,127,0.012252800166606903
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,255,0.012932799756526947
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,2,1,128,1,float16,fp8,255,0.014209599792957306
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,511,0.015854400396347047
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,2,1,128,1,float16,fp8,511,0.01812800019979477
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,1023,0.016731199622154237
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,2,1,128,1,float16,fp8,1023,0.01791200041770935
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,2047,0.01661760061979294
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,2,1,128,1,float16,fp8,2047,0.018105599284172057
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,4095,0.016940799355506898
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,2,1,128,1,float16,fp8,4095,0.01825760006904602
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,8191,0.018985599279403687
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,2,1,128,1,float16,fp8,8191,0.019940799474716185
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,16383,0.023876799643039702
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,2,1,128,1,float16,fp8,16383,0.02332639992237091
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,32767,0.02720319926738739
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,2,1,128,1,float16,fp8,32767,0.02704479992389679
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,65535,0.03700959980487824
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,16383,0.022337600588798523
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,131071,0.052713602781295776
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,2,1,128,1,float16,fp8,131071,0.04935039877891541
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,1,0.012272000312805176
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,2,1,128,1,float16,fp8,1,0.013049599528312684
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,131071,0.03813599944114685
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,3,0.012247999757528305
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,7,0.012254399806261062
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,2,1,128,1,float16,fp8,131071,0.03607040047645569
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,2,1,128,1,float16,fp8,7,0.013046400249004364
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,15,0.01284639984369278
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,2,1,128,1,float16,fp8,15,0.013172799348831176
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,31,0.01284160017967224
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,2,1,128,1,float16,fp8,31,0.013219200074672699
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,63,0.012439999729394913
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,2,1,128,1,float16,fp8,63,0.01324319988489151
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,127,0.012675200402736665
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,7,0.011265599727630615
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,255,0.014340800046920777
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,2,1,128,1,float16,fp8,255,0.015216000378131866
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,511,0.017366400361061095
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,2,1,128,1,float16,fp8,511,0.018932799994945525
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,1023,0.01956319957971573
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,2,1,128,1,float16,fp8,1023,0.019393600523471832
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,2,1,128,1,float16,fp8,63,0.013896000385284425
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,2047,0.021598400175571443
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,2,1,128,1,float16,fp8,2047,0.023019200563430785
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,255,0.01311040073633194
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,2,1,128,1,float16,fp8,4095,0.029254400730133058
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,2,1,128,1,float16,fp8,8191,0.04469119906425476
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,16383,0.08436959981918335
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,2,1,128,1,float16,fp8,16383,0.0733456015586853
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,32767,0.14272799491882324
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,2,1,128,1,float16,fp8,32767,0.12427840232849122
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,65535,0.2611840009689331
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,2,1,128,1,float16,fp8,65535,0.22196478843688966
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2,1,2,1,128,1,float16,fp8,65535,0.033214399218559267
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,1,0.011400000005960465
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,2,1,128,1,float16,fp8,1,0.012070400267839431
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,3,0.011350400000810622
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,131071,0.4935152053833008
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,2,1,128,1,float16,fp8,3,0.01223199963569641
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,2,1,128,1,float16,fp8,7,0.012216000258922577
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,7,0.011446399986743927
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,15,0.011537600308656693
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,2,1,128,1,float16,fp8,15,0.012129600346088409
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,2,1,128,1,float16,fp8,31,0.012300799787044524
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,63,0.01168000027537346
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,2,1,128,1,float16,fp8,63,0.012430399656295776
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,2,1,128,1,float16,fp8,127,0.013284799456596375
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,2,1,128,1,float16,fp8,127,0.012355200201272964
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,255,0.013131199777126313
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,2,1,128,1,float16,fp8,255,0.014163200557231904
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,511,0.01595519930124283
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,2,1,128,1,float16,fp8,511,0.01796640008687973
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,1023,0.01671999990940094
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,2,1,128,1,float16,fp8,1023,0.01790879964828491
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,2047,0.016977599263191222
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,2,1,128,1,float16,fp8,2047,0.017979200184345245
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,4095,0.01720000058412552
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,2,1,128,1,float16,fp8,4095,0.01825760006904602
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,8191,0.020524799823760986
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,2,1,128,1,float16,fp8,8191,0.020259200036525725
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,16383,0.027134400606155396
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,2,1,128,1,float16,fp8,16383,0.02714880108833313
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,32767,0.03650560081005096
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,2,1,128,1,float16,fp8,32767,0.03319199979305267
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,65535,0.05275840163230896
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,2,1,128,1,float16,fp8,65535,0.04939840137958527
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,131071,0.08358880281448364
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,2,1,128,1,float16,fp8,131071,0.07640320062637329
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,1,0.014963200688362122
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,2,1,128,1,float16,fp8,1,0.01563359946012497
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,3,0.014473600685596466
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,2,1,128,1,float16,fp8,3,0.015403200685977936
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,7,0.014659200608730317
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,2,1,128,1,float16,fp8,7,0.015371200442314149
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,15,0.014752000570297241
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,2,1,128,1,float16,fp8,15,0.015433600544929505
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,31,0.014720000326633453
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,2,1,128,1,float16,fp8,31,0.015588800609111785
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,63,0.015011200308799743
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,2,1,128,1,float16,fp8,63,0.015387199819087982
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,127,0.014894400537014008
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,2,1,128,1,float16,fp8,127,0.015300799906253815
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,255,0.016505600512027742
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,2,1,128,1,float16,fp8,131071,0.4140511989593506
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,511,0.01951040029525757
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,2,1,128,1,float16,fp8,511,0.021771200001239777
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,2,1,128,1,float16,fp8,3,0.01337919980287552
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,31,0.011547199636697768
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,2047,0.04757919907569885
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,2,1,128,1,float16,fp8,2047,0.042531201243400575
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,127,0.011699199676513672
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,4095,0.07874559760093688
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,2,1,128,1,float16,fp8,4095,0.06715840101242065
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,8191,0.13682559728622437
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,2,1,128,1,float16,fp8,8191,0.1181488037109375
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,4095,0.03253439962863922
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,8191,0.04907999932765961
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,16383,0.25610880851745604
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,2,1,128,1,float16,fp8,16383,0.2139280080795288
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,32767,0.4879136085510254
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,2,1,128,1,float16,fp8,32767,0.4078479766845703
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,2,1,128,1,float16,float16,1,0.021049599349498748
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,2,1,128,1,float16,fp8,1,0.022968000173568724
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,2,1,128,1,float16,float16,3,0.021092799305915833
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,2,1,128,1,float16,fp8,3,0.023148800432682037
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,2,1,128,1,float16,float16,7,0.020931200683116914
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,2,1,128,1,float16,fp8,7,0.02263839989900589
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,2,1,128,1,float16,float16,15,0.021044799685478212
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,2,1,128,1,float16,fp8,15,0.022724799811840057
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,2,1,128,1,float16,float16,31,0.020692799985408784
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,2,1,128,1,float16,fp8,31,0.023039999604225158
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,65535,0.9543600082397461
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,2,1,128,1,float16,fp8,63,0.022767999768257143
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,2,1,128,1,float16,fp8,127,0.02279520034790039
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,2,1,128,1,float16,float16,127,0.02078080028295517
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,2,1,128,1,float16,float16,255,0.024721600115299225
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,2,1,128,1,float16,fp8,255,0.02967199981212616
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,1023,0.031758400797843936
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,2,1,128,1,float16,fp8,1023,0.02938399910926819
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,2,1,128,1,float16,float16,1023,0.05375360250473023
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,2,1,128,1,float16,fp8,511,0.03613280057907105
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,2,1,128,1,float16,fp8,1023,0.05048959851264954
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,2,1,128,1,float16,float16,2047,0.08328800201416016
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,2,1,128,1,float16,fp8,2047,0.07689279913902283
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,2,1,128,1,float16,float16,4095,0.14202879667282103
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,2,1,128,1,float16,fp8,4095,0.12423360347747803
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,2,1,128,1,float16,float16,8191,0.26075680255889894
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,2,1,128,1,float16,fp8,8191,0.22251360416412352
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,2,1,128,1,float16,float16,1,0.03416799902915955
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,2,1,128,1,float16,fp8,1,0.03709439933300018
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,2,1,128,1,float16,float16,16383,0.49086880683898926
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,2,1,128,1,float16,float16,3,0.03413600027561188
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,2,1,128,1,float16,fp8,16383,0.42272157669067384
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,2,1,128,1,float16,fp8,3,0.03729119896888733
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,2,1,128,1,float16,float16,15,0.034118399024009705
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,2,1,128,1,float16,fp8,15,0.037390398979187014
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,2,1,128,1,float16,float16,31,0.03425759971141815
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,2,1,128,1,float16,fp8,31,0.03745119869709015
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,2,1,128,1,float16,float16,63,0.034467199444770814
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,2,1,128,1,float16,fp8,63,0.037662398815155027
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,2,1,128,1,float16,float16,127,0.034748798608779906
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,2,1,128,1,float16,fp8,127,0.037536001205444335
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,2,1,128,1,float16,float16,63,0.02078240066766739
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,2,1,128,1,float16,fp8,65535,0.7946224212646484
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,2,1,128,1,float16,float16,255,0.0465472012758255
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,2,1,128,1,float16,fp8,255,0.04503040015697479
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,128,1,2,1,128,1,float16,fp8,255,0.01724800020456314
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,2,1,128,1,float16,fp8,511,0.062243199348449706
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,2,1,128,1,float16,float16,511,0.06364960074424744
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,2,1,128,1,float16,float16,1023,0.09542400240898133
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,256,1,2,1,128,1,float16,float16,511,0.035743999481201175
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,2,1,128,1,float16,float16,2047,0.15308480262756347
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,2,1,128,1,float16,fp8,2047,0.13637919425964357
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,1,0.011699199676513672
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,2,1,128,1,float16,float16,4095,0.2725503921508789
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,2,1,128,1,float16,fp8,4095,0.23479681015014647
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,2,1,128,1,float16,fp8,1,0.012387199699878693
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,3,0.011641599982976914
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,2,1,128,1,float16,fp8,3,0.012481600046157837
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,2,1,128,1,float16,fp8,7,0.012332800030708312
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,2,1,128,1,float16,float16,8191,0.49686079025268554
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,2,1,128,1,float16,fp8,15,0.01234079971909523
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,2,1,128,1,float16,fp8,8191,0.43327040672302247
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,2,1,128,1,float16,fp8,31,0.012436799705028534
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,31,0.011483199894428253
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,2,1,128,1,float16,fp8,63,0.012503999471664428
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,63,0.011838400363922119
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,127,0.01156959980726242
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,255,0.013409599661827087
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,2,1,128,1,float16,fp8,255,0.01424960047006607
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,511,0.016345599293708803
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,2,1,128,1,float16,fp8,511,0.01783519983291626
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,1023,0.01725600063800812
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,2,1,128,1,float16,fp8,1023,0.018198400735855103
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,2,1,128,1,float16,float16,7,0.034062400460243225
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,2047,0.01740960031747818
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,2,1,128,1,float16,fp8,2047,0.018321600556373597
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,4095,0.019009600579738616
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,8191,0.024038399755954742
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,2,1,128,1,float16,fp8,7,0.03764480054378509
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,2,1,128,1,float16,fp8,8191,0.024374400079250336
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,16383,0.037302398681640626
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,2,1,128,1,float16,fp8,16383,0.033529600501060484
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,2,1,128,1,float16,fp8,32767,0.04932160079479218
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,65535,0.08482239842414856
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,2,1,128,1,float16,fp8,65535,0.07569599747657776
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,131071,0.1464319944381714
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,2,1,128,1,float16,fp8,131071,0.12784639596939087
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,2,1,128,1,float16,float16,1,0.055144000053405764
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,2,1,128,1,float16,fp8,1,0.06123039722442627
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,2,1,128,1,float16,float16,3,0.05528479814529419
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,2,1,128,1,float16,fp8,3,0.06093760132789612
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,2,1,128,1,float16,float16,7,0.05498239994049072
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,2,1,128,1,float16,fp8,7,0.061110401153564455
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,2,1,128,1,float16,float16,15,0.055255997180938723
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,2,1,128,1,float16,fp8,15,0.06130399703979492
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,7,0.011577600240707397
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,2,1,128,1,float16,float16,31,0.05537440180778504
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,15,0.01167839989066124
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,2,1,128,1,float16,float16,63,0.055687999725341795
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,2,1,128,1,float16,fp8,63,0.06104480028152466
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,2,1,128,1,float16,float16,127,0.060046398639678956
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,2,1,128,1,float16,fp8,127,0.061185598373413086
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,2,1,128,1,float16,fp8,127,0.012459199875593185
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,2,1,128,1,float16,float16,255,0.07624800205230713
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,2,1,128,1,float16,fp8,255,0.07874879837036133
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,2,1,128,1,float16,float16,511,0.10504640340805053
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,2,1,128,1,float16,fp8,511,0.11035360097885132
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,2,1,128,1,float16,float16,1023,0.17021759748458862
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,2,1,128,1,float16,fp8,4095,0.01865919977426529
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,2,1,128,1,float16,fp8,1023,0.16084640026092528
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,2,1,128,1,float16,float16,2047,0.28672959804534914
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,32767,0.0525983989238739
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,2,1,128,1,float16,fp8,2047,0.25275359153747556
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,2,1,128,1,float16,float16,1,0.09523040056228638
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,2,1,128,1,float16,float16,3,0.09435679912567138
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,2,1,128,1,float16,float16,4095,0.5169360160827636
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,2,1,128,1,float16,fp8,3,0.107150399684906
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,2,1,128,1,float16,fp8,4095,0.4499519824981689
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,2,1,128,1,float16,float16,7,0.09439679980278015
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,2,1,128,1,float16,float16,15,0.09517120122909546
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,2,1,128,1,float16,fp8,7,0.10617599487304688
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,2,1,128,1,float16,float16,31,0.09443519711494446
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,2,1,128,1,float16,fp8,15,0.10636639595031738
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,2,1,128,1,float16,fp8,31,0.10707839727401733
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,2,1,128,1,float16,fp8,63,0.10712319612503052
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,2,1,128,1,float16,float16,63,0.10009759664535522
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,2,1,128,1,float16,float16,127,0.1034816026687622
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,2,1,128,1,float16,fp8,127,0.11408159732818604
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,512,1,2,1,128,1,float16,fp8,1023,0.08862239718437195
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,2,1,128,1,float16,float16,255,0.13181760311126708
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,2,1,128,1,float16,fp8,255,0.14165600538253784
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,2,1,128,1,float16,float16,511,0.18778560161590577
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,2,1,128,1,float16,fp8,511,0.20426719188690184
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,1,0.011902400106191636
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,2,1,128,1,float16,fp8,1,0.012531200051307678
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,2,1,128,1,float16,float16,1023,0.31064319610595703
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,2,1,128,1,float16,fp8,1023,0.30103039741516113
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,2,1,128,1,float16,fp8,3,0.01250080019235611
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,7,0.011633600294589996
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,1024,1,2,1,128,1,float16,fp8,31,0.06082720160484314
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,2,1,128,1,float16,fp8,7,0.012559999525547028
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,15,0.011966399848461151
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,2,1,128,1,float16,fp8,15,0.012641599774360657
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,2,1,128,1,float16,fp8,31,0.01273760050535202
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,63,0.01196959987282753
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,127,0.012038400024175644
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,2,1,128,1,float16,fp8,63,0.012462399899959564
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,2,1,128,1,float16,fp8,127,0.012563200294971466
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,255,0.013631999492645264
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,2,1,128,1,float16,fp8,255,0.014375999569892883
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,2,1,128,1,float16,fp8,511,0.01863519996404648
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,1023,0.01733279973268509
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,2,1,128,1,float16,fp8,1023,0.018467199802398682
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,2047,0.019072000682353974
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,2,1,128,1,float16,fp8,2047,0.018651199340820313
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,4095,0.022484800219535826
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,8191,0.033616000413894655
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,2,1,128,1,float16,fp8,8191,0.03007200062274933
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,16383,0.05154399871826172
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,2,1,128,1,float16,fp8,16383,0.04749920070171356
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,32767,0.08340799808502197
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,2,1,128,1,float16,fp8,32767,0.0725823998451233
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,65535,0.1422719955444336
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,2,1,128,1,float16,fp8,65535,0.12306720018386841
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,131071,0.26122241020202636
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,3,0.011751999706029892
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,2,1,128,1,float16,float16,2047,0.5418911933898926
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,2,1,128,1,float16,fp8,131071,0.21989920139312744
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,2,1,128,1,float16,fp8,2047,0.4883408069610596
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,511,0.016641600430011748
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,2,1,128,1,float16,fp8,4095,0.02242079973220825
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,2048,1,2,1,128,1,float16,fp8,1,0.10630079507827758
SGLang,0.5.9,NVIDIA H200,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,31,0.011686400324106217
